From 46f99c95f28d181856368acb6af4e628383b46e6 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 20:28:46 +0000 Subject: [PATCH 01/38] test(engine-ipc): pin what rocm and rocmd each send an engine child and how they read its answer rocm and rocmd each carry their own client for the JSON-over-stdio request to an engine process, and the two have drifted: they build the child's command line differently, set different environment, treat a failing exit status differently, word their errors differently, and only rocm guards the request write against SIGPIPE. Nothing pinned any of it, so the first step toward one client is to pin what each does today. Both clients are driven against a stand-in engine child, a shell script that records its argv, environment and stdin and then answers (or crashes) as the test asks. The stand-in lives in rocm-engine-protocol behind a test-support feature so the two callers share one. Each caller's out-of-process half is split into a small function that takes the executable to run (engine_stdio_child_request in rocm, engine_request_via in rocmd), so a test can aim it at the stand-in; the bodies are unchanged and the public entry points call them with the same arguments as before. The tests assert the child's argv (rocm: stdio; rocmd: __engine-stdio ), the single newline-terminated request line, the environment each caller sets (rocm: the app path variables, the engine environments root, the progress variable for installs; rocmd: the endpoint key file when it holds a usable key), what each does with a nonzero exit, an unparseable answer and an error envelope, and the SIGPIPE case under the default action in a re-executed child. Mutating rocm's write guard away turns its SIGPIPE test red. No behaviour change. Signed-off-by: Jussi Elo --- Cargo.lock | 2 + apps/rocm/Cargo.toml | 2 + apps/rocm/src/engine_ipc.rs | 303 +++++++++++++++++- apps/rocmd/Cargo.toml | 8 + apps/rocmd/src/engine_ipc.rs | 249 +++++++++++++- crates/rocm-engine-protocol/Cargo.toml | 6 + .../rocm-engine-protocol/src/fake_engine.rs | 138 ++++++++ crates/rocm-engine-protocol/src/lib.rs | 3 + 8 files changed, 708 insertions(+), 3 deletions(-) create mode 100644 crates/rocm-engine-protocol/src/fake_engine.rs diff --git a/Cargo.lock b/Cargo.lock index 7071a703eb..96dc60ebad 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3905,6 +3905,7 @@ dependencies = [ name = "rocm-engine-protocol" version = "0.1.0" dependencies = [ + "anyhow", "rocm-core", "serde", "serde_json", @@ -3930,6 +3931,7 @@ dependencies = [ "anyhow", "axum", "clap", + "libc", "rocm-core", "rocm-engine-protocol", "serde", diff --git a/apps/rocm/Cargo.toml b/apps/rocm/Cargo.toml index e48eb7cf8a..f4eb6b32ec 100644 --- a/apps/rocm/Cargo.toml +++ b/apps/rocm/Cargo.toml @@ -74,6 +74,8 @@ apple-native-keyring-store = { version = "1.0", features = ["keychain"] } zbus-secret-service-keyring-store = { version = "1.0", features = ["rt-tokio-crypto-rust"] } [dev-dependencies] +# The fake engine child both engine IPC callers are tested against. +rocm-engine-protocol = { path = "../../crates/rocm-engine-protocol", features = ["test-support"] } # Property-based tests for the byte-size formatter behind the `download: # approved up to …` line. "A size is printed in its own unit" has to hold at # every unit boundary, and the narrow band where `{:.1}` rounding reaches 1024.0 diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index 3e9565d00e..ab82720bab 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -146,6 +146,22 @@ where "unable to locate engine binary for {engine}; build the workspace or install the engine package" ) })?; + engine_stdio_child_request(&engine_binary, paths, &envelope, env_root, stream_progress) +} + +/// The out-of-process half of [`engine_request_with_env_root`]: run +/// `engine_binary stdio`, send `envelope` on its stdin and decode its answer. +/// Split out so a test can point it at a stand-in binary. +fn engine_stdio_child_request( + engine_binary: &Path, + paths: Option<&AppPaths>, + envelope: &EngineRequestEnvelope, + env_root: Option<&Path>, + stream_progress: bool, +) -> Result +where + R: DeserializeOwned, +{ let mut command = ProcessCommand::new(engine_binary); command.arg("stdio"); if let Some(paths) = paths { @@ -176,7 +192,7 @@ where // surface as the normal "failed to write engine request" error and then // let `wait()` report why the engine exited. with_sigpipe_ignored(|| { - serde_json::to_writer(&mut stdin, &envelope) + serde_json::to_writer(&mut stdin, envelope) .context("failed to write engine request")?; stdin.write_all(b"\n")?; Ok::<(), anyhow::Error>(()) @@ -490,4 +506,289 @@ mod tests { assert_eq!(discovered, Some(primary_path)); Ok(()) } + + // ---- The stdio client, driven against a stand-in engine child. ---- + // + // These pin what `rocm` puts on an engine child's command line, environment + // and stdin and how it reads the answer back. `rocmd` keeps its own client + // with a different contract; its tests are in `apps/rocmd/src/engine_ipc.rs`. + + fn envelope(method: EngineMethod, payload: serde_json::Value) -> EngineRequestEnvelope { + EngineRequestEnvelope { method, payload } + } + + #[test] + fn decode_engine_response_reports_each_way_an_answer_can_be_unusable() { + let failed = EngineResponseEnvelope { + ok: false, + data: None, + error: Some(rocm_engine_protocol::EngineErrorDetail { + code: "stop_failed".to_owned(), + message: "it broke".to_owned(), + }), + }; + let error = decode_engine_response::(failed).unwrap_err(); + assert_eq!(error.to_string(), "stop_failed: it broke"); + + let unexplained = EngineResponseEnvelope { + ok: false, + data: None, + error: None, + }; + let error = decode_engine_response::(unexplained).unwrap_err(); + assert_eq!(error.to_string(), "unknown engine error"); + + let empty = EngineResponseEnvelope { + ok: true, + data: None, + error: None, + }; + let error = decode_engine_response::(empty).unwrap_err(); + assert_eq!( + error.to_string(), + "engine response envelope did not contain data" + ); + + let wrong_shape = EngineResponseEnvelope::success(serde_json::json!({"stopped": "yes"})); + let error = + decode_engine_response::(wrong_shape).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to decode engine response payload" + ); + } + + #[cfg(unix)] + mod stdio_child { + use super::*; + use rocm_engine_protocol::fake_engine::FakeEngine; + use serde_json::{Value, json}; + use std::os::unix::process::ExitStatusExt as _; + + const STOPPED: &str = r#"{"ok":true,"data":{"stopped":true}}"#; + + fn stop_request() -> EngineRequestEnvelope { + envelope(EngineMethod::Stop, json!({"service_id": "svc"})) + } + + fn request( + engine: &FakeEngine, + paths: Option<&AppPaths>, + env_root: Option<&Path>, + stream_progress: bool, + ) -> Result { + engine.run(|| { + engine_stdio_child_request::( + engine.path(), + paths, + &stop_request(), + env_root, + stream_progress, + ) + }) + } + + fn ambient(key: &str) -> Option { + std::env::var(key).ok() + } + + #[test] + fn the_child_gets_the_stdio_verb_and_one_line_of_request() { + let engine = FakeEngine::answering("rocm-framing", STOPPED, 0); + let answer = request(&engine, None, None, false).unwrap(); + + assert_eq!(answer, json!({"stopped": true})); + assert_eq!(engine.argv(), ["stdio"]); + assert_eq!( + engine.stdin(), + "{\"method\":\"stop\",\"payload\":{\"service_id\":\"svc\"}}\n" + ); + } + + #[test] + fn the_child_runs_on_the_callers_app_paths_when_given_them() { + let (root, paths) = crate::tests::test_paths("engine-ipc-app-paths"); + let engine = FakeEngine::answering("rocm-app-paths", STOPPED, 0); + request(&engine, Some(&paths), None, false).unwrap(); + let _ = fs::remove_dir_all(root); + + assert_eq!( + engine.env("ROCM_CLI_CONFIG_DIR").as_deref(), + paths.config_dir.to_str() + ); + assert_eq!( + engine.env("ROCM_CLI_DATA_DIR").as_deref(), + paths.data_dir.to_str() + ); + assert_eq!( + engine.env("ROCM_CLI_CACHE_DIR").as_deref(), + paths.cache_dir.to_str() + ); + } + + #[test] + fn the_child_inherits_the_callers_environment_when_no_app_paths_are_given() { + let engine = FakeEngine::answering("rocm-no-app-paths", STOPPED, 0); + request(&engine, None, None, false).unwrap(); + + for key in [ + "ROCM_CLI_CONFIG_DIR", + "ROCM_CLI_DATA_DIR", + "ROCM_CLI_CACHE_DIR", + ] { + assert_eq!(engine.env(key), ambient(key), "{key}"); + } + } + + #[test] + fn an_env_root_reaches_the_child_as_the_engine_envs_root() { + let engine = FakeEngine::answering("rocm-env-root", STOPPED, 0); + let env_root = Path::new("/an/engine/envs/root"); + request(&engine, None, Some(env_root), false).unwrap(); + assert_eq!( + engine.env("ROCM_CLI_ENGINE_ENVS_ROOT").as_deref(), + Some("/an/engine/envs/root") + ); + + let without = FakeEngine::answering("rocm-no-env-root", STOPPED, 0); + request(&without, None, None, false).unwrap(); + assert_eq!( + without.env("ROCM_CLI_ENGINE_ENVS_ROOT"), + ambient("ROCM_CLI_ENGINE_ENVS_ROOT") + ); + } + + #[test] + fn progress_streaming_is_asked_for_only_when_the_caller_wants_it() { + let streaming = FakeEngine::answering("rocm-progress-on", STOPPED, 0); + request(&streaming, None, None, true).unwrap(); + assert_eq!( + streaming.env("ROCM_ENGINE_PROGRESS_STDERR").as_deref(), + Some("1") + ); + + let quiet = FakeEngine::answering("rocm-progress-off", STOPPED, 0); + request(&quiet, None, None, false).unwrap(); + assert_eq!( + quiet.env("ROCM_ENGINE_PROGRESS_STDERR"), + ambient("ROCM_ENGINE_PROGRESS_STDERR") + ); + } + + #[test] + fn the_endpoint_key_is_not_passed_to_the_child() { + // `rocm`'s requests are not about a running service's endpoint (detect, + // install, resolve-model, stop), so this client has no service to find + // a key for. `rocmd`'s healthcheck is the call that threads one. + let engine = FakeEngine::answering("rocm-no-key", STOPPED, 0); + request(&engine, None, None, false).unwrap(); + assert_eq!( + engine.env(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV), + ambient(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV) + ); + } + + #[test] + fn a_nonzero_exit_with_no_answer_reports_the_status_and_stderr() { + let engine = FakeEngine::new("rocm-crash", true, "echo boom >&2\nexit 3"); + let error = request(&engine, None, None, false).unwrap_err(); + assert_eq!( + error.to_string(), + "engine stdio process exited with status exit status: 3: boom" + ); + + let silent = FakeEngine::new("rocm-silent-crash", true, "exit 4"); + let error = request(&silent, None, None, false).unwrap_err(); + assert_eq!( + error.to_string(), + "engine stdio process exited with status exit status: 4" + ); + } + + #[test] + fn a_nonzero_exit_that_still_answered_is_decoded_not_discarded() { + // The engine's own error envelope is the useful diagnostic, so an + // answer on stdout wins over the exit status. + let engine = FakeEngine::answering( + "rocm-answers-then-fails", + r#"{"ok":false,"error":{"code":"stop_failed","message":"it broke"}}"#, + 1, + ); + let error = request(&engine, None, None, false).unwrap_err(); + assert_eq!(error.to_string(), "stop_failed: it broke"); + + let engine = FakeEngine::answering("rocm-answers-then-exits-2", STOPPED, 2); + assert_eq!( + request(&engine, None, None, false).unwrap(), + json!({"stopped": true}) + ); + } + + #[test] + fn an_unparseable_answer_carries_the_childs_stderr() { + let engine = FakeEngine::new( + "rocm-garbage", + true, + "printf 'not json'\necho oops >&2\nexit 0", + ); + let error = request(&engine, None, None, false).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to parse engine response envelope; stderr: oops" + ); + + let silent = FakeEngine::new("rocm-garbage-quiet", true, "printf 'not json'"); + let error = request(&silent, None, None, false).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to parse engine response envelope" + ); + } + + /// The write of a request to an engine that has already exited must come + /// back as an error even where SIGPIPE is at its default action, which is + /// how `main` runs `rocm`. Re-executes this test in a fresh process that + /// installs `SIG_DFL` first (the test harness leaves SIGPIPE ignored, and + /// changing it process-wide would race sibling tests). + #[test] + fn a_request_to_an_engine_that_already_exited_is_an_error_under_sig_dfl() { + if std::env::var_os("ROCM_TEST_ENGINE_SIGPIPE_ARM").is_some() { + crate::reset_sigpipe(); + let engine = FakeEngine::new("rocm-sigpipe", false, "exit 3"); + // Larger than a pipe buffer, so the write reaches the closed read end. + let big = "x".repeat(1 << 20); + let result = engine.run(|| { + engine_stdio_child_request::( + engine.path(), + None, + &envelope(EngineMethod::Stop, json!({ "blob": big })), + None, + false, + ) + }); + let error = result.expect_err("the engine never answered"); + assert!( + format!("{error:#}").contains("failed to write engine request"), + "{error:#}" + ); + return; + } + + let status = ProcessCommand::new(std::env::current_exe().unwrap()) + .args([ + "engine_ipc::tests::stdio_child::a_request_to_an_engine_that_already_exited_is_an_error_under_sig_dfl", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_ENGINE_SIGPIPE_ARM", "1") + .status() + .unwrap(); + assert_eq!( + status.signal(), + None, + "SIGPIPE killed the process instead of surfacing as an error: {status:?}" + ); + assert!(status.success(), "{status:?}"); + } + } } diff --git a/apps/rocmd/Cargo.toml b/apps/rocmd/Cargo.toml index 2f0064e18a..7219350fa8 100644 --- a/apps/rocmd/Cargo.toml +++ b/apps/rocmd/Cargo.toml @@ -27,3 +27,11 @@ ureq = { version = "2.12", features = ["native-certs"] } [target.'cfg(windows)'.dependencies] windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } + +[dev-dependencies] +# The fake engine child both engine IPC callers are tested against. +rocm-engine-protocol = { path = "../../crates/rocm-engine-protocol", features = ["test-support"] } + +[target.'cfg(unix)'.dev-dependencies] +# The SIGPIPE disposition a test restores in a re-executed child. +libc.workspace = true diff --git a/apps/rocmd/src/engine_ipc.rs b/apps/rocmd/src/engine_ipc.rs index 32f1834547..a1fc67018e 100644 --- a/apps/rocmd/src/engine_ipc.rs +++ b/apps/rocmd/src/engine_ipc.rs @@ -120,7 +120,22 @@ where }; let engine_binary = std::env::current_exe().context("failed to resolve current rocm executable path")?; - let mut command = ProcessCommand::new(&engine_binary); + engine_request_via(&engine_binary, paths, engine, service_id, &envelope) +} + +/// [`engine_request`] with the executable that serves `__engine-stdio` named by +/// the caller, so a test can point it at a stand-in. +fn engine_request_via( + engine_binary: &std::path::Path, + paths: &AppPaths, + engine: &str, + service_id: &str, + envelope: &EngineRequestEnvelope, +) -> Result +where + R: DeserializeOwned, +{ + let mut command = ProcessCommand::new(engine_binary); command .arg("__engine-stdio") .arg(engine) @@ -148,7 +163,7 @@ where .stdin .as_mut() .context("engine stdio child did not expose stdin")?; - serde_json::to_writer(&mut *stdin, &envelope).context("failed to write engine request")?; + serde_json::to_writer(&mut *stdin, envelope).context("failed to write engine request")?; stdin.write_all(b"\n")?; } @@ -341,4 +356,234 @@ mod tests { }; assert!(!healthcheck_response_ready(&listing_only)); } + + // ---- The stdio client, driven against a stand-in engine child. ---- + // + // These pin what `rocmd` puts on the child's command line, environment and + // stdin and how it reads the answer back. `rocm` keeps its own client with a + // different contract; its tests are in `apps/rocm/src/engine_ipc.rs`. + #[cfg(unix)] + mod stdio_child { + use super::*; + use rocm_engine_protocol::fake_engine::FakeEngine; + use serde_json::{Value, json}; + use std::os::unix::process::ExitStatusExt as _; + + const SERVICE_ID: &str = "svc-stdio-child"; + const STOPPED: &str = r#"{"ok":true,"data":{"stopped":true}}"#; + + fn healthcheck_envelope() -> EngineRequestEnvelope { + EngineRequestEnvelope { + method: EngineMethod::Healthcheck, + payload: json!({"service_id": SERVICE_ID}), + } + } + + fn request(engine: &FakeEngine, paths: &AppPaths) -> Result { + engine.run(|| { + engine_request_via::( + engine.path(), + paths, + "vllm", + SERVICE_ID, + &healthcheck_envelope(), + ) + }) + } + + fn ambient(key: &str) -> Option { + std::env::var(key).ok() + } + + #[test] + fn the_child_is_asked_to_serve_the_engine_over_stdio_with_one_line_of_request() { + let (_root, paths) = temp_app_paths("stdio-framing"); + let engine = FakeEngine::answering("rocmd-framing", STOPPED, 0); + let answer = request(&engine, &paths).unwrap(); + + assert_eq!(answer, json!({"stopped": true})); + assert_eq!(engine.argv(), ["__engine-stdio", "vllm"]); + assert_eq!( + engine.stdin(), + "{\"method\":\"healthcheck\",\"payload\":{\"service_id\":\"svc-stdio-child\"}}\n" + ); + } + + #[test] + fn the_endpoint_key_file_reaches_the_child_only_when_it_holds_a_usable_key() { + let (_root, paths) = temp_app_paths("stdio-endpoint-key"); + let key_env = rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV; + + let keyless = FakeEngine::answering("rocmd-keyless", STOPPED, 0); + request(&keyless, &paths).unwrap(); + assert_eq!(keyless.env(key_env), ambient(key_env)); + + let key_path = rocm_engine_protocol::endpoint_key_file_path(&paths, SERVICE_ID); + fs::create_dir_all(paths.services_dir()).unwrap(); + fs::write(&key_path, " \n").unwrap(); + let unusable = FakeEngine::answering("rocmd-unusable-key", STOPPED, 0); + request(&unusable, &paths).unwrap(); + assert_eq!(unusable.env(key_env), ambient(key_env)); + + fs::write(&key_path, "secret-key").unwrap(); + let keyed = FakeEngine::answering("rocmd-keyed", STOPPED, 0); + request(&keyed, &paths).unwrap(); + assert_eq!(keyed.env(key_env).as_deref(), key_path.to_str()); + } + + #[test] + fn the_child_is_not_given_the_daemons_app_paths() { + // It relies on the environment it inherits to find the same folders + // the daemon runs on; nothing here sets `ROCM_CLI_*_DIR` for it. + let (_root, paths) = temp_app_paths("stdio-no-app-paths"); + let engine = FakeEngine::answering("rocmd-no-app-paths", STOPPED, 0); + request(&engine, &paths).unwrap(); + for key in [ + "ROCM_CLI_CONFIG_DIR", + "ROCM_CLI_DATA_DIR", + "ROCM_CLI_CACHE_DIR", + ] { + assert_eq!(engine.env(key), ambient(key), "{key}"); + } + } + + #[test] + fn any_nonzero_exit_is_an_error_even_when_the_child_answered() { + let (_root, paths) = temp_app_paths("stdio-nonzero"); + let crash = FakeEngine::new("rocmd-crash", true, "echo boom >&2\nexit 3"); + assert_eq!( + request(&crash, &paths).unwrap_err().to_string(), + "engine stdio process exited with status exit status: 3: boom" + ); + + let silent = FakeEngine::new("rocmd-silent-crash", true, "exit 4"); + assert_eq!( + request(&silent, &paths).unwrap_err().to_string(), + "engine stdio process exited with status exit status: 4" + ); + + // `rocm` decodes an answer that came with a failing exit; this client + // discards it for the status. + let answered = FakeEngine::answering("rocmd-answers-then-fails", STOPPED, 2); + assert_eq!( + request(&answered, &paths).unwrap_err().to_string(), + "engine stdio process exited with status exit status: 2" + ); + } + + #[test] + fn an_unusable_answer_is_reported_in_this_clients_own_words() { + let (_root, paths) = temp_app_paths("stdio-answers"); + + let garbage = + FakeEngine::new("rocmd-garbage", true, "printf 'not json'\necho oops >&2"); + assert_eq!( + request(&garbage, &paths).unwrap_err().to_string(), + "failed to parse engine response" + ); + + let failed = FakeEngine::answering( + "rocmd-failed", + r#"{"ok":false,"error":{"code":"stop_failed","message":"it broke"}}"#, + 0, + ); + assert_eq!( + request(&failed, &paths).unwrap_err().to_string(), + "stop_failed: it broke" + ); + + let unexplained = FakeEngine::answering("rocmd-unexplained", r#"{"ok":false}"#, 0); + assert_eq!( + request(&unexplained, &paths).unwrap_err().to_string(), + "unknown engine error" + ); + + let empty = FakeEngine::answering("rocmd-empty", r#"{"ok":true}"#, 0); + assert_eq!( + request(&empty, &paths).unwrap_err().to_string(), + "engine response did not include data" + ); + + let wrong = FakeEngine::answering("rocmd-wrong", r#"{"ok":true,"data":[]}"#, 0); + let error = engine_request_via::( + wrong.path(), + &paths, + "vllm", + SERVICE_ID, + &healthcheck_envelope(), + ) + .unwrap_err(); + assert_eq!( + error.to_string(), + "failed to deserialize engine response data" + ); + } + + /// Writing a request to a child that has already exited, in this client: + /// where SIGPIPE is ignored (Rust's startup default, which the test + /// harness keeps) the write fails as an ordinary error, and where it is + /// at its default action (`rocm` resets it to that in `main`, and `rocm + /// daemon run` runs this code in that process) the write kills the + /// process. This client has no guard around the write, so the second half + /// pins that gap. Re-executes this test in a fresh process for each case, + /// since changing the disposition process-wide would race sibling tests. + #[test] + #[allow(unsafe_code)] // libc::signal, to restore the default action in the child + fn a_request_to_a_child_that_already_exited_depends_on_the_sigpipe_disposition() { + if let Some(arm) = std::env::var_os("ROCMD_TEST_ENGINE_SIGPIPE_ARM") { + if arm == "default-action" { + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + } + let (_root, paths) = temp_app_paths("stdio-sigpipe"); + let engine = FakeEngine::new("rocmd-sigpipe", false, "exit 3"); + // Larger than a pipe buffer, so the write reaches the closed read end. + let big = "x".repeat(1 << 20); + let envelope = EngineRequestEnvelope { + method: EngineMethod::Healthcheck, + payload: json!({ "blob": big }), + }; + let result = engine.run(|| { + engine_request_via::( + engine.path(), + &paths, + "vllm", + SERVICE_ID, + &envelope, + ) + }); + let error = result.expect_err("the engine never answered"); + assert!( + format!("{error:#}").contains("failed to write engine request"), + "{error:#}" + ); + return; + } + + let run_arm = |arm: &str| { + std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "engine_ipc::tests::stdio_child::a_request_to_a_child_that_already_exited_depends_on_the_sigpipe_disposition", + "--exact", + "--nocapture", + ]) + .env("ROCMD_TEST_ENGINE_SIGPIPE_ARM", arm) + .status() + .unwrap() + }; + + let ignored = run_arm("ignored"); + assert!(ignored.success(), "{ignored:?}"); + + let default_action = run_arm("default-action"); + assert_eq!( + default_action.signal(), + Some(libc::SIGPIPE), + "sanity check: this client has no SIGPIPE guard, so under the default action the \ + write kills the process ({default_action:?})" + ); + } + } } diff --git a/crates/rocm-engine-protocol/Cargo.toml b/crates/rocm-engine-protocol/Cargo.toml index 19bd5f483a..6a258001c0 100644 --- a/crates/rocm-engine-protocol/Cargo.toml +++ b/crates/rocm-engine-protocol/Cargo.toml @@ -10,7 +10,13 @@ publish.workspace = true [lints] workspace = true +[features] +# Compiles `fake_engine`, a stand-in engine child for tests, into the `rocm` and +# `rocmd` test builds. Never enabled in a release build. +test-support = [] + [dependencies] +anyhow.workspace = true rocm-core = { path = "../rocm-core" } serde.workspace = true serde_json.workspace = true diff --git a/crates/rocm-engine-protocol/src/fake_engine.rs b/crates/rocm-engine-protocol/src/fake_engine.rs new file mode 100644 index 0000000000..2d8baa14a7 --- /dev/null +++ b/crates/rocm-engine-protocol/src/fake_engine.rs @@ -0,0 +1,138 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! A stand-in engine child for tests. +//! +//! It is a `/bin/sh` script that records how it was launched (arguments, +//! environment, stdin) into a scratch directory and then runs a caller-supplied +//! body, so a test can assert what an engine IPC client put on the child's +//! command line, environment and stdin, and shape what comes back (a response +//! envelope, a crash, stderr output). +//! +//! Test-only: compiled for this crate's own tests and for the `rocm` and +//! `rocmd` test builds through the `test-support` feature, so the two engine +//! IPC callers are exercised against the same fake rather than each growing a +//! private one. Unix-only, because the fake is a shell script; the pure parts +//! of the client are tested on every platform without it. + +use std::fs; +use std::io; +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU32, Ordering}; + +static NEXT_ENGINE: AtomicU32 = AtomicU32::new(0); + +/// `errno` for "text file busy" (`ETXTBSY`): another thread's `fork` briefly +/// held the script's write descriptor open when this one tried to execute it. +const TEXT_FILE_BUSY: i32 = 26; + +pub struct FakeEngine { + dir: PathBuf, + script: PathBuf, +} + +impl FakeEngine { + /// Write the script. `reads_stdin` makes the fake drain its stdin into + /// `stdin` first (what a well-behaved engine does); pass `false` for an + /// engine that exits without reading its request. `body` runs after the + /// recording and may use `$dir`, the scratch directory. + pub fn new(label: &str, reads_stdin: bool, body: &str) -> Self { + let dir = std::env::temp_dir().join(format!( + "rocm-fake-engine-{label}-{}-{}", + std::process::id(), + NEXT_ENGINE.fetch_add(1, Ordering::Relaxed) + )); + fs::create_dir_all(&dir).expect("create the fake engine's scratch directory"); + let script = dir.join("engine.sh"); + let read_stdin = if reads_stdin { + "cat > \"$dir/stdin\"\n" + } else { + "" + }; + fs::write( + &script, + format!( + "#!/bin/sh\ndir='{}'\nprintf '%s\\n' \"$@\" > \"$dir/argv\"\nenv > \"$dir/env\"\n{read_stdin}{body}\n", + dir.display() + ), + ) + .expect("write the fake engine script"); + fs::set_permissions(&script, fs::Permissions::from_mode(0o755)) + .expect("make the fake engine executable"); + Self { dir, script } + } + + /// A fake that reads its request and answers with `response` (a complete + /// response envelope as JSON text), exiting with `status`. + pub fn answering(label: &str, response: &str, status: i32) -> Self { + Self::new( + label, + true, + &format!("printf '%s' '{response}'\nexit {status}"), + ) + } + + pub fn path(&self) -> &Path { + &self.script + } + + /// The scratch directory, for a test that wants to point `$dir` somewhere. + pub fn dir(&self) -> &Path { + &self.dir + } + + /// The arguments the fake was started with, one per entry. + pub fn argv(&self) -> Vec { + fs::read_to_string(self.dir.join("argv")) + .expect("the fake engine never recorded its arguments") + .lines() + .map(str::to_owned) + .collect() + } + + /// Everything the fake read from its stdin, verbatim. + pub fn stdin(&self) -> String { + fs::read_to_string(self.dir.join("stdin")).expect("the fake engine never read its stdin") + } + + /// The value of `key` in the fake's environment, if it was set. + pub fn env(&self, key: &str) -> Option { + let prefix = format!("{key}="); + fs::read_to_string(self.dir.join("env")) + .expect("the fake engine never recorded its environment") + .lines() + .find_map(|line| line.strip_prefix(&prefix).map(str::to_owned)) + } + + /// Run `launch`, retrying while the kernel refuses to execute the script + /// because a concurrent `fork` in this test process still holds its write + /// descriptor. A fresh script is executed the instant it is written, and the + /// test harness runs tests on parallel threads, so the window is real. + pub fn run(&self, mut launch: impl FnMut() -> anyhow::Result) -> anyhow::Result { + for attempt in 0..50 { + match launch() { + Err(error) if attempt < 49 && is_text_file_busy(&error) => { + std::thread::sleep(std::time::Duration::from_millis(20)); + } + other => return other, + } + } + unreachable!("the final attempt always returns") + } +} + +fn is_text_file_busy(error: &anyhow::Error) -> bool { + error.chain().any(|cause| { + cause + .downcast_ref::() + .is_some_and(|io| io.raw_os_error() == Some(TEXT_FILE_BUSY)) + }) +} + +impl Drop for FakeEngine { + fn drop(&mut self) { + let _ = fs::remove_dir_all(&self.dir); + } +} diff --git a/crates/rocm-engine-protocol/src/lib.rs b/crates/rocm-engine-protocol/src/lib.rs index 6ca773ccd6..a51b427dc3 100644 --- a/crates/rocm-engine-protocol/src/lib.rs +++ b/crates/rocm-engine-protocol/src/lib.rs @@ -2,6 +2,9 @@ // // SPDX-License-Identifier: MIT +#[cfg(all(unix, any(test, feature = "test-support")))] +pub mod fake_engine; + use serde::Deserialize; use serde::Serialize; use serde_json::Value; From b8535d54ecc7e2edf06c06e3bf859517282c9a40 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 20:55:47 +0000 Subject: [PATCH 02/38] refactor(engine-ipc): run engine children through one stdio client shared by rocm and rocmd rocm and rocmd each carried their own client for the JSON-over-stdio request to an engine process, and the two had drifted. They now run the child through one implementation in rocm-engine-protocol (stdio_client.rs); each caller still chooses which program to run, and what it asks of the client is a small set of options: the app paths, the engine environments root, an endpoint key file, a progress request, and whether to forward the child's stderr as it arrives. rocm still answers built-in engines in its own process and resolves the engine binary; rocmd still re-executes its own executable as __engine-stdio. Where the two behaved differently, the client takes the behaviour that reports more or fails more safely: - A failing exit status is an error only when the child printed no answer; an answer on stdout is decoded even if the child then exited non-zero (rocm's rule; rocmd discarded the answer for the status). - A parse failure carries the child's stderr, and the decode errors use rocm's wording (rocmd's said "failed to parse engine response", "engine response did not include data" and "failed to deserialize engine response data"). - The write of the request is guarded against SIGPIPE (rocm's rule). rocmd had no guard, so the daemon hosted in rocm, which runs with SIGPIPE at its default action, was killed by a write to an engine that had already exited instead of reporting why. - The child runs on the caller's app paths in both (rocmd now sets the ROCM_CLI_*_DIR variables; it relied on inheriting them), and the spawn error names the program in both. - A request whose write fails kills and reaps the child, so the long-lived daemon does not collect a zombie per failure. - Forwarded stderr no longer panics if this process's stderr is closed. The endpoint key stays an option only rocmd uses, since none of rocm's requests is about a running service's endpoint; rocm still sets no key. Stderr is forwarded for rocm and only kept for rocmd. Neither client has, or gains, a timeout. Also moved to where both can reach them: with_sigpipe_ignored (and its re-exec test, same name) from rocm's main.rs to rocm-core; the three ROCM_CLI_*_DIR variables as AppPaths::env_vars, which rocm's app_path_env_vars and rocmd's capture wrapper now call instead of each listing them; and the byte-identical optional_arg, from both binaries into rocm-engine-protocol. The fake engine written for the tests now plants its script with rocm_core::fake_executable instead of retrying on ETXTBSY. Both callers' tests now run through the shared client and flip what the previous commit pinned (rocmd's app paths, nonzero-with-answer, SIGPIPE under the default action, error wording). Breaking either caller's options, the client's SIGPIPE guard or an environment variable turns a test red. Signed-off-by: Jussi Elo --- Cargo.lock | 1 + apps/rocm/src/engine_ipc.rs | 361 +++------- apps/rocm/src/engine_runtime_select.rs | 26 +- apps/rocm/src/main.rs | 114 ---- apps/rocmd/src/common.rs | 17 +- apps/rocmd/src/engine_ipc.rs | 284 ++++---- apps/rocmd/src/supervise.rs | 15 +- crates/rocm-core/src/lib.rs | 4 +- crates/rocm-core/src/paths.rs | 41 +- crates/rocm-core/src/sigpipe.rs | 128 ++++ crates/rocm-engine-protocol/Cargo.toml | 4 + .../rocm-engine-protocol/src/fake_engine.rs | 43 +- crates/rocm-engine-protocol/src/lib.rs | 26 + .../rocm-engine-protocol/src/stdio_client.rs | 636 ++++++++++++++++++ docs/architecture.md | 13 +- docs/testing.md | 21 + 16 files changed, 1121 insertions(+), 613 deletions(-) create mode 100644 crates/rocm-core/src/sigpipe.rs create mode 100644 crates/rocm-engine-protocol/src/stdio_client.rs diff --git a/Cargo.lock b/Cargo.lock index 96dc60ebad..b856bfcdc4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3906,6 +3906,7 @@ name = "rocm-engine-protocol" version = "0.1.0" dependencies = [ "anyhow", + "libc", "rocm-core", "serde", "serde_json", diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index ab82720bab..db883adcf5 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -4,35 +4,34 @@ //! Engine plugin binary resolution and the engine IPC request/response //! envelope: locating an engine's executable (packaged plugin vs. the -//! platform-packaged fallback), encoding/decoding the JSON-over-stdio -//! envelope used to talk to it, and the built-in (in-process) fast path for -//! `lemonade`/`vllm` that skips spawning a child process entirely. +//! platform-packaged fallback), talking to it over JSON-over-stdio, and the +//! built-in (in-process) fast path for `lemonade`/`vllm`/`aim` that skips +//! spawning a child process entirely. +//! +//! The stdio conversation itself is not implemented here. The request framing, +//! the child's environment, stderr handling, the SIGPIPE guard around the write +//! and the reading of the answer are the client in +//! [`rocm_engine_protocol`]'s `stdio_client` module, which `rocmd` uses too. +//! This module chooses what to run (the built-in handler, or the resolved +//! engine binary with `stdio`) and what `rocm` asks of the client for it +//! ([`engine_stdio_child_request`]). //! -//! Mechanically relocated from `main.rs` with no behavior change — Phase 6f -//! (ROCMAI-213) of the `main.rs` modularization (see `docs/architecture.md`). //! `dispatch()` calls directly into this module at two points — //! `Command::EngineStdio { engine } => run_builtin_engine_stdio(&engine)` and //! the `__engine-serve-http` arm's `run_builtin_engine_serve_http(...)` call — //! both re-imported via `use crate::engine_ipc::{run_builtin_engine_serve_http, -//! run_builtin_engine_stdio};`; `dispatch()`'s own body is otherwise -//! unaffected. +//! run_builtin_engine_stdio};`. //! //! This module owns: resolving an engine's binary path //! (`resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, -//! `find_engine_plugin_binary`), the stdio request/response envelope used to -//! talk to an external engine binary or the in-process built-in handler +//! `find_engine_plugin_binary`), the request entry points //! (`engine_request`, `engine_request_with_env_root`, -//! `decode_engine_response`, the `ScopedEnvVar`/ +//! `engine_stdio_child_request`), the `ScopedEnvVar`/ //! `with_scoped_builtin_engine_env` env-scoping helpers and the //! `BUILTIN_ENGINE_ENV_LOCK` mutex they serialize on, `builtin_engine_request`, -//! `run_builtin_engine_stdio`), and the built-in HTTP-serving path +//! `run_builtin_engine_stdio`, and the built-in HTTP-serving path //! (`builtin_engine_available`, `run_builtin_engine_serve_http`, -//! `builtin_engine_serve_http_args`). One item sits physically inside this -//! span in `main.rs` but stays there rather than moving: `impl -//! From for WatcherMode` is unrelated clap-arg-to-domain-type -//! conversion that only happens to sit between `find_engine_plugin_binary` and -//! `engine_request` in the original file (it has since moved to `cli.rs`, -//! next to the enum it converts). +//! `builtin_engine_serve_http_args`). //! //! `main.rs` calls into it for `resolve_engine_binary_path`, //! `resolve_engine_binary_path_with_paths`, `engine_request`, @@ -42,39 +41,32 @@ //! `runtime_install_finalize.rs` call into it for //! `engine_request_with_env_root`; `service_launch.rs` calls into it for //! `builtin_engine_serve_http_args`. `find_engine_plugin_binary`, -//! `decode_engine_response`, `with_scoped_builtin_engine_env`, -//! `builtin_engine_request`, and `ScopedEnvVar` have no callers outside this -//! module and stay private. Shared helpers this module calls back into at the -//! crate root — `with_sigpipe_ignored` and `device_policy_name` — stay in -//! `main.rs` and are reached unqualified via `use crate::{...}`, same as for -//! other mechanically relocated modules. `parse_device_policy` moved out of -//! `main.rs` into `gpu_policy.rs` in ROCMAI-214 (Phase 6f); it is reached via -//! `use crate::gpu_policy::parse_device_policy;` instead. `apply_app_path_env`, -//! `app_path_env_vars`, `optional_arg`, and `engine_recipe_json_arg` moved out -//! of `main.rs` into `engine_runtime_select.rs` in ROCMAI-215 (Phase 6f); this -//! file's `use` list below points there now instead of at the crate root. +//! `with_scoped_builtin_engine_env`, `builtin_engine_request`, and +//! `ScopedEnvVar` have no callers outside this module and stay private. The +//! one helper this module calls back into at the crate root, `device_policy_name`, +//! stays in `main.rs` and is reached via `use crate::device_policy_name;`. +//! `parse_device_policy` is in `gpu_policy.rs`; `app_path_env_vars` and +//! `engine_recipe_json_arg` are in `engine_runtime_select.rs`, and +//! `optional_arg` is in `rocm_engine_protocol`. use std::ffi::OsString; -use std::io::{self, BufRead, Read, Write}; +use std::io::{self, Read}; use std::path::{Path, PathBuf}; -use std::process::{Command as ProcessCommand, Stdio}; use std::sync::{Mutex, OnceLock}; -use std::thread; use anyhow::{Context, Result, bail}; -use rocm_core::{AppPaths, engine_binary_path, engine_plugin_dirs}; +use rocm_core::{AppPaths, ENGINE_ENVS_ROOT_ENV, engine_binary_path, engine_plugin_dirs}; use rocm_engine_protocol::{ DevicePolicy, EngineMethod, EnginePluginDescriptor, EngineRecipeHint, EngineRequestEnvelope, - EngineResponseEnvelope, + EngineResponseEnvelope, EngineStdioOptions, decode_engine_response, engine_request_envelope, + optional_arg, run_engine_stdio, }; use serde::Serialize; use serde::de::DeserializeOwned; -use crate::engine_runtime_select::{ - app_path_env_vars, apply_app_path_env, engine_recipe_json_arg, optional_arg, -}; +use crate::device_policy_name; +use crate::engine_runtime_select::{app_path_env_vars, engine_recipe_json_arg}; use crate::gpu_policy::parse_device_policy; -use crate::{device_policy_name, with_sigpipe_ignored}; static BUILTIN_ENGINE_ENV_LOCK: OnceLock> = OnceLock::new(); @@ -130,11 +122,7 @@ where R: DeserializeOwned, { let stream_progress = matches!(&method, EngineMethod::Install); - let envelope = EngineRequestEnvelope { - method, - payload: serde_json::to_value(request) - .context("failed to encode engine request payload")?, - }; + let envelope = engine_request_envelope(method, request)?; if let Some(envelope) = with_scoped_builtin_engine_env(paths, env_root, || { builtin_engine_request(engine, &envelope) }) { @@ -150,8 +138,16 @@ where } /// The out-of-process half of [`engine_request_with_env_root`]: run -/// `engine_binary stdio`, send `envelope` on its stdin and decode its answer. -/// Split out so a test can point it at a stand-in binary. +/// `engine_binary stdio` through the client `rocm` shares with `rocmd`, send +/// `envelope` on its stdin and decode its answer. +/// +/// What `rocm` asks of that client: the child runs on this process's app paths +/// when it has them and on the engine environments root when the request names +/// one, an install asks for progress on stderr, and the child's stderr is +/// written to this process's as it arrives, which is how a terminal user +/// watches an install. No endpoint key is handed over, because none of the +/// requests `rocm` makes is about a running service's endpoint. Split out so a +/// test can point it at a stand-in binary. fn engine_stdio_child_request( engine_binary: &Path, paths: Option<&AppPaths>, @@ -162,118 +158,29 @@ fn engine_stdio_child_request( where R: DeserializeOwned, { - let mut command = ProcessCommand::new(engine_binary); - command.arg("stdio"); - if let Some(paths) = paths { - apply_app_path_env(&mut command, paths); - } - if let Some(env_root) = env_root { - command.env("ROCM_CLI_ENGINE_ENVS_ROOT", env_root); - } - if stream_progress { - command.env("ROCM_ENGINE_PROGRESS_STDERR", "1"); - } - let mut child = command - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn() - .context("failed to spawn engine stdio process")?; - - { - let mut stdin = child - .stdin - .take() - .context("engine stdio child did not expose stdin")?; - // Ignore SIGPIPE for the duration of the write: with SIG_DFL in effect - // process-wide, an engine that exits before reading its stdin would - // otherwise kill `rocm` with SIGPIPE before the exit-status diagnostics - // below can run. Under SIG_IGN the write returns EPIPE instead, which we - // surface as the normal "failed to write engine request" error and then - // let `wait()` report why the engine exited. - with_sigpipe_ignored(|| { - serde_json::to_writer(&mut stdin, envelope) - .context("failed to write engine request")?; - stdin.write_all(b"\n")?; - Ok::<(), anyhow::Error>(()) - })?; - } - - let stderr_handle = child.stderr.take().map(|stderr| { - thread::spawn(move || { - let mut reader = io::BufReader::new(stderr); - let mut collected = Vec::new(); - let mut line = Vec::new(); - loop { - line.clear(); - match reader.read_until(b'\n', &mut line) { - Ok(0) => break, - Ok(_) => { - if let Ok(text) = std::str::from_utf8(&line) { - eprint!("{text}"); - } else { - let _ = io::stderr().write_all(&line); - } - collected.extend_from_slice(&line); - } - Err(error) => { - let message = format!("failed to read engine progress: {error}\n"); - eprint!("{message}"); - collected.extend_from_slice(message.as_bytes()); - break; - } - } - } - collected - }) - }); - let mut stdout = Vec::new(); - child - .stdout - .take() - .context("engine stdio child did not expose stdout")? - .read_to_end(&mut stdout) - .context("failed to read engine stdio response")?; - let status = child - .wait() - .context("failed waiting for engine stdio response")?; - let stderr = stderr_handle - .map(|handle| handle.join().unwrap_or_else(|_| Vec::new())) - .unwrap_or_default(); - - if !status.success() && stdout.is_empty() { - let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); - if stderr.is_empty() { - bail!("engine stdio process exited with status {status}"); - } - bail!("engine stdio process exited with status {status}: {stderr}"); - } - let envelope: EngineResponseEnvelope = serde_json::from_slice(&stdout).with_context(|| { - let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); - if stderr.is_empty() { - "failed to parse engine response envelope".to_owned() - } else { - format!("failed to parse engine response envelope; stderr: {stderr}") - } - })?; - decode_engine_response(envelope) + let options = stdio_options(paths, env_root, stream_progress); + decode_engine_response(run_engine_stdio( + engine_binary, + ["stdio"], + &options, + envelope, + )?) } -fn decode_engine_response(envelope: EngineResponseEnvelope) -> Result -where - R: DeserializeOwned, -{ - if !envelope.ok { - let error = envelope.error.map_or_else( - || "unknown engine error".to_owned(), - |value| format!("{}: {}", value.code, value.message), - ); - bail!("{error}"); +/// What `rocm` asks of the shared stdio client; see +/// [`engine_stdio_child_request`]. +const fn stdio_options<'a>( + paths: Option<&'a AppPaths>, + env_root: Option<&'a Path>, + stream_progress: bool, +) -> EngineStdioOptions<'a> { + EngineStdioOptions { + app_paths: paths, + env_root, + endpoint_key_file: None, + request_progress: stream_progress, + forward_stderr: true, } - let data = envelope - .data - .context("engine response envelope did not contain data")?; - serde_json::from_value(data).context("failed to decode engine response payload") } struct ScopedEnvVar { @@ -321,10 +228,7 @@ fn with_scoped_builtin_engine_env( } } if let Some(env_root) = env_root { - vars.push(ScopedEnvVar::set_path( - "ROCM_CLI_ENGINE_ENVS_ROOT", - env_root, - )); + vars.push(ScopedEnvVar::set_path(ENGINE_ENVS_ROOT_ENV, env_root)); } let result = f(); drop(vars); @@ -507,55 +411,36 @@ mod tests { Ok(()) } - // ---- The stdio client, driven against a stand-in engine child. ---- + // ---- What `rocm` asks of the shared stdio client. ---- // - // These pin what `rocm` puts on an engine child's command line, environment - // and stdin and how it reads the answer back. `rocmd` keeps its own client - // with a different contract; its tests are in `apps/rocmd/src/engine_ipc.rs`. - - fn envelope(method: EngineMethod, payload: serde_json::Value) -> EngineRequestEnvelope { - EngineRequestEnvelope { method, payload } - } + // The conversation itself (framing, SIGPIPE guard, exit status and answer + // handling) is the client's, tested in `rocm-engine-protocol`. These tests + // run `rocm`'s entry point against a stand-in engine child and pin what + // `rocm` chooses: the verb, the environment and the stderr handling. The + // other caller's choices are pinned in `apps/rocmd/src/engine_ipc.rs`. #[test] - fn decode_engine_response_reports_each_way_an_answer_can_be_unusable() { - let failed = EngineResponseEnvelope { - ok: false, - data: None, - error: Some(rocm_engine_protocol::EngineErrorDetail { - code: "stop_failed".to_owned(), - message: "it broke".to_owned(), - }), - }; - let error = decode_engine_response::(failed).unwrap_err(); - assert_eq!(error.to_string(), "stop_failed: it broke"); - - let unexplained = EngineResponseEnvelope { - ok: false, - data: None, - error: None, - }; - let error = decode_engine_response::(unexplained).unwrap_err(); - assert_eq!(error.to_string(), "unknown engine error"); - - let empty = EngineResponseEnvelope { - ok: true, - data: None, - error: None, - }; - let error = decode_engine_response::(empty).unwrap_err(); - assert_eq!( - error.to_string(), - "engine response envelope did not contain data" - ); + fn rocm_asks_for_forwarded_stderr_and_no_endpoint_key() { + let (root, paths) = crate::tests::test_paths("engine-ipc-options"); + let env_root = root.join("envs"); - let wrong_shape = EngineResponseEnvelope::success(serde_json::json!({"stopped": "yes"})); - let error = - decode_engine_response::(wrong_shape).unwrap_err(); + let options = stdio_options(Some(&paths), Some(&env_root), true); assert_eq!( - error.to_string(), - "failed to decode engine response payload" + options.app_paths.map(|p| &p.data_dir), + Some(&paths.data_dir) ); + assert_eq!(options.env_root, Some(env_root.as_path())); + assert!(options.request_progress); + // The terminal user watches an install through the child's stderr. + assert!(options.forward_stderr); + // No request `rocm` makes is about a running service's endpoint. + assert_eq!(options.endpoint_key_file, None); + + let bare = stdio_options(None, None, false); + assert!(bare.app_paths.is_none() && bare.env_root.is_none()); + assert!(!bare.request_progress); + assert!(bare.forward_stderr); + let _ = fs::remove_dir_all(root); } #[cfg(unix)] @@ -567,6 +452,10 @@ mod tests { const STOPPED: &str = r#"{"ok":true,"data":{"stopped":true}}"#; + fn envelope(method: EngineMethod, payload: serde_json::Value) -> EngineRequestEnvelope { + EngineRequestEnvelope { method, payload } + } + fn stop_request() -> EngineRequestEnvelope { envelope(EngineMethod::Stop, json!({"service_id": "svc"})) } @@ -577,15 +466,13 @@ mod tests { env_root: Option<&Path>, stream_progress: bool, ) -> Result { - engine.run(|| { - engine_stdio_child_request::( - engine.path(), - paths, - &stop_request(), - env_root, - stream_progress, - ) - }) + engine_stdio_child_request::( + engine.path(), + paths, + &stop_request(), + env_root, + stream_progress, + ) } fn ambient(key: &str) -> Option { @@ -688,23 +575,6 @@ mod tests { ); } - #[test] - fn a_nonzero_exit_with_no_answer_reports_the_status_and_stderr() { - let engine = FakeEngine::new("rocm-crash", true, "echo boom >&2\nexit 3"); - let error = request(&engine, None, None, false).unwrap_err(); - assert_eq!( - error.to_string(), - "engine stdio process exited with status exit status: 3: boom" - ); - - let silent = FakeEngine::new("rocm-silent-crash", true, "exit 4"); - let error = request(&silent, None, None, false).unwrap_err(); - assert_eq!( - error.to_string(), - "engine stdio process exited with status exit status: 4" - ); - } - #[test] fn a_nonzero_exit_that_still_answered_is_decoded_not_discarded() { // The engine's own error envelope is the useful diagnostic, so an @@ -724,27 +594,6 @@ mod tests { ); } - #[test] - fn an_unparseable_answer_carries_the_childs_stderr() { - let engine = FakeEngine::new( - "rocm-garbage", - true, - "printf 'not json'\necho oops >&2\nexit 0", - ); - let error = request(&engine, None, None, false).unwrap_err(); - assert_eq!( - error.to_string(), - "failed to parse engine response envelope; stderr: oops" - ); - - let silent = FakeEngine::new("rocm-garbage-quiet", true, "printf 'not json'"); - let error = request(&silent, None, None, false).unwrap_err(); - assert_eq!( - error.to_string(), - "failed to parse engine response envelope" - ); - } - /// The write of a request to an engine that has already exited must come /// back as an error even where SIGPIPE is at its default action, which is /// how `main` runs `rocm`. Re-executes this test in a fresh process that @@ -757,15 +606,13 @@ mod tests { let engine = FakeEngine::new("rocm-sigpipe", false, "exit 3"); // Larger than a pipe buffer, so the write reaches the closed read end. let big = "x".repeat(1 << 20); - let result = engine.run(|| { - engine_stdio_child_request::( - engine.path(), - None, - &envelope(EngineMethod::Stop, json!({ "blob": big })), - None, - false, - ) - }); + let result = engine_stdio_child_request::( + engine.path(), + None, + &envelope(EngineMethod::Stop, json!({ "blob": big })), + None, + false, + ); let error = result.expect_err("the engine never answered"); assert!( format!("{error:#}").contains("failed to write engine request"), @@ -774,7 +621,7 @@ mod tests { return; } - let status = ProcessCommand::new(std::env::current_exe().unwrap()) + let status = std::process::Command::new(std::env::current_exe().unwrap()) .args([ "engine_ipc::tests::stdio_child::a_request_to_an_engine_that_already_exited_is_an_error_under_sig_dfl", "--exact", diff --git a/apps/rocm/src/engine_runtime_select.rs b/apps/rocm/src/engine_runtime_select.rs index 793a82f3ab..4f4a1816e5 100644 --- a/apps/rocm/src/engine_runtime_select.rs +++ b/apps/rocm/src/engine_runtime_select.rs @@ -7,8 +7,8 @@ //! to (`EngineSelection`, `resolve_engine_selection`, //! `validate_engine_selection_runtime`, `single_ready_runtime_key`, //! `resolve_runtime_selector_to_exact_key`), small argv-building helpers used -//! when relaunching a managed engine (`optional_arg`, -//! `engine_recipe_json_arg`, `parse_engine_recipe_json_arg`, +//! when relaunching a managed engine (`engine_recipe_json_arg`, +//! `parse_engine_recipe_json_arg`, //! `daemon_run_argv`, `managed_service_launcher_path`), the `ROCM_CLI_*` //! environment variables every managed child inherits //! (`app_path_env_vars`/`app_path_env_var_values`/`app_path_env_var_refs`/ @@ -61,13 +61,12 @@ //! it does not decide which runtime counts as "the one ready runtime" itself, //! it asks `runtime_adopt::select_single_ready_runtime`, the rule //! `rocm comfyui install` shares. -//! `optional_arg` and -//! `engine_recipe_json_arg` are `pub(crate)`, read from `engine_ipc.rs`'s +//! `engine_recipe_json_arg` is `pub(crate)`, read from `engine_ipc.rs`'s //! `engine_request`; `parse_engine_recipe_json_arg` is `pub(crate)`, read //! from `main.rs`'s `dispatch()` and `service_control.rs`'s `restart_child_argv`. -//! `app_path_env_vars`/`apply_app_path_env` are `pub(crate)`, read from -//! `engine_ipc.rs`, `app_path_env_vars` is also read from -//! `chat/mcp_bridge.rs`, and `apply_app_path_env` is also read from +//! `app_path_env_vars` (which is [`AppPaths::env_vars`]) and `apply_app_path_env` +//! are `pub(crate)`, read from `engine_ipc.rs`, `app_path_env_vars` is also +//! read from `chat/mcp_bridge.rs`, and `apply_app_path_env` is also read from //! `engines_cmd.rs`, //! `service_launch.rs`, and `service_control.rs`'s //! `restart_internal_managed_service_with`; `app_path_env_var_values`/ @@ -246,13 +245,6 @@ pub(crate) fn resolve_runtime_selector_to_exact_key( } } -pub(crate) fn optional_arg(flag: &str, value: Option<&str>) -> Vec { - match value { - Some(value) => vec![flag.to_owned(), value.to_owned()], - None => Vec::new(), - } -} - pub(crate) fn engine_recipe_json_arg( engine_recipe: Option<&EngineRecipeHint>, ) -> Result> { @@ -276,11 +268,7 @@ pub(crate) fn parse_engine_recipe_json_arg( } pub(crate) fn app_path_env_vars(paths: &AppPaths) -> [(&'static str, &Path); 3] { - [ - ("ROCM_CLI_CONFIG_DIR", paths.config_dir.as_path()), - ("ROCM_CLI_DATA_DIR", paths.data_dir.as_path()), - ("ROCM_CLI_CACHE_DIR", paths.cache_dir.as_path()), - ] + paths.env_vars() } #[cfg_attr(not(windows), allow(dead_code))] diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 61a6f67ff8..bb6ba7b041 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -191,10 +191,6 @@ use std::fs; use std::io::{self, Write}; use std::path::PathBuf; use std::process::ExitCode; -#[cfg(all(test, not(windows)))] -use std::process::Stdio; -#[cfg(all(test, not(windows)))] -use std::process::{Command as ProcessCommand, ExitStatus}; #[cfg(test)] use std::sync::Mutex; @@ -223,33 +219,6 @@ fn reset_sigpipe() { #[allow(clippy::missing_const_for_fn)] fn reset_sigpipe() {} -/// Run `f` with SIGPIPE temporarily ignored, restoring the previous -/// disposition afterwards. -/// -/// [`reset_sigpipe`] sets `SIG_DFL` process-wide so stdout/stderr pipe writes -/// terminate the process conventionally. But a write to a spawned child's stdin -/// whose reader has already exited would then kill `rocm` with SIGPIPE before -/// the caller can turn the resulting `EPIPE` into a diagnostic. Wrap such writes -/// in this helper so the failure surfaces as an `io::Error` (`BrokenPipe`) -/// instead of a signal. -#[cfg(unix)] -#[allow(unsafe_code)] // libc FFI -fn with_sigpipe_ignored(f: impl FnOnce() -> T) -> T { - // SAFETY: reading/installing a signal disposition is always safe. - let previous = unsafe { libc::signal(libc::SIGPIPE, libc::SIG_IGN) }; - let result = f(); - // SAFETY: restoring the disposition captured above is always safe. - unsafe { - libc::signal(libc::SIGPIPE, previous); - } - result -} - -#[cfg(not(unix))] -fn with_sigpipe_ignored(f: impl FnOnce() -> T) -> T { - f() -} - /// Marker error carrying `rocm fix`'s exit code back through `main()`'s /// ordinary return path, instead of calling `std::process::exit` mid-stack /// and skipping the `_log_guard` destructor held in `run()`. @@ -1456,89 +1425,6 @@ pub(crate) mod tests { } } - /// Regression test for the engine child-stdin write under the process-wide - /// `SIG_DFL` that `main` installs via [`reset_sigpipe`]. If an engine child - /// exits before reading its stdin, the parent's write to that pipe must - /// surface as a `BrokenPipe` error — handled by the caller's diagnostics — - /// rather than killing `rocm` with SIGPIPE before those diagnostics run. - /// - /// The bug only reproduces under `SIG_DFL`; the test harness leaves SIGPIPE - /// at Rust's default `SIG_IGN`, and flipping it process-wide would race - /// sibling tests. So this re-execs itself in fresh processes that install - /// `SIG_DFL` first, and checks BOTH directions: - /// - unguarded write to a dead child's stdin is fatal (killed by SIGPIPE) — - /// this is the regression the plain `SIG_DFL` reset introduced; - /// - the same write wrapped in [`with_sigpipe_ignored`] survives and returns - /// an error instead — this is the fix. - /// - /// Each child reaps its short-lived grandchild before writing, so the read - /// end is closed deterministically and the outcome does not race. - #[cfg(unix)] - #[test] - fn engine_stdin_write_under_sig_dfl_is_guarded_not_fatal() { - use std::os::unix::process::ExitStatusExt as _; - - // Grandchild helper: reproduce main()'s SIG_DFL, spawn a process that - // exits immediately, reap it, then write to its now-closed stdin. The - // "guarded" arm wraps the write; the "unguarded" arm does not. - if let Some(mode) = std::env::var_os("ROCM_TEST_SIGPIPE_ARM") { - reset_sigpipe(); - let mut grandchild = ProcessCommand::new("true") - .stdin(Stdio::piped()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - .expect("spawn a process that exits immediately"); - let mut stdin = grandchild.stdin.take().expect("grandchild stdin"); - grandchild.wait().expect("reap grandchild"); - // Heap-allocated so the buffer doesn't trip clippy::large-stack-arrays; - // it must exceed the pipe buffer so the write actually reaches the - // closed read end rather than being swallowed by kernel buffering. - let payload = vec![b'x'; 64 * 1024]; - if mode == "guarded" { - let result = with_sigpipe_ignored(|| stdin.write_all(&payload)); - assert!( - result.is_err(), - "guarded write to a dead pipe should return BrokenPipe, not succeed" - ); - } else { - // Unguarded: under SIG_DFL this write delivers SIGPIPE and never - // returns. If the process somehow survives, exit 0 so the parent's - // "should have been signalled" assertion fails loudly. - let _ = stdin.write_all(&payload); - } - return; - } - - let exe = std::env::current_exe().expect("current test executable"); - let run_arm = |mode: &str| -> ExitStatus { - ProcessCommand::new(&exe) - .args([ - "tests::engine_stdin_write_under_sig_dfl_is_guarded_not_fatal", - "--exact", - "--nocapture", - ]) - .env("ROCM_TEST_SIGPIPE_ARM", mode) - .status() - .expect("re-exec the test arm in a child process") - }; - - let unguarded = run_arm("unguarded"); - assert_eq!( - unguarded.signal(), - Some(libc::SIGPIPE), - "sanity check: an unguarded stdin write under SIG_DFL must be killed by \ - SIGPIPE (got {unguarded:?}); if not, the test no longer proves the guard matters" - ); - - let guarded = run_arm("guarded"); - assert!( - guarded.success(), - "guarded child-stdin write was fatal under SIG_DFL ({guarded:?}) — \ - SIGPIPE reached the process instead of surfacing as an error" - ); - } - #[test] fn completions_generate_for_every_shell() { use clap_complete::Shell; diff --git a/apps/rocmd/src/common.rs b/apps/rocmd/src/common.rs index 417491a716..c95ea8d24a 100644 --- a/apps/rocmd/src/common.rs +++ b/apps/rocmd/src/common.rs @@ -27,22 +27,7 @@ pub(crate) fn run_rocm_capture_for_paths( args: &[&str], timeout: Duration, ) -> Result { - rocm_core::run_rocm_capture_with_env( - args, - &[ - ("ROCM_CLI_CONFIG_DIR", paths.config_dir.as_path()), - ("ROCM_CLI_DATA_DIR", paths.data_dir.as_path()), - ("ROCM_CLI_CACHE_DIR", paths.cache_dir.as_path()), - ], - timeout, - ) -} - -pub(crate) fn optional_arg(flag: &str, value: Option<&str>) -> Vec { - match value { - Some(value) => vec![flag.to_owned(), value.to_owned()], - None => Vec::new(), - } + rocm_core::run_rocm_capture_with_env(args, &paths.env_vars(), timeout) } pub(crate) fn payload_string(payload: &Value, key: &str) -> Option { diff --git a/apps/rocmd/src/engine_ipc.rs b/apps/rocmd/src/engine_ipc.rs index a1fc67018e..e238d45c02 100644 --- a/apps/rocmd/src/engine_ipc.rs +++ b/apps/rocmd/src/engine_ipc.rs @@ -5,13 +5,13 @@ use anyhow::{Context, Result, bail}; use rocm_core::AppPaths; use rocm_engine_protocol::{ - EngineMethod, EngineRequestEnvelope, EngineResponseEnvelope, HealthcheckRequest, - HealthcheckResponse, + EngineMethod, EngineStdioOptions, HealthcheckRequest, HealthcheckResponse, + decode_engine_response, engine_request_envelope, run_engine_stdio, }; use serde::Serialize; use serde::de::DeserializeOwned; -use std::io::Write; -use std::process::{Command as ProcessCommand, Stdio}; +use std::path::{Path, PathBuf}; +use std::process::Command as ProcessCommand; pub(crate) fn engine_healthcheck_response( paths: &AppPaths, @@ -60,15 +60,19 @@ pub(crate) fn apply_endpoint_key_env( paths: &AppPaths, service_id: &str, ) -> bool { - if let Some(key_file) = rocm_engine_protocol::endpoint_key_file_if_present(paths, service_id) - .and_then(|path| rocm_engine_protocol::endpoint_api_key_file_if_valid(&path)) - { + if let Some(key_file) = usable_endpoint_key_file(paths, service_id) { command.env(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file); return true; } false } +/// The service's endpoint key file, when it exists and holds a usable key. +fn usable_endpoint_key_file(paths: &AppPaths, service_id: &str) -> Option { + rocm_engine_protocol::endpoint_key_file_if_present(paths, service_id) + .and_then(|path| rocm_engine_protocol::endpoint_api_key_file_if_valid(&path)) +} + /// Refuse to respawn a recorded service on a public host without its endpoint /// API key, so daemon recovery cannot reopen a protected endpoint anonymously. /// @@ -103,6 +107,11 @@ pub(crate) fn ensure_public_service_has_endpoint_key( Ok(()) } +/// Ask an engine a question about a managed service, through the stdio client +/// `rocmd` shares with `rocm`. +/// +/// The engine is reached by re-executing this process's own executable as +/// `__engine-stdio `, the verb `rocm` serves for its built-in engines. pub(crate) fn engine_request( paths: &AppPaths, engine: &str, @@ -114,10 +123,7 @@ where T: Serialize, R: DeserializeOwned, { - let envelope = EngineRequestEnvelope { - method, - payload: serde_json::to_value(request).context("failed to serialize engine request")?, - }; + let envelope = engine_request_envelope(method, request)?; let engine_binary = std::env::current_exe().context("failed to resolve current rocm executable path")?; engine_request_via(&engine_binary, paths, engine, service_id, &envelope) @@ -126,22 +132,15 @@ where /// [`engine_request`] with the executable that serves `__engine-stdio` named by /// the caller, so a test can point it at a stand-in. fn engine_request_via( - engine_binary: &std::path::Path, + engine_binary: &Path, paths: &AppPaths, engine: &str, service_id: &str, - envelope: &EngineRequestEnvelope, + envelope: &rocm_engine_protocol::EngineRequestEnvelope, ) -> Result where R: DeserializeOwned, { - let mut command = ProcessCommand::new(engine_binary); - command - .arg("__engine-stdio") - .arg(engine) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); // A protected public endpoint enforces its key on every request, including // this stdio-transported healthcheck — without the carrier the engine // adapter's HTTP probe is unauthenticated and the endpoint looks @@ -150,50 +149,33 @@ where // // No host to check here: this is a stdio plugin call, not a listener, so a // missing key only weakens this probe rather than opening a port. - let _ = apply_endpoint_key_env(&mut command, paths, service_id); - let mut child = command.spawn().with_context(|| { - format!( - "failed to spawn engine stdio process {}", - engine_binary.display() - ) - })?; - - { - let stdin = child - .stdin - .as_mut() - .context("engine stdio child did not expose stdin")?; - serde_json::to_writer(&mut *stdin, envelope).context("failed to write engine request")?; - stdin.write_all(b"\n")?; - } + let key_file = usable_endpoint_key_file(paths, service_id); + let options = stdio_options(paths, key_file.as_deref()); + decode_engine_response(run_engine_stdio( + engine_binary, + ["__engine-stdio", engine], + &options, + envelope, + )?) +} - let output = child - .wait_with_output() - .context("failed waiting for engine stdio response")?; - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); - if stderr.is_empty() { - bail!("engine stdio process exited with status {}", output.status); - } - bail!( - "engine stdio process exited with status {}: {}", - output.status, - stderr - ); - } - let envelope: EngineResponseEnvelope = - serde_json::from_slice(&output.stdout).context("failed to parse engine response")?; - if !envelope.ok { - let detail = envelope.error.map_or_else( - || "unknown engine error".to_owned(), - |error| format!("{}: {}", error.code, error.message), - ); - bail!(detail); +/// What `rocmd` asks of the shared stdio client; see [`engine_request`]. +/// +/// The child runs on the daemon's own app paths and carries the service's +/// endpoint key file when there is a usable one. Its stderr is kept for the +/// error, not forwarded: nobody watches a daemon's terminal, and a healthcheck +/// asks for no progress. +const fn stdio_options<'a>( + paths: &'a AppPaths, + endpoint_key_file: Option<&'a Path>, +) -> EngineStdioOptions<'a> { + EngineStdioOptions { + app_paths: Some(paths), + env_root: None, + endpoint_key_file, + request_progress: false, + forward_stderr: false, } - let data = envelope - .data - .context("engine response did not include data")?; - serde_json::from_value(data).context("failed to deserialize engine response data") } #[cfg(test)] @@ -357,14 +339,38 @@ mod tests { assert!(!healthcheck_response_ready(&listing_only)); } - // ---- The stdio client, driven against a stand-in engine child. ---- + // ---- What `rocmd` asks of the shared stdio client. ---- // - // These pin what `rocmd` puts on the child's command line, environment and - // stdin and how it reads the answer back. `rocm` keeps its own client with a - // different contract; its tests are in `apps/rocm/src/engine_ipc.rs`. + // The conversation itself (framing, SIGPIPE guard, exit status and answer + // handling) is the client's, tested in `rocm-engine-protocol`. These tests + // run `rocmd`'s entry point against a stand-in engine child and pin what + // `rocmd` chooses: the verb, the environment and the stderr handling. The + // other caller's choices are pinned in `apps/rocm/src/engine_ipc.rs`. + + #[test] + fn rocmd_runs_the_child_on_its_own_paths_keeps_stderr_and_asks_for_no_progress() { + let (_root, paths) = temp_app_paths("stdio-options"); + let key_file = Path::new("/keys/svc.endpoint-key"); + + let options = stdio_options(&paths, Some(key_file)); + assert_eq!( + options.app_paths.map(|p| &p.data_dir), + Some(&paths.data_dir) + ); + assert_eq!(options.endpoint_key_file, Some(key_file)); + // A healthcheck has no install to report progress on, and nobody watches + // a daemon's terminal. + assert!(!options.request_progress); + assert!(!options.forward_stderr); + assert_eq!(options.env_root, None); + + assert_eq!(stdio_options(&paths, None).endpoint_key_file, None); + } + #[cfg(unix)] mod stdio_child { use super::*; + use rocm_engine_protocol::EngineRequestEnvelope; use rocm_engine_protocol::fake_engine::FakeEngine; use serde_json::{Value, json}; use std::os::unix::process::ExitStatusExt as _; @@ -380,15 +386,13 @@ mod tests { } fn request(engine: &FakeEngine, paths: &AppPaths) -> Result { - engine.run(|| { - engine_request_via::( - engine.path(), - paths, - "vllm", - SERVICE_ID, - &healthcheck_envelope(), - ) - }) + engine_request_via::( + engine.path(), + paths, + "vllm", + SERVICE_ID, + &healthcheck_envelope(), + ) } fn ambient(key: &str) -> Option { @@ -432,104 +436,74 @@ mod tests { } #[test] - fn the_child_is_not_given_the_daemons_app_paths() { - // It relies on the environment it inherits to find the same folders - // the daemon runs on; nothing here sets `ROCM_CLI_*_DIR` for it. - let (_root, paths) = temp_app_paths("stdio-no-app-paths"); - let engine = FakeEngine::answering("rocmd-no-app-paths", STOPPED, 0); + fn the_child_runs_on_the_daemons_app_paths() { + let (_root, paths) = temp_app_paths("stdio-app-paths"); + let engine = FakeEngine::answering("rocmd-app-paths", STOPPED, 0); request(&engine, &paths).unwrap(); - for key in [ - "ROCM_CLI_CONFIG_DIR", - "ROCM_CLI_DATA_DIR", - "ROCM_CLI_CACHE_DIR", - ] { - assert_eq!(engine.env(key), ambient(key), "{key}"); - } - } - - #[test] - fn any_nonzero_exit_is_an_error_even_when_the_child_answered() { - let (_root, paths) = temp_app_paths("stdio-nonzero"); - let crash = FakeEngine::new("rocmd-crash", true, "echo boom >&2\nexit 3"); assert_eq!( - request(&crash, &paths).unwrap_err().to_string(), - "engine stdio process exited with status exit status: 3: boom" + engine.env("ROCM_CLI_CONFIG_DIR").as_deref(), + paths.config_dir.to_str() ); - - let silent = FakeEngine::new("rocmd-silent-crash", true, "exit 4"); assert_eq!( - request(&silent, &paths).unwrap_err().to_string(), - "engine stdio process exited with status exit status: 4" + engine.env("ROCM_CLI_DATA_DIR").as_deref(), + paths.data_dir.to_str() ); - - // `rocm` decodes an answer that came with a failing exit; this client - // discards it for the status. - let answered = FakeEngine::answering("rocmd-answers-then-fails", STOPPED, 2); assert_eq!( - request(&answered, &paths).unwrap_err().to_string(), - "engine stdio process exited with status exit status: 2" + engine.env("ROCM_CLI_CACHE_DIR").as_deref(), + paths.cache_dir.to_str() ); } #[test] - fn an_unusable_answer_is_reported_in_this_clients_own_words() { - let (_root, paths) = temp_app_paths("stdio-answers"); - - let garbage = - FakeEngine::new("rocmd-garbage", true, "printf 'not json'\necho oops >&2"); + fn a_nonzero_exit_that_still_answered_is_decoded_not_discarded() { + let (_root, paths) = temp_app_paths("stdio-nonzero"); + let answered = FakeEngine::answering("rocmd-answers-then-fails", STOPPED, 2); assert_eq!( - request(&garbage, &paths).unwrap_err().to_string(), - "failed to parse engine response" + request(&answered, &paths).unwrap(), + json!({"stopped": true}) ); let failed = FakeEngine::answering( - "rocmd-failed", + "rocmd-error-then-fails", r#"{"ok":false,"error":{"code":"stop_failed","message":"it broke"}}"#, - 0, + 1, ); assert_eq!( request(&failed, &paths).unwrap_err().to_string(), "stop_failed: it broke" ); + } - let unexplained = FakeEngine::answering("rocmd-unexplained", r#"{"ok":false}"#, 0); - assert_eq!( - request(&unexplained, &paths).unwrap_err().to_string(), - "unknown engine error" - ); - - let empty = FakeEngine::answering("rocmd-empty", r#"{"ok":true}"#, 0); + #[test] + fn a_crash_with_no_answer_reports_the_status_and_stderr() { + let (_root, paths) = temp_app_paths("stdio-crash"); + let crash = FakeEngine::new("rocmd-crash", true, "echo boom >&2\nexit 3"); assert_eq!( - request(&empty, &paths).unwrap_err().to_string(), - "engine response did not include data" + request(&crash, &paths).unwrap_err().to_string(), + "engine stdio process exited with status exit status: 3: boom" ); + } - let wrong = FakeEngine::answering("rocmd-wrong", r#"{"ok":true,"data":[]}"#, 0); - let error = engine_request_via::( - wrong.path(), - &paths, - "vllm", - SERVICE_ID, - &healthcheck_envelope(), - ) - .unwrap_err(); + #[test] + fn an_unparseable_answer_carries_the_childs_stderr() { + let (_root, paths) = temp_app_paths("stdio-garbage"); + let garbage = + FakeEngine::new("rocmd-garbage", true, "printf 'not json'\necho oops >&2"); assert_eq!( - error.to_string(), - "failed to deserialize engine response data" + request(&garbage, &paths).unwrap_err().to_string(), + "failed to parse engine response envelope; stderr: oops" ); } - /// Writing a request to a child that has already exited, in this client: - /// where SIGPIPE is ignored (Rust's startup default, which the test - /// harness keeps) the write fails as an ordinary error, and where it is - /// at its default action (`rocm` resets it to that in `main`, and `rocm - /// daemon run` runs this code in that process) the write kills the - /// process. This client has no guard around the write, so the second half - /// pins that gap. Re-executes this test in a fresh process for each case, - /// since changing the disposition process-wide would race sibling tests. + /// A request to a child that has already exited is an ordinary error + /// wherever SIGPIPE stands: ignored (Rust's startup default, which the + /// test harness keeps) or at its default action, which is how `rocm` + /// runs and so how `rocm daemon` runs this code. Re-executes this + /// test in a fresh process for each case, since changing the disposition + /// process-wide would race sibling tests. #[test] #[allow(unsafe_code)] // libc::signal, to restore the default action in the child - fn a_request_to_a_child_that_already_exited_depends_on_the_sigpipe_disposition() { + fn a_request_to_a_child_that_already_exited_is_an_error_whatever_the_sigpipe_disposition() { if let Some(arm) = std::env::var_os("ROCMD_TEST_ENGINE_SIGPIPE_ARM") { if arm == "default-action" { // SAFETY: installing a signal disposition is always safe. @@ -545,15 +519,13 @@ mod tests { method: EngineMethod::Healthcheck, payload: json!({ "blob": big }), }; - let result = engine.run(|| { - engine_request_via::( - engine.path(), - &paths, - "vllm", - SERVICE_ID, - &envelope, - ) - }); + let result = engine_request_via::( + engine.path(), + &paths, + "vllm", + SERVICE_ID, + &envelope, + ); let error = result.expect_err("the engine never answered"); assert!( format!("{error:#}").contains("failed to write engine request"), @@ -565,7 +537,7 @@ mod tests { let run_arm = |arm: &str| { std::process::Command::new(std::env::current_exe().unwrap()) .args([ - "engine_ipc::tests::stdio_child::a_request_to_a_child_that_already_exited_depends_on_the_sigpipe_disposition", + "engine_ipc::tests::stdio_child::a_request_to_a_child_that_already_exited_is_an_error_whatever_the_sigpipe_disposition", "--exact", "--nocapture", ]) @@ -580,10 +552,10 @@ mod tests { let default_action = run_arm("default-action"); assert_eq!( default_action.signal(), - Some(libc::SIGPIPE), - "sanity check: this client has no SIGPIPE guard, so under the default action the \ - write kills the process ({default_action:?})" + None, + "SIGPIPE killed the process instead of surfacing as an error: {default_action:?}" ); + assert!(default_action.success(), "{default_action:?}"); } } } diff --git a/apps/rocmd/src/supervise.rs b/apps/rocmd/src/supervise.rs index eb594e2167..0cd866f3f0 100644 --- a/apps/rocmd/src/supervise.rs +++ b/apps/rocmd/src/supervise.rs @@ -277,9 +277,12 @@ fn engine_serve_http_args( if let Some(csv) = rocm_engine_protocol::gpu_indices_to_csv(gpu_indices) { args.extend(["--gpu".to_owned(), csv]); } - args.extend(crate::common::optional_arg("--runtime-id", runtime_id)); - args.extend(crate::common::optional_arg("--env-id", env_id)); - args.extend(crate::common::optional_arg( + args.extend(rocm_engine_protocol::optional_arg( + "--runtime-id", + runtime_id, + )); + args.extend(rocm_engine_protocol::optional_arg("--env-id", env_id)); + args.extend(rocm_engine_protocol::optional_arg( "--engine-recipe-json", engine_recipe_json, )); @@ -805,18 +808,18 @@ fn recovery_supervise_args(record: &ManagedServiceRecord) -> Vec { .unwrap_or("gpu_required") .to_owned(), ]; - args.extend(crate::common::optional_arg( + args.extend(rocm_engine_protocol::optional_arg( "--runtime-id", record.runtime_id.as_deref(), )); - args.extend(crate::common::optional_arg( + args.extend(rocm_engine_protocol::optional_arg( "--env-id", record.env_id.as_deref(), )); if let Some(csv) = rocm_engine_protocol::gpu_indices_to_csv(&record.gpu_indices) { args.extend(["--gpu".to_owned(), csv]); } - args.extend(crate::common::optional_arg( + args.extend(rocm_engine_protocol::optional_arg( "--engine-recipe-json", record.engine_recipe_json.as_deref(), )); diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index aad2dda627..51151f123d 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -55,6 +55,7 @@ pub mod rocm_install; pub mod runtime; pub mod service_identity; pub mod service_record; +mod sigpipe; pub mod terminal; pub mod user_path; @@ -167,7 +168,7 @@ pub use net::{ http_get_text_with_auth, http_post_json_with_auth, parse_http_endpoint, read_http_response_bounded, write_all_tcp_stream, }; -pub use paths::{AppPaths, engine_plugin_dirs}; +pub use paths::{AppPaths, ENGINE_ENVS_ROOT_ENV, engine_plugin_dirs}; pub use proc_lifecycle::{ IdentityState, KillScope, ProcessIdentity, TerminationOutcome, identity_state, process_start_ticks, terminate_verified, @@ -222,6 +223,7 @@ pub use service_record::{ INFERENCE_VERIFIED_STATE_KEY, ManagedServiceNotFound, ManagedServiceRecord, engine_stop_decides_confirmation, }; +pub use sigpipe::with_sigpipe_ignored; pub use terminal::{quotable_in_single_quotes, strip_terminal_control_sequences}; pub use uv::{ DEFAULT_UV_TIMEOUT_SECS, DependencyViolation, UV_CACHE_DIR_ENV, UV_CACHE_DIR_OVERRIDE_ENV, diff --git a/crates/rocm-core/src/paths.rs b/crates/rocm-core/src/paths.rs index 15840e56e9..0ee01eeb3d 100644 --- a/crates/rocm-core/src/paths.rs +++ b/crates/rocm-core/src/paths.rs @@ -34,7 +34,24 @@ pub struct AppPaths { pub cache_dir: PathBuf, } +/// The variable that moves where engine virtualenvs live (see +/// [`AppPaths::engine_envs_root`]), and the one `rocm` sets on an engine child +/// that must install into a particular root. +pub const ENGINE_ENVS_ROOT_ENV: &str = "ROCM_CLI_ENGINE_ENVS_ROOT"; + impl AppPaths { + /// The three `ROCM_CLI_*_DIR` variables that make a child process run on + /// these roots, in the order config, data, cache. [`Self::discover`] reads + /// the same three back, so a child started with these finds the roots its + /// parent was given. + pub fn env_vars(&self) -> [(&'static str, &Path); 3] { + [ + ("ROCM_CLI_CONFIG_DIR", self.config_dir.as_path()), + ("ROCM_CLI_DATA_DIR", self.data_dir.as_path()), + ("ROCM_CLI_CACHE_DIR", self.cache_dir.as_path()), + ] + } + pub fn discover() -> Result { let data_dir_override = env_path_override("ROCM_CLI_DATA_DIR"); let cache_dir_override = env_path_override("ROCM_CLI_CACHE_DIR"); @@ -159,7 +176,7 @@ impl AppPaths { /// dropped; `engine_envs_dir_reads_its_root_from_the_environment` is what /// keeps the lookup honest meanwhile. pub fn engine_envs_root(&self) -> PathBuf { - self.engine_envs_root_from(env_path_override("ROCM_CLI_ENGINE_ENVS_ROOT").as_deref()) + self.engine_envs_root_from(env_path_override(ENGINE_ENVS_ROOT_ENV).as_deref()) } /// [`Self::engine_envs_root`] against a caller-supplied override. @@ -343,6 +360,28 @@ mod tests { use crate::test_support::temp_app_paths; + #[test] + fn env_vars_name_the_three_roots_discover_reads_back() { + let (_root, paths) = temp_app_paths("env-vars"); + assert_eq!( + paths.env_vars(), + [ + ("ROCM_CLI_CONFIG_DIR", paths.config_dir.as_path()), + ("ROCM_CLI_DATA_DIR", paths.data_dir.as_path()), + ("ROCM_CLI_CACHE_DIR", paths.cache_dir.as_path()), + ] + ); + // `discover` reads these same names; a rename in one place must not + // leave the other behind. + let source = include_str!("paths.rs"); + for (name, _) in paths.env_vars() { + assert!( + source.contains(&format!("env_path_override(\"{name}\")")), + "{name} is not read by AppPaths::discover" + ); + } + } + #[test] fn audit_events_path_lives_under_data_audit() { let (_root, paths) = temp_app_paths("audit-path"); diff --git a/crates/rocm-core/src/sigpipe.rs b/crates/rocm-core/src/sigpipe.rs new file mode 100644 index 0000000000..07c5f8b44d --- /dev/null +++ b/crates/rocm-core/src/sigpipe.rs @@ -0,0 +1,128 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Writing to a pipe whose reader has gone away, without being killed for it. +//! +//! Rust starts every process with SIGPIPE ignored, so a write to a closed pipe +//! returns `EPIPE`. `rocm` restores the default action at startup so it ends +//! quietly when its own output is piped into `head`, and `rocm daemon` +//! hosts the daemon in that same process. Under the default action a write to +//! a child that has already exited kills the writer before it can report why +//! the child exited, so code that writes to a child's stdin wraps the write in +//! [`with_sigpipe_ignored`]. + +/// Run `f` with SIGPIPE ignored, restoring the previous disposition afterwards. +/// +/// A write that reaches a closed pipe inside `f` then returns an `EPIPE` error +/// instead of delivering a signal. Where the disposition was already ignored +/// (Rust's startup default) this changes nothing. The disposition is +/// process-wide, so keep `f` to the write itself. Windows has no SIGPIPE and +/// runs `f` as is. +#[cfg(unix)] +#[allow(unsafe_code)] // libc FFI +pub fn with_sigpipe_ignored(f: impl FnOnce() -> T) -> T { + // SAFETY: reading/installing a signal disposition is always safe. + let previous = unsafe { libc::signal(libc::SIGPIPE, libc::SIG_IGN) }; + let result = f(); + // SAFETY: restoring the disposition captured above is always safe. + unsafe { + libc::signal(libc::SIGPIPE, previous); + } + result +} + +#[cfg(not(unix))] +pub fn with_sigpipe_ignored(f: impl FnOnce() -> T) -> T { + f() +} + +#[cfg(all(test, unix))] +mod tests { + use super::with_sigpipe_ignored; + use std::io::Write as _; + use std::os::unix::process::ExitStatusExt as _; + use std::process::{Command, ExitStatus, Stdio}; + + /// A write to a dead child's stdin under the default SIGPIPE action is + /// fatal, and the same write inside [`with_sigpipe_ignored`] returns an + /// error instead. + /// + /// The bug only reproduces under `SIG_DFL`; the test harness leaves SIGPIPE + /// at Rust's default `SIG_IGN`, and flipping it process-wide would race + /// sibling tests. So this re-execs itself in fresh processes that install + /// `SIG_DFL` first, and checks BOTH directions: + /// - unguarded write to a dead child's stdin is fatal (killed by SIGPIPE) — + /// this is the regression the plain `SIG_DFL` reset introduced; + /// - the same write wrapped in [`with_sigpipe_ignored`] survives and returns + /// an error instead — this is the fix. + /// + /// Each child reaps its short-lived grandchild before writing, so the read + /// end is closed deterministically and the outcome does not race. + #[test] + #[allow(unsafe_code)] // libc::signal, to restore the default action in the child + fn engine_stdin_write_under_sig_dfl_is_guarded_not_fatal() { + // Grandchild helper: install the default action, spawn a process that + // exits immediately, reap it, then write to its now-closed stdin. The + // "guarded" arm wraps the write; the "unguarded" arm does not. + if let Some(mode) = std::env::var_os("ROCM_TEST_SIGPIPE_ARM") { + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + let mut grandchild = Command::new("true") + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn a process that exits immediately"); + let mut stdin = grandchild.stdin.take().expect("grandchild stdin"); + grandchild.wait().expect("reap grandchild"); + // Heap-allocated so the buffer doesn't trip clippy::large-stack-arrays; + // it must exceed the pipe buffer so the write actually reaches the + // closed read end rather than being swallowed by kernel buffering. + let payload = vec![b'x'; 64 * 1024]; + if mode == "guarded" { + let result = with_sigpipe_ignored(|| stdin.write_all(&payload)); + assert!( + result.is_err(), + "guarded write to a dead pipe should return BrokenPipe, not succeed" + ); + } else { + // Unguarded: under SIG_DFL this write delivers SIGPIPE and never + // returns. If the process somehow survives, exit 0 so the parent's + // "should have been signalled" assertion fails loudly. + let _ = stdin.write_all(&payload); + } + return; + } + + let exe = std::env::current_exe().expect("current test executable"); + let run_arm = |mode: &str| -> ExitStatus { + Command::new(&exe) + .args([ + "sigpipe::tests::engine_stdin_write_under_sig_dfl_is_guarded_not_fatal", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_SIGPIPE_ARM", mode) + .status() + .expect("re-exec the test arm in a child process") + }; + + let unguarded = run_arm("unguarded"); + assert_eq!( + unguarded.signal(), + Some(libc::SIGPIPE), + "sanity check: an unguarded stdin write under SIG_DFL must be killed by \ + SIGPIPE (got {unguarded:?}); if not, the test no longer proves the guard matters" + ); + + let guarded = run_arm("guarded"); + assert!( + guarded.success(), + "guarded child-stdin write was fatal under SIG_DFL ({guarded:?}) — \ + SIGPIPE reached the process instead of surfacing as an error" + ); + } +} diff --git a/crates/rocm-engine-protocol/Cargo.toml b/crates/rocm-engine-protocol/Cargo.toml index 6a258001c0..e0eb049751 100644 --- a/crates/rocm-engine-protocol/Cargo.toml +++ b/crates/rocm-engine-protocol/Cargo.toml @@ -20,3 +20,7 @@ anyhow.workspace = true rocm-core = { path = "../rocm-core" } serde.workspace = true serde_json.workspace = true + +[target.'cfg(unix)'.dev-dependencies] +# The SIGPIPE disposition a test installs in a re-executed child. +libc.workspace = true diff --git a/crates/rocm-engine-protocol/src/fake_engine.rs b/crates/rocm-engine-protocol/src/fake_engine.rs index 2d8baa14a7..276e9bdb6c 100644 --- a/crates/rocm-engine-protocol/src/fake_engine.rs +++ b/crates/rocm-engine-protocol/src/fake_engine.rs @@ -17,17 +17,11 @@ //! of the client are tested on every platform without it. use std::fs; -use std::io; -use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU32, Ordering}; static NEXT_ENGINE: AtomicU32 = AtomicU32::new(0); -/// `errno` for "text file busy" (`ETXTBSY`): another thread's `fork` briefly -/// held the script's write descriptor open when this one tried to execute it. -const TEXT_FILE_BUSY: i32 = 26; - pub struct FakeEngine { dir: PathBuf, script: PathBuf, @@ -38,6 +32,10 @@ impl FakeEngine { /// `stdin` first (what a well-behaved engine does); pass `false` for an /// engine that exits without reading its request. `body` runs after the /// recording and may use `$dir`, the scratch directory. + /// + /// The script is written through [`rocm_core::fake_executable`], so running + /// it straight away cannot fail with `ETXTBSY` while other tests on parallel + /// threads spawn children of their own. pub fn new(label: &str, reads_stdin: bool, body: &str) -> Self { let dir = std::env::temp_dir().join(format!( "rocm-fake-engine-{label}-{}-{}", @@ -51,7 +49,7 @@ impl FakeEngine { } else { "" }; - fs::write( + rocm_core::fake_executable::write_fake_executable( &script, format!( "#!/bin/sh\ndir='{}'\nprintf '%s\\n' \"$@\" > \"$dir/argv\"\nenv > \"$dir/env\"\n{read_stdin}{body}\n", @@ -59,8 +57,6 @@ impl FakeEngine { ), ) .expect("write the fake engine script"); - fs::set_permissions(&script, fs::Permissions::from_mode(0o755)) - .expect("make the fake engine executable"); Self { dir, script } } @@ -78,11 +74,6 @@ impl FakeEngine { &self.script } - /// The scratch directory, for a test that wants to point `$dir` somewhere. - pub fn dir(&self) -> &Path { - &self.dir - } - /// The arguments the fake was started with, one per entry. pub fn argv(&self) -> Vec { fs::read_to_string(self.dir.join("argv")) @@ -105,30 +96,6 @@ impl FakeEngine { .lines() .find_map(|line| line.strip_prefix(&prefix).map(str::to_owned)) } - - /// Run `launch`, retrying while the kernel refuses to execute the script - /// because a concurrent `fork` in this test process still holds its write - /// descriptor. A fresh script is executed the instant it is written, and the - /// test harness runs tests on parallel threads, so the window is real. - pub fn run(&self, mut launch: impl FnMut() -> anyhow::Result) -> anyhow::Result { - for attempt in 0..50 { - match launch() { - Err(error) if attempt < 49 && is_text_file_busy(&error) => { - std::thread::sleep(std::time::Duration::from_millis(20)); - } - other => return other, - } - } - unreachable!("the final attempt always returns") - } -} - -fn is_text_file_busy(error: &anyhow::Error) -> bool { - error.chain().any(|cause| { - cause - .downcast_ref::() - .is_some_and(|io| io.raw_os_error() == Some(TEXT_FILE_BUSY)) - }) } impl Drop for FakeEngine { diff --git a/crates/rocm-engine-protocol/src/lib.rs b/crates/rocm-engine-protocol/src/lib.rs index a51b427dc3..6b3f4065bf 100644 --- a/crates/rocm-engine-protocol/src/lib.rs +++ b/crates/rocm-engine-protocol/src/lib.rs @@ -5,6 +5,12 @@ #[cfg(all(unix, any(test, feature = "test-support")))] pub mod fake_engine; +mod stdio_client; +pub use stdio_client::{ + ENGINE_PROGRESS_STDERR_ENV, EngineStdioOptions, decode_engine_response, + engine_request_envelope, run_engine_stdio, +}; + use serde::Deserialize; use serde::Serialize; use serde_json::Value; @@ -283,6 +289,17 @@ impl GpuSelection { } } +/// The two arguments `flag value` when `value` is set, none when it is not, for +/// building the command line of an engine or supervisor child. Shared so `rocm` +/// and `rocmd` spell optional flags the same way. +#[must_use] +pub fn optional_arg(flag: &str, value: Option<&str>) -> Vec { + match value { + Some(value) => vec![flag.to_owned(), value.to_owned()], + None => Vec::new(), + } +} + /// Render an explicit GPU ordinal as a comma-separated `HIP_VISIBLE_DEVICES` /// value. Returns `None` when the slice is empty (no pinning requested). #[must_use] @@ -895,6 +912,15 @@ mod tests { assert!(GpuSelection::parse_cli_value("0,2,3").is_err()); } + #[test] + fn optional_arg_is_the_flag_and_value_or_nothing() { + assert_eq!( + optional_arg("--runtime-id", Some("therock-release:gfx120X-all")), + ["--runtime-id", "therock-release:gfx120X-all"] + ); + assert!(optional_arg("--runtime-id", None).is_empty()); + } + #[test] fn gpu_indices_helpers_handle_auto_and_pinned() { assert_eq!(gpu_indices_to_csv(&[]), None); diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs new file mode 100644 index 0000000000..0a247bda9a --- /dev/null +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -0,0 +1,636 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The client side of the engine stdio protocol: run an engine as a child +//! process, send it one [`EngineRequestEnvelope`] on stdin as a single line of +//! JSON, and read one [`EngineResponseEnvelope`] back from stdout. +//! +//! `rocm` and `rocmd` both talk to engines this way and share this one +//! implementation. They differ in *which* process they start (`rocm` runs the +//! resolved engine binary with `stdio`, or answers a built-in engine in its own +//! process without a child at all; `rocmd` runs `rocm __engine-stdio `), +//! and that stays with each caller. What is shared is everything that follows +//! the choice of program: the request framing, the environment, how stderr is +//! handled, the write guard against SIGPIPE, what a failing exit status or an +//! unusable answer is reported as, and decoding the response. The few things +//! the callers want differently are the fields of [`EngineStdioOptions`]. +//! +//! The request and the child's lifetime are not bounded by a timeout: an engine +//! install legitimately runs for minutes, and no caller has ever set one. + +use std::ffi::OsStr; +use std::io::{self, BufRead, Read, Write}; +use std::path::Path; +use std::process::{Command, Stdio}; +use std::thread; + +use anyhow::{Context, Result, bail}; +use rocm_core::{AppPaths, ENGINE_ENVS_ROOT_ENV, with_sigpipe_ignored}; +use serde::Serialize; +use serde::de::DeserializeOwned; + +use crate::{ + ENDPOINT_API_KEY_FILE_ENV, EngineMethod, EngineRequestEnvelope, EngineResponseEnvelope, +}; + +/// Set to `1` on an engine child whose caller wants its progress lines on +/// stderr as they happen (an engine install). +pub const ENGINE_PROGRESS_STDERR_ENV: &str = "ROCM_ENGINE_PROGRESS_STDERR"; + +/// What a caller wants different about the child [`run_engine_stdio`] starts. +/// +/// The default asks for nothing beyond the request itself: the child inherits +/// the caller's environment and its stderr is only collected. +#[derive(Debug, Clone, Copy, Default)] +pub struct EngineStdioOptions<'a> { + /// Run the child on these roots, by setting the `ROCM_CLI_*_DIR` variables + /// ([`AppPaths::env_vars`]). `None` leaves the child to inherit whatever the + /// caller's own environment says. + pub app_paths: Option<&'a AppPaths>, + /// The root engine virtualenvs go under for this request, set as + /// [`ENGINE_ENVS_ROOT_ENV`]. Only an install into a particular root needs it. + pub env_root: Option<&'a Path>, + /// A key file to hand the engine through [`ENDPOINT_API_KEY_FILE_ENV`], so + /// its HTTP probe of a protected endpoint authenticates. The caller decides + /// whether a file holds a usable key; this only carries it. + pub endpoint_key_file: Option<&'a Path>, + /// Ask the engine for progress on stderr ([`ENGINE_PROGRESS_STDERR_ENV`]). + pub request_progress: bool, + /// Write the child's stderr to this process's stderr as it arrives, so a + /// terminal user watches an install. Either way the whole of it is kept and + /// shown in the error when the request fails. + pub forward_stderr: bool, +} + +/// Build the request envelope for `method` from `request`. +pub fn engine_request_envelope( + method: EngineMethod, + request: &T, +) -> Result { + Ok(EngineRequestEnvelope { + method, + payload: serde_json::to_value(request) + .context("failed to encode engine request payload")?, + }) +} + +/// The command [`run_engine_stdio`] runs: `program` with `args`, all three +/// standard streams piped, and the environment `options` asks for. Separate so +/// a test can read the command line and environment without starting anything. +fn engine_stdio_command(program: &Path, args: I, options: &EngineStdioOptions<'_>) -> Command +where + I: IntoIterator, + S: AsRef, +{ + let mut command = Command::new(program); + command + .args(args) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + if let Some(paths) = options.app_paths { + for (key, value) in paths.env_vars() { + command.env(key, value); + } + } + if let Some(env_root) = options.env_root { + command.env(ENGINE_ENVS_ROOT_ENV, env_root); + } + if let Some(key_file) = options.endpoint_key_file { + command.env(ENDPOINT_API_KEY_FILE_ENV, key_file); + } + if options.request_progress { + command.env(ENGINE_PROGRESS_STDERR_ENV, "1"); + } + command +} + +/// Send `envelope` to the engine child `program args…` and return what it +/// answered. +/// +/// A failing exit status is an error only when the child printed no answer: an +/// engine that reports its own failure in an error envelope and then exits +/// non-zero is read as that envelope, which says more than the status does. +/// Every other failure names what went wrong and carries the child's stderr. +pub fn run_engine_stdio( + program: &Path, + args: I, + options: &EngineStdioOptions<'_>, + envelope: &EngineRequestEnvelope, +) -> Result +where + I: IntoIterator, + S: AsRef, +{ + let mut child = engine_stdio_command(program, args, options) + .spawn() + .with_context(|| format!("failed to spawn engine stdio process {}", program.display()))?; + + let mut stdin = child + .stdin + .take() + .context("engine stdio child did not expose stdin")?; + // Ignore SIGPIPE for the duration of the write. Under the default action, an + // engine that exits before reading its stdin would kill this process with + // SIGPIPE (`rocm` runs that way, and so does the daemon hosted in it); + // ignored, the write returns EPIPE and fails as an ordinary error. + let written = with_sigpipe_ignored(|| { + serde_json::to_writer(&mut stdin, envelope).context("failed to write engine request")?; + stdin.write_all(b"\n")?; + Ok::<(), anyhow::Error>(()) + }); + if let Err(error) = written { + // The engine will never get its request, so do not leave it running or + // unreaped: a long-lived caller would collect one zombie per failure. + let _ = child.kill(); + let _ = child.wait(); + return Err(error); + } + // End of the request: the engine reads to EOF before it answers. + drop(stdin); + + let forward_stderr = options.forward_stderr; + let stderr_reader = child.stderr.take().map(|stderr| { + thread::spawn(move || collect_stderr(stderr, forward_stderr.then(io::stderr))) + }); + let mut stdout = Vec::new(); + child + .stdout + .take() + .context("engine stdio child did not expose stdout")? + .read_to_end(&mut stdout) + .context("failed to read engine stdio response")?; + let status = child + .wait() + .context("failed waiting for engine stdio response")?; + let stderr = stderr_reader + .map(|reader| reader.join().unwrap_or_default()) + .unwrap_or_default(); + + if !status.success() && stdout.is_empty() { + let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); + if stderr.is_empty() { + bail!("engine stdio process exited with status {status}"); + } + bail!("engine stdio process exited with status {status}: {stderr}"); + } + serde_json::from_slice(&stdout).with_context(|| { + let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); + if stderr.is_empty() { + "failed to parse engine response envelope".to_owned() + } else { + format!("failed to parse engine response envelope; stderr: {stderr}") + } + }) +} + +/// Read `stderr` to the end line by line, optionally writing each line to +/// `forward` (this process's stderr, in production) as it arrives, and return +/// everything read. A failed forward is ignored: a closed terminal must not +/// cost the request its answer. A read error ends the collection and is itself +/// recorded as the last line. +fn collect_stderr(stderr: impl Read, mut forward: Option) -> Vec { + let mut reader = io::BufReader::new(stderr); + let mut collected = Vec::new(); + let mut line = Vec::new(); + loop { + line.clear(); + match reader.read_until(b'\n', &mut line) { + Ok(0) => break, + Ok(_) => { + if let Some(sink) = forward.as_mut() { + let _ = sink.write_all(&line); + } + collected.extend_from_slice(&line); + } + Err(error) => { + let message = format!("failed to read engine progress: {error}\n"); + if let Some(sink) = forward.as_mut() { + let _ = sink.write_all(message.as_bytes()); + } + collected.extend_from_slice(message.as_bytes()); + break; + } + } + } + collected +} + +/// Turn an engine's answer into the response type the caller asked for: the +/// engine's own error when it reported one, otherwise the decoded `data`. +pub fn decode_engine_response(envelope: EngineResponseEnvelope) -> Result +where + R: DeserializeOwned, +{ + if !envelope.ok { + let error = envelope.error.map_or_else( + || "unknown engine error".to_owned(), + |value| format!("{}: {}", value.code, value.message), + ); + bail!("{error}"); + } + let data = envelope + .data + .context("engine response envelope did not contain data")?; + serde_json::from_value(data).context("failed to decode engine response payload") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{EngineErrorDetail, StopResponse}; + use serde_json::{Value, json}; + use std::collections::BTreeMap; + + /// The environment `command` would add to its child, as plain strings. + fn added_env(command: &Command) -> BTreeMap { + command + .get_envs() + .filter_map(|(key, value)| { + Some(( + key.to_string_lossy().into_owned(), + value?.to_string_lossy().into_owned(), + )) + }) + .collect() + } + + fn args_of(command: &Command) -> Vec { + command + .get_args() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect() + } + + #[test] + fn the_default_options_add_nothing_to_the_childs_environment() { + let command = engine_stdio_command( + Path::new("/an/engine"), + ["stdio"], + &EngineStdioOptions::default(), + ); + assert_eq!(command.get_program(), "/an/engine"); + assert_eq!(args_of(&command), ["stdio"]); + assert!(added_env(&command).is_empty(), "{:?}", added_env(&command)); + } + + #[test] + fn each_option_sets_exactly_its_own_variable() { + let paths = AppPaths { + config_dir: "/c".into(), + data_dir: "/d".into(), + cache_dir: "/k".into(), + }; + let command = engine_stdio_command( + Path::new("/an/engine"), + ["__engine-stdio", "vllm"], + &EngineStdioOptions { + app_paths: Some(&paths), + env_root: Some(Path::new("/envs")), + endpoint_key_file: Some(Path::new("/keys/svc.endpoint-key")), + request_progress: true, + forward_stderr: true, + }, + ); + assert_eq!(args_of(&command), ["__engine-stdio", "vllm"]); + assert_eq!( + added_env(&command), + BTreeMap::from( + [ + ("ROCM_CLI_CONFIG_DIR", "/c"), + ("ROCM_CLI_DATA_DIR", "/d"), + ("ROCM_CLI_CACHE_DIR", "/k"), + ("ROCM_CLI_ENGINE_ENVS_ROOT", "/envs"), + ("ROCM_SERVE_API_KEY_FILE", "/keys/svc.endpoint-key"), + ("ROCM_ENGINE_PROGRESS_STDERR", "1"), + ] + .map(|(key, value)| (key.to_owned(), value.to_owned())) + ) + ); + + for (name, options) in [ + ( + "ROCM_CLI_ENGINE_ENVS_ROOT", + EngineStdioOptions { + env_root: Some(Path::new("/envs")), + ..EngineStdioOptions::default() + }, + ), + ( + "ROCM_SERVE_API_KEY_FILE", + EngineStdioOptions { + endpoint_key_file: Some(Path::new("/keys/k")), + ..EngineStdioOptions::default() + }, + ), + ( + "ROCM_ENGINE_PROGRESS_STDERR", + EngineStdioOptions { + request_progress: true, + ..EngineStdioOptions::default() + }, + ), + ] { + let command = engine_stdio_command(Path::new("/an/engine"), ["stdio"], &options); + let env = added_env(&command); + assert_eq!( + env.keys().map(String::as_str).collect::>(), + [name], + "an option must set only its own variable" + ); + } + } + + #[test] + fn the_variable_names_are_the_ones_the_engines_read() { + assert_eq!(ENGINE_ENVS_ROOT_ENV, "ROCM_CLI_ENGINE_ENVS_ROOT"); + assert_eq!(ENDPOINT_API_KEY_FILE_ENV, "ROCM_SERVE_API_KEY_FILE"); + assert_eq!(ENGINE_PROGRESS_STDERR_ENV, "ROCM_ENGINE_PROGRESS_STDERR"); + } + + #[test] + fn the_request_envelope_is_the_method_and_the_encoded_payload() { + let envelope = + engine_request_envelope(EngineMethod::Stop, &json!({"service_id": "svc"})).unwrap(); + assert_eq!(envelope.method, EngineMethod::Stop); + assert_eq!(envelope.payload, json!({"service_id": "svc"})); + assert_eq!( + serde_json::to_string(&envelope).unwrap(), + r#"{"method":"stop","payload":{"service_id":"svc"}}"# + ); + } + + #[test] + fn decode_engine_response_reports_each_way_an_answer_can_be_unusable() { + let failed = EngineResponseEnvelope { + ok: false, + data: None, + error: Some(EngineErrorDetail { + code: "stop_failed".to_owned(), + message: "it broke".to_owned(), + }), + }; + let error = decode_engine_response::(failed).unwrap_err(); + assert_eq!(error.to_string(), "stop_failed: it broke"); + + let unexplained = EngineResponseEnvelope { + ok: false, + data: None, + error: None, + }; + let error = decode_engine_response::(unexplained).unwrap_err(); + assert_eq!(error.to_string(), "unknown engine error"); + + let empty = EngineResponseEnvelope { + ok: true, + data: None, + error: None, + }; + let error = decode_engine_response::(empty).unwrap_err(); + assert_eq!( + error.to_string(), + "engine response envelope did not contain data" + ); + + let wrong_shape = EngineResponseEnvelope::success(json!({"stopped": "yes"})); + let error = decode_engine_response::(wrong_shape).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to decode engine response payload" + ); + } + + /// A stderr whose reads fail. + struct Fails; + impl Read for Fails { + fn read(&mut self, _: &mut [u8]) -> io::Result { + Err(io::Error::other("pipe went away")) + } + } + + /// A sink that refuses every write, like a closed terminal. + struct Closed; + impl Write for Closed { + fn write(&mut self, _: &[u8]) -> io::Result { + Err(io::Error::from(io::ErrorKind::BrokenPipe)) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + #[test] + fn stderr_is_collected_whole_and_a_read_error_is_recorded() { + let collected = collect_stderr(&b"first\nsecond"[..], None::>); + assert_eq!(collected, b"first\nsecond"); + + let collected = String::from_utf8(collect_stderr(Fails, None::>)).unwrap(); + assert_eq!( + collected, + "failed to read engine progress: pipe went away\n" + ); + } + + #[test] + fn forwarded_stderr_is_written_out_and_still_collected() { + let mut shown = Vec::new(); + let collected = collect_stderr(&b"step 1\nstep 2\n"[..], Some(&mut shown)); + assert_eq!(shown, b"step 1\nstep 2\n"); + assert_eq!(collected, b"step 1\nstep 2\n"); + + // A sink that refuses writes (a closed terminal) costs nothing. + let collected = collect_stderr(&b"still kept\n"[..], Some(Closed)); + assert_eq!(collected, b"still kept\n"); + } + + #[cfg(unix)] + mod child { + use super::*; + use crate::fake_engine::FakeEngine; + use std::os::unix::process::ExitStatusExt as _; + + const STOPPED: &str = r#"{"ok":true,"data":{"stopped":true}}"#; + + fn stop_request() -> EngineRequestEnvelope { + engine_request_envelope(EngineMethod::Stop, &json!({"service_id": "svc"})).unwrap() + } + + fn run(engine: &FakeEngine, options: &EngineStdioOptions<'_>) -> Result { + decode_engine_response(run_engine_stdio( + engine.path(), + ["stdio"], + options, + &stop_request(), + )?) + } + + #[test] + fn a_request_is_one_newline_terminated_line_on_the_childs_stdin() { + let engine = FakeEngine::answering("client-framing", STOPPED, 0); + let answer = run(&engine, &EngineStdioOptions::default()).unwrap(); + assert_eq!(answer, json!({"stopped": true})); + assert_eq!(engine.argv(), ["stdio"]); + assert_eq!( + engine.stdin(), + "{\"method\":\"stop\",\"payload\":{\"service_id\":\"svc\"}}\n" + ); + } + + #[test] + fn the_options_reach_the_running_child() { + let engine = FakeEngine::answering("client-options", STOPPED, 0); + let paths = AppPaths { + config_dir: "/c".into(), + data_dir: "/d".into(), + cache_dir: "/k".into(), + }; + run( + &engine, + &EngineStdioOptions { + app_paths: Some(&paths), + env_root: Some(Path::new("/envs")), + endpoint_key_file: Some(Path::new("/keys/k")), + request_progress: true, + forward_stderr: false, + }, + ) + .unwrap(); + assert_eq!(engine.env("ROCM_CLI_DATA_DIR").as_deref(), Some("/d")); + assert_eq!( + engine.env("ROCM_CLI_ENGINE_ENVS_ROOT").as_deref(), + Some("/envs") + ); + assert_eq!( + engine.env("ROCM_SERVE_API_KEY_FILE").as_deref(), + Some("/keys/k") + ); + assert_eq!( + engine.env("ROCM_ENGINE_PROGRESS_STDERR").as_deref(), + Some("1") + ); + } + + #[test] + fn a_nonzero_exit_with_no_answer_reports_the_status_and_stderr() { + let engine = FakeEngine::new("client-crash", true, "echo boom >&2\nexit 3"); + let error = run(&engine, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "engine stdio process exited with status exit status: 3: boom" + ); + + let silent = FakeEngine::new("client-silent-crash", true, "exit 4"); + let error = run(&silent, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "engine stdio process exited with status exit status: 4" + ); + } + + #[test] + fn a_nonzero_exit_that_still_answered_is_decoded_not_discarded() { + let engine = FakeEngine::answering( + "client-answers-then-fails", + r#"{"ok":false,"error":{"code":"stop_failed","message":"it broke"}}"#, + 1, + ); + let error = run(&engine, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!(error.to_string(), "stop_failed: it broke"); + + let engine = FakeEngine::answering("client-answers-then-exits-2", STOPPED, 2); + assert_eq!( + run(&engine, &EngineStdioOptions::default()).unwrap(), + json!({"stopped": true}) + ); + } + + #[test] + fn an_unparseable_answer_carries_the_childs_stderr() { + let engine = FakeEngine::new( + "client-garbage", + true, + "printf 'not json'\necho oops >&2\nexit 0", + ); + let error = run(&engine, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to parse engine response envelope; stderr: oops" + ); + + let silent = FakeEngine::new("client-garbage-quiet", true, "printf 'not json'"); + let error = run(&silent, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to parse engine response envelope" + ); + } + + #[test] + fn a_program_that_cannot_start_is_named_in_the_error() { + let error = run_engine_stdio( + Path::new("/no/such/engine"), + ["stdio"], + &EngineStdioOptions::default(), + &stop_request(), + ) + .unwrap_err(); + assert_eq!( + error.to_string(), + "failed to spawn engine stdio process /no/such/engine" + ); + } + + /// A request written to an engine that has already exited comes back as an + /// ordinary error, also where SIGPIPE is at its default action. Re-executes + /// this test in a fresh process that installs `SIG_DFL` first, because the + /// harness leaves SIGPIPE ignored and changing it process-wide would race + /// sibling tests; a write that is not guarded kills that process, and the + /// parent reports the signal. + #[test] + #[allow(unsafe_code)] // libc::signal, to install the default action in the child + fn a_request_to_an_engine_that_already_exited_is_an_error_under_sig_dfl() { + if std::env::var_os("ROCM_TEST_CLIENT_SIGPIPE_ARM").is_some() { + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + let engine = FakeEngine::new("client-sigpipe", false, "exit 3"); + // Larger than a pipe buffer, so the write reaches the closed read end. + let envelope = engine_request_envelope( + EngineMethod::Stop, + &json!({"blob": "x".repeat(1 << 20)}), + ) + .unwrap(); + let error = run_engine_stdio( + engine.path(), + ["stdio"], + &EngineStdioOptions::default(), + &envelope, + ) + .expect_err("the engine never answered"); + assert!( + format!("{error:#}").contains("failed to write engine request"), + "{error:#}" + ); + return; + } + + let status = Command::new(std::env::current_exe().unwrap()) + .args([ + "stdio_client::tests::child::a_request_to_an_engine_that_already_exited_is_an_error_under_sig_dfl", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_CLIENT_SIGPIPE_ARM", "1") + .status() + .unwrap(); + assert_eq!( + status.signal(), + None, + "SIGPIPE killed the process instead of surfacing as an error: {status:?}" + ); + assert!(status.success(), "{status:?}"); + } + } +} diff --git a/docs/architecture.md b/docs/architecture.md index ad51f2aaf6..7c1e12b11a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -59,8 +59,8 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`driver_plan/rocdxg.rs`](../apps/rocm/src/driver_plan/rocdxg.rs) — full domain extraction, a child of `driver_plan.rs` that depends on the plan model, `driver_command`, `driver_root_preflight_checks` and `resolve_shell_default_template` in its parent, and on `truthy_env` in [`therock.rs`](../apps/rocm/src/therock.rs) (split out of [`driver_install.rs`](../apps/rocm/src/driver_install.rs)): the WSL2 rocDXG plan. `wsl_rocdxg_driver_plan` builds the plan that installs the `rocdxg-roct` package, `resolve_rocdxg_verification` decides how that download is authenticated (a supplied digest, the digest pinned for the version in `ROCDXG_PINNED_DIGESTS`, or the explicit opt-out), and `wsl_rocdxg_refusal_plan` is the plan that installs nothing and says why when it cannot be authenticated or the version is not a well-formed package version. `ROCDXG_PINNED_DIGESTS` is the trust anchor an unattended root install is checked against, so a change to it is a security-relevant diff; it moved byte for byte. `wsl_rocdxg_driver_plan` is `pub(crate)` and re-exported by `driver_plan.rs` (its builders call it, and tests in `driver_install.rs` use it as a plan fixture); everything else in production code is private. Its tests are the `wsl_rocdxg_*` plan tests, including the one that runs the generated digest step against a real file on Unix. The test-only `scoped_rocdxg_env` helper that clears the rocDXG environment lives here too, `pub(super)` for the plan-builder tests in `driver_plan.rs`. - [`endpoint_keys.rs`](../apps/rocm/src/endpoint_keys.rs) — mechanical relocation, no owned types, reached by qualified path. - [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) — mechanical relocation out of [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), no owned types: which runtime an engine installs into and where its environment lives. `engine_manages_own_runtime` says whether an engine brings its own runtime; `env_root_for_engine_install`, `env_root_for_runtime` and `env_root_for_self_managed_engine` resolve the engine environment root; `runtime_manifest_for_selector` resolves a selector to one installed runtime; `runtime_key_for_python` names the runtime whose install root contains an interpreter (the private `runtime_key_owning_python` is the decision behind it). It is the neutral layer that [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) and [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) (with its `torch_alignment.rs` child) both import from, so neither imports the other for these helpers. It imports `recover_setup_runtime_registration` from [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs), `select_runtime_manifest` and `validate_runtime_manifest_for_activation` from [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), and the runtime-manifest model from [`therock.rs`](../apps/rocm/src/therock.rs); it does not go through the re-exports in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs), which would lead back to it via `runtime_services.rs`. Its functions are `pub(crate)` for those two modules plus [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs). Their tests moved with them. -- [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-213): resolving an engine's binary path (`resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, `find_engine_plugin_binary`), the stdio request/response envelope used to talk to an external engine binary or the in-process built-in handler (`engine_request`, `engine_request_with_env_root`, `decode_engine_response`, the `ScopedEnvVar`/`with_scoped_builtin_engine_env` env-scoping helpers and the `BUILTIN_ENGINE_ENV_LOCK` mutex they serialize on, `builtin_engine_request`, `run_builtin_engine_stdio`), and the built-in HTTP-serving path (`builtin_engine_available`, `run_builtin_engine_serve_http`, `builtin_engine_serve_http_args`). One item that sits physically inside this span in `main.rs` stayed there instead of moving: `impl From for WatcherMode` is unrelated clap-arg conversion that merely happened to be interleaved between `find_engine_plugin_binary` and `engine_request` (it has since moved to [`cli.rs`](../apps/rocm/src/cli.rs), next to the enum it converts). `resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, `engine_request`, `engine_request_with_env_root`, `builtin_engine_available`, and `builtin_engine_serve_http_args` are `pub(crate)` and read from other modules (for example [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) and [`service_launch.rs`](../apps/rocm/src/service_launch.rs)), not from `main.rs`. `run_builtin_engine_stdio` and `run_builtin_engine_serve_http` are `pub(crate)` and reached directly from `main.rs`'s `dispatch()` (the `Command::EngineStdio` and `__engine-serve-http` arms); `find_engine_plugin_binary`, `decode_engine_response`, `with_scoped_builtin_engine_env`, `builtin_engine_request`, and `ScopedEnvVar` have no callers outside this module and stay private. -- [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-215): deciding which runtime/env a `serve`/`engines install` invocation resolves to (`EngineSelection`, `resolve_engine_selection`, `validate_engine_selection_runtime`, `single_ready_runtime_key`, `resolve_runtime_selector_to_exact_key`), small argv-building helpers used when relaunching a managed engine (`optional_arg`, `engine_recipe_json_arg`, `parse_engine_recipe_json_arg`, `daemon_run_argv`, `managed_service_launcher_path`), the `ROCM_CLI_*` environment variables every managed child inherits (`app_path_env_vars`/`app_path_env_var_values`/`app_path_env_var_refs`/`apply_app_path_env`), and polling a freshly launched service until it can answer inference requests (`wait_for_service_http_ready`, `wait_for_service_http_ready_with_progress`, `service_http_readiness_paths`, `http_get_local_service`, `http_post_local_service_json`, `service_http_readiness_response_ready`, `lemonade_health_ready_for_model`, `model_list_ready_for_model`, `service_model_gpu_ready`, `service_model_reports_rocm_backend`, `service_model_names_match`). `single_ready_runtime_key` does not decide which runtimes count as ready: it asks [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs)'s `select_single_ready_runtime`, which `rocm comfyui install` shares. `const fn status_for_readiness` sits physically interleaved in this span in `main.rs` but stayed there instead of moving — it has callers outside this cluster ([`service_launch.rs`](../apps/rocm/src/service_launch.rs) and [`service_control.rs`](../apps/rocm/src/service_control.rs)'s `record_restart_readiness`, which `restart_internal_managed_service_with` calls), so it stays a shared crate-root function, reached only from this module's own tests, via `use crate::status_for_readiness;`. Which items are `pub(crate)` and who calls them is documented in the module's own doc comment rather than repeated here. The cluster's regression tests live in the module's own `mod tests`. What remains in `main.rs` immediately after this cluster is `treat_as_natural_language`, dispatch-adjacent natural-language-invocation routing left over from an earlier phase (ROCMAI-87), explicitly out of scope and untouched by this split. +- [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-213): resolving an engine's binary path (`resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, `find_engine_plugin_binary`), the request entry points, which either answer from the in-process built-in handler or run the resolved engine binary with `stdio` (`engine_request`, `engine_request_with_env_root`, and `engine_stdio_child_request`, which hands the child to the stdio client `rocm` shares with `rocmd`, from [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs), and chooses what `rocm` asks of it: the app paths and engine environments root for the child, progress for an install, and the child's stderr forwarded to the terminal), the `ScopedEnvVar`/`with_scoped_builtin_engine_env` env-scoping helpers and the `BUILTIN_ENGINE_ENV_LOCK` mutex they serialize on, `builtin_engine_request`, `run_builtin_engine_stdio`), and the built-in HTTP-serving path (`builtin_engine_available`, `run_builtin_engine_serve_http`, `builtin_engine_serve_http_args`). One item that sits physically inside this span in `main.rs` stayed there instead of moving: `impl From for WatcherMode` is unrelated clap-arg conversion that merely happened to be interleaved between `find_engine_plugin_binary` and `engine_request` (it has since moved to [`cli.rs`](../apps/rocm/src/cli.rs), next to the enum it converts). `resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, `engine_request`, `engine_request_with_env_root`, `builtin_engine_available`, and `builtin_engine_serve_http_args` are `pub(crate)` and read from other modules (for example [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) and [`service_launch.rs`](../apps/rocm/src/service_launch.rs)), not from `main.rs`. `run_builtin_engine_stdio` and `run_builtin_engine_serve_http` are `pub(crate)` and reached directly from `main.rs`'s `dispatch()` (the `Command::EngineStdio` and `__engine-serve-http` arms); `find_engine_plugin_binary`, `with_scoped_builtin_engine_env`, `builtin_engine_request`, and `ScopedEnvVar` have no callers outside this module and stay private. +- [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-215): deciding which runtime/env a `serve`/`engines install` invocation resolves to (`EngineSelection`, `resolve_engine_selection`, `validate_engine_selection_runtime`, `single_ready_runtime_key`, `resolve_runtime_selector_to_exact_key`), small argv-building helpers used when relaunching a managed engine (`engine_recipe_json_arg`, `parse_engine_recipe_json_arg`, `daemon_run_argv`, `managed_service_launcher_path`), the `ROCM_CLI_*` environment variables every managed child inherits (`app_path_env_vars`, which is [`AppPaths::env_vars`](../crates/rocm-core/src/paths.rs), `app_path_env_var_values`/`app_path_env_var_refs`/`apply_app_path_env`), and polling a freshly launched service until it can answer inference requests (`wait_for_service_http_ready`, `wait_for_service_http_ready_with_progress`, `service_http_readiness_paths`, `http_get_local_service`, `http_post_local_service_json`, `service_http_readiness_response_ready`, `lemonade_health_ready_for_model`, `model_list_ready_for_model`, `service_model_gpu_ready`, `service_model_reports_rocm_backend`, `service_model_names_match`). `single_ready_runtime_key` does not decide which runtimes count as ready: it asks [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs)'s `select_single_ready_runtime`, which `rocm comfyui install` shares. `const fn status_for_readiness` sits physically interleaved in this span in `main.rs` but stayed there instead of moving — it has callers outside this cluster ([`service_launch.rs`](../apps/rocm/src/service_launch.rs) and [`service_control.rs`](../apps/rocm/src/service_control.rs)'s `record_restart_readiness`, which `restart_internal_managed_service_with` calls), so it stays a shared crate-root function, reached only from this module's own tests, via `use crate::status_for_readiness;`. Which items are `pub(crate)` and who calls them is documented in the module's own doc comment rather than repeated here. The cluster's regression tests live in the module's own `mod tests`. What remains in `main.rs` immediately after this cluster is `treat_as_natural_language`, dispatch-adjacent natural-language-invocation routing left over from an earlier phase (ROCMAI-87), explicitly out of scope and untouched by this split. - [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) — mechanical relocation, owns private types (Phase 6a): `engines()` (the `rocm engines` command), `ensure_self_managed_engine_ready` (the readiness check [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) runs for engines that bring their own runtime) and `env_root_for_service` (read from [`service_launch.rs`](../apps/rocm/src/service_launch.rs) and [`service_control.rs`](../apps/rocm/src/service_control.rs)). Which runtime an engine installs into and where its environment lives is answered in [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs), which this module imports from. This module also calls into [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) for `settle_engine_install` and the host system-package installs; that module does not import this one. - [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs) — mechanical relocation, dispatch-adjacent, owns private types: `rocm examine`'s command handler (`examine()`), which prints the human report through [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs)'s `examine_human_report`, or with `--json` the document built from the private `ExamineJson`/`ExamineJsonSummary` types (the `Examination` verbatim plus a summary of what the CLI knows but the probe cannot). `examine()` is `pub(crate)` and re-imported in [`main.rs`](../apps/rocm/src/main.rs) for `dispatch()`. No test in the crate calls it or builds the document directly. - [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs) — mechanical relocation: the launch-summary/examine text surfaces (`render_launch_summary`, `render_examine_text`, `render_examine_text_with_paths`, `examine_human_report`, `render_examine_plain_header`, `plain_status_label`) and the engine-inventory listing shared by `rocm engines` and `rocm examine` (`write_default_engine_legend`, `render_engine_inventory_text_with_paths`, `append_examine_runtime_state`, `append_examine_engine_inventory`, `append_engine_detect_summary`, `friendly_engine_runtime_kind`, `friendly_engine_detect_notes`, `friendly_engine_detect_note_fallback`, `engine_runtime_status_label`). The `DEFAULT_ENGINE_MARKER` constant is shared by both engine-inventory renderers so the two copies cannot drift apart. The `rocm model` catalog renderer is not here; it lives in [`model_catalog_text.rs`](../apps/rocm/src/model_catalog_text.rs), and nothing in this file calls into it. [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs)'s `examine()`, which the `Command::Examine` arm calls, imports and calls `examine_human_report`; `main.rs`'s own `launch_default` calls `render_launch_summary`, re-imported there. `render_examine_text`, `render_examine_text_with_paths` and `render_engine_inventory_text_with_paths` are `pub(crate)`, read from [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs)'s read-only command runner; `render_engine_inventory_text_with_paths` is also read from [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs)'s `render_engine_inventory_text`; `append_examine_runtime_state` is also `pub(crate)`, read only from [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)'s own test module, so `main.rs`'s re-import of it is itself `#[cfg(test)]` rather than carrying an otherwise-unused import into a non-test build. Every other item in this list has no caller outside this module and stays private. This module also reaches items that live at the crate root because they have other callers: `engine_inventory()` (also read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs) and [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs)), `PlatformRuleOut` and `engine_ruled_out_by_platform` (defined in [`model_fit.rs`](../apps/rocm/src/model_fit.rs) and re-imported in `main.rs`) — all reached via `use crate::{...};`. The regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` and `write_test_pip_runtime`/`runtime_manifest_path` through `crate::test_support::`/`crate::runtimes_cmd::`. Its tests do not call into `model_catalog_text.rs`: the test of the note the verbose `rocm model` listing prints for a rule-out engine lives beside that renderer. @@ -145,7 +145,7 @@ Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): - [`cli.rs`](../apps/rocmd/src/cli.rs) — the `Cli`/`Command` clap definitions, the bridge-snapshot diagnostic printer, the pretty-JSON printer for the subcommands that print a result (`print_json`), and the top-level dispatch in `run_cli`/`run_bin_cli`/`run_from_args`. - [`common.rs`](../apps/rocmd/src/common.rs) — helpers the other modules call into: the read-only `rocm update` message, the `rocm`-subprocess capture wrapper (`run_rocm_capture_for_paths`) the sandbox and MCP clusters both call (it only chooses the child's `ROCM_CLI_*_DIR` environment; `CommandCapture` and the spawn itself live in [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), shared with `rocm`'s chat tool bridge, and the bounded-execution half in [`process.rs`](../crates/rocm-core/src/process.rs)), and small arg/JSON-payload utilities (`payload_string`, for example, is read by the [`watchers/cache_warm.rs`](../apps/rocmd/src/watchers/cache_warm.rs), [`watchers/driver_upgrade.rs`](../apps/rocmd/src/watchers/driver_upgrade.rs) and [`watchers/gpu_thermal_protect.rs`](../apps/rocmd/src/watchers/gpu_thermal_protect.rs) handlers and by [`webhook.rs`](../apps/rocmd/src/webhook.rs)). - [`daemon.rs`](../apps/rocmd/src/daemon.rs) — the foreground `rocmd run` loop (`run_daemon`: evaluate the watchers each tick, dispatch local webhook events, write the runtime state, stop on a signal; with no watcher enabled it exits by itself, deciding that under the autostart lock and publishing "stopped" and clearing its spawn claim before releasing it — `record_daemon_stop`, `release_own_autostart_claim`) and `rocmd status` (`print_status`), with the watcher tick interval, which `LoopHooks` makes injectable for tests. It uses [`watchers.rs`](../apps/rocmd/src/watchers.rs), [`webhook.rs`](../apps/rocmd/src/webhook.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs); [`cli.rs`](../apps/rocmd/src/cli.rs) calls it. -- [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) — talking to an engine and guarding its endpoint key: `engine_request` (the stdio request to an engine process), the healthcheck call and its readiness and recoverability checks, and the endpoint-key helpers `apply_endpoint_key_env` and `ensure_public_service_has_endpoint_key`. It uses no other module of the crate; [`service.rs`](../apps/rocmd/src/service.rs), [`supervise.rs`](../apps/rocmd/src/supervise.rs) and [`watchers/server_recover.rs`](../apps/rocmd/src/watchers/server_recover.rs) call it. +- [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) — talking to an engine and guarding its endpoint key: `engine_request` (the stdio request to an engine process, run through the same client `rocm` uses, [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs); `rocmd` chooses what to run, `__engine-stdio ` of its own executable, and what to ask of the client: its own app paths, the service's endpoint key file when it holds a usable key, and stderr kept for the error rather than forwarded), the healthcheck call and its readiness and recoverability checks, and the endpoint-key helpers `apply_endpoint_key_env` and `ensure_public_service_has_endpoint_key`. It uses no other module of the crate; [`service.rs`](../apps/rocmd/src/service.rs), [`supervise.rs`](../apps/rocmd/src/supervise.rs) and [`watchers/server_recover.rs`](../apps/rocmd/src/watchers/server_recover.rs) call it. - [`host_snapshot.rs`](../apps/rocmd/src/host_snapshot.rs) — the host snapshot: `build_bridge_snapshot` (the bridge snapshot printed by `rocmd bridge-snapshot` and returned by the `bridge_snapshot` MCP tool, assembled from the examine summary, the GPU snapshot, the automation runtime state and recent events, the engine inventory and the managed services), the amd-smi GPU snapshot (`gather_gpu_snapshot_for_config`, which honours the telemetry setting, over time-bounded amd-smi probes) and `bridge_engine_inventory`. It uses only [`persistence.rs`](../apps/rocmd/src/persistence.rs); [`cli.rs`](../apps/rocmd/src/cli.rs), [`mcp.rs`](../apps/rocmd/src/mcp.rs) and [`watchers.rs`](../apps/rocmd/src/watchers.rs) call it. - [`mcp.rs`](../apps/rocmd/src/mcp.rs) — the MCP stdio server, tool schema table, tool dispatch, and the result-shaping helpers behind the MCP tools (the `rocm` subprocess itself is spawned through [`common.rs`](../apps/rocmd/src/common.rs), via `run_rocm_capture` here). The argument handling the dispatch calls is in [`mcp_argv.rs`](../apps/rocmd/src/mcp_argv.rs). - [`mcp_argv.rs`](../apps/rocmd/src/mcp_argv.rs) — the argument handling for the MCP tools that spawn `rocm`: `normalized_rocm_command_args` and the read-only allowlist `ensure_rocm_command_is_read_only` behind the `rocm_command` tool, and the argv builders for `install_sdk`, `install_engine`, `launch_server` and `watcher_enable` (`build_install_sdk_args`, `build_install_engine_args`, `build_launch_server_args`, `build_watcher_enable_args`, with the public-bind and system-prefix acknowledgement checks they apply). Functions over the tool's JSON arguments: it uses no other module of the crate, spawns nothing and writes nothing, and [`mcp.rs`](../apps/rocmd/src/mcp.rs) is its only caller. Its own `mod tests` holds the allowlist and argv-builder tests. @@ -194,7 +194,7 @@ Already-extracted subsystem modules, following the same full-domain-extraction p - [`model_catalog.rs`](../crates/rocm-core/src/model_catalog.rs) — (ROCMAI-152, Phase 7c) the curated model catalog's types, built-in catalog JSON loading, signed external recipe indexes, and artifact cache-marker bookkeeping — see the module's own doc comment for what it owns. - [`model_readiness.rs`](../crates/rocm-core/src/model_readiness.rs) — the `rocm diagnose --model` fit assessment: curated-recipe lookup against a host's measured GPU/RAM. `rocm model --verbose` renders its own fit labels in [`model_catalog_text.rs`](../apps/rocm/src/model_catalog_text.rs), which uses exactly two items from this module: `curated_alternatives`, called directly, and `recipe_display_ref`, reached through a private one-line wrapper of the same name in that file. For a GGUF outside the catalog it also attaches a context-size estimate (the three `gguf*` modules above). - [`net.rs`](../crates/rocm-core/src/net.rs) — (ROCMAI-154) networking primitives: the default local host and port, URL/endpoint formatting and parsing, and the hand-rolled loopback HTTP/TCP client (`http_get_text*`, `http_post_json_with_auth`, `read_http_response_bounded`). It depends on no other service module: the managed-service readiness probes built on it live in [`endpoint_readiness.rs`](../crates/rocm-core/src/endpoint_readiness.rs) and the streaming downloads in [`download.rs`](../crates/rocm-core/src/download.rs). -- [`paths.rs`](../crates/rocm-core/src/paths.rs) — (ROCMAI-154, Phase 7b) `AppPaths` and its filesystem-location primitives. +- [`paths.rs`](../crates/rocm-core/src/paths.rs) — `AppPaths` and its filesystem-location primitives, including `AppPaths::env_vars`, the three `ROCM_CLI_*_DIR` variables that make a child process run on the same roots, and `ENGINE_ENVS_ROOT_ENV`. - [`placeholder.rs`](../crates/rocm-core/src/placeholder.rs) — the `` placeholder rendered reports print for an empty value: `empty_as_unknown` (returns the value untouched) and `trimmed_or_unknown` (trims first), shared by `apps/rocm`'s driver-install plan and chat tool summaries and by [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs)'s Windows GPU diagnostics, replacing the copy those two `apps/rocm` callers shared in [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) and `host_gpu.rs`'s own trimming copy. - [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs) — the host probes' subprocess and file-read helpers, split out of [`examine.rs`](../crates/rocm-core/src/examine.rs) so `fix.rs`, `process.rs` and the child modules of [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) reach them without importing the module that owns the `Examination` document: `run`/`run_with_env`/`run_raw` (run a command under a timeout; `rc` is `127` when it cannot be spawned and `124` on timeout), `run_utf16le`/`decode_utf16le` (for `wsl.exe`'s UTF-16LE output), `read_text`/`read_host_text`, `which`, the `SHORT`/`MEDIUM` timeouts and `truncate_to_chars`. All are `pub(crate)` except `run_raw`, which stays private to the module. The module itself is private to the crate and is not re-exported. - [`process.rs`](../crates/rocm-core/src/process.rs) — (ROCMAI-154, Phase 7b) process-spawn/terminate primitives, including `FileLock`, plus `run_with_timeout`: it drains both pipes on their own threads from the moment the child starts, so output size cannot stall a command into its own timeout, and it bounds how much of a timed-out child's output the error quotes, because that text reaches persisted automation logs. It is not yet the *only* such helper — [`therock/process.rs`](../apps/rocm/src/therock/process.rs)'s `wait_with_output_bounded`, [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs)'s `run_raw`, and [`host_gpu/capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs)'s `capture_optional_command_candidate_with_timeout` (plus the Windows-only `capture_diagnostic_command` in [`host_gpu/diagnostics.rs`](../crates/rocm-core/src/host_gpu/diagnostics.rs)) still join their reader threads with no time limit, so a grandchild holding the pipe can outlast their timeout; converging them onto `run_with_timeout` is follow-up work to ROCMAI-448 tracked in [#623](https://github.com/ROCm/rocm-cli/issues/623). @@ -235,6 +235,7 @@ Further `rocm-core` modules, each with its own doc comment except where noted: - [`report.rs`](../crates/rocm-core/src/report.rs) — the content of a Doctor report and the rule deciding whether one may exist; it sends nothing. - [`report_delivery.rs`](../crates/rocm-core/src/report_delivery.rs) — how a report leaves the machine: building its link and deciding whether a browser may be started. - [`runtime.rs`](../crates/rocm-core/src/runtime.rs) — the host-platform layer: `RuntimePlatform`/`RuntimeHost`, OS-specific executable and Python-environment names (including `runtime_python_env_root_for_executable`, the one rule for finding an environment's folder from its interpreter's path: only an interpreter directly inside a `bin` or `Scripts` folder, in any case, has one, and everything else gets `None`; the adopt command, the ComfyUI runtime environment and the vLLM launch all call it rather than keeping a copy, since the engine crate cannot reach `apps/rocm`), default config/data/cache directories, path-text and path-list helpers, and `install_root_has_payload`, the one definition of a non-empty install root (dot-entries such as the runtime marker do not count; a path that does not exist holds nothing; a root that cannot be listed is an error) that tarball extraction, update repair detection and activation validation in `apps/rocm` all ask (no module doc comment). +- [`sigpipe.rs`](../crates/rocm-core/src/sigpipe.rs) — `with_sigpipe_ignored`, which turns a write to a pipe whose reader has gone into an ordinary error instead of a signal; `rocm` runs with SIGPIPE at its default action, so the engine stdio client wraps its write to the child in it. - [`terminal.rs`](../crates/rocm-core/src/terminal.rs) — one ECMA-48 walk over untrusted terminal output, shared by every caller that has to quote or re-render it (`serve_summary.rs`, `vllm/startup_diagnostics.rs`, `vllm_oom.rs`, `diagnose/engine.rs`). - [`user_path.rs`](../crates/rocm-core/src/user_path.rs) — removing the install directory from the per-user Windows `Path` that the installer added it to. - [`uv.rs`](../crates/rocm-core/src/uv.rs) — acquiring the `uv` package manager into the managed cache, plus the argument/environment helpers `apps/rocm` and the vLLM engine share. @@ -303,7 +304,9 @@ The other three dashboard crates are described at crate level here, not module b A contract surface: verify all impacted engines after any protocol change here (see [`AGENTS.md`](../AGENTS.md)). -- [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs) — the whole crate: the protocol and recipe-contract version constants, the request/response envelopes and the per-method request and response types for the `EngineMethod` calls (`detect`, `install`, `capabilities`, `resolve_model`, `launch`, `healthcheck`, `endpoint`, `stop`, `logs`), the `DevicePolicy` and `GpuSelection` types with the GPU-visibility helpers engines apply to a child process, engine-plugin binary naming and discovery, and the endpoint API key file carrier (`ENDPOINT_API_KEY_FILE_ENV`, `resolve_endpoint_api_key`, `is_public_bind_host`) that `rocm` and `rocmd` share. +- [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) — the client side of the engine stdio protocol, the one implementation `rocm` and `rocmd` both run an engine child through: the request line, the child's environment (`EngineStdioOptions`: app paths, engine environments root, endpoint key file, progress request), stderr handling (kept for the error, optionally forwarded as it arrives), the SIGPIPE guard around the write, what a failing exit status or an unusable answer is reported as, and `decode_engine_response`. Each caller still chooses which program to run: `rocm` the resolved engine binary with `stdio` (or the built-in handler, with no child), `rocmd` its own executable with `__engine-stdio `. It adds no wire format; the envelope types are the ones in [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs). +- [`fake_engine.rs`](../crates/rocm-engine-protocol/src/fake_engine.rs) — test-only, behind the `test-support` feature the `rocm` and `rocmd` dev-dependencies turn on: a shell-script stand-in for an engine child that records its argv, environment and stdin and answers (or crashes) as a test asks, so both callers' tests run against the same fake. +- [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs) — the whole crate: the protocol and recipe-contract version constants, the request/response envelopes and the per-method request and response types for the `EngineMethod` calls (`detect`, `install`, `capabilities`, `resolve_model`, `launch`, `healthcheck`, `endpoint`, `stop`, `logs`), the `DevicePolicy` and `GpuSelection` types with the GPU-visibility helpers engines apply to a child process, engine-plugin binary naming and discovery, and the endpoint API key file carrier (`ENDPOINT_API_KEY_FILE_ENV`, `resolve_endpoint_api_key`, `is_public_bind_host`) that `rocm` and `rocmd` share, and `optional_arg`, the `flag value` argument helper both binaries build child command lines with. ### `crates/rocm-deps` diff --git a/docs/testing.md b/docs/testing.md index b3369520b6..6518078c5e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -205,6 +205,27 @@ child write the file instead, so this process never holds a write fd on it. The same applies to E2E steps: scenarios run concurrently in one process, and the `rocm` child that runs the shim can be the one refused. +### The engine stdio client + +`rocm` and `rocmd` run an engine child through one client, +[`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) in +`rocm-engine-protocol`. Its conversation (the request line, the environment each +option sets, what a failing exit or an unusable answer is reported as, the +SIGPIPE guard) is tested there against `FakeEngine`, a shell-script engine child +from [`fake_engine.rs`](../crates/rocm-engine-protocol/src/fake_engine.rs) that +records its argv, environment and stdin; the pure parts (the command a call +builds, `decode_engine_response`, stderr collection) run on every platform, and +the ones that start the fake are `#[cfg(unix)]`. The `rocm` and `rocmd` callers +each run their own entry point against the same fake and pin only what they +choose: the program and verb, the app paths, and whether stderr is forwarded (see +`apps/rocm/src/engine_ipc.rs` and `apps/rocmd/src/engine_ipc.rs`). A caller that +grew its own private copy of the conversation would fail those tests. + +The SIGPIPE tests re-execute the test binary in a fresh process with SIGPIPE at +its default action, which is how `rocm` runs, because the test harness leaves it +ignored and changing it process-wide would race sibling tests. A write that is +not guarded kills that process, and the parent test reports the signal. + ### Advised commands must parse `apps/rocm/src/advised_commands.rs` checks the `rocm`/`rocmd` commands that the From d2ac4eb5d207c866c4ea7a5aaf7a89857421a17d Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 21:01:33 +0000 Subject: [PATCH 03/38] fix(rocmd): reach engines through the rocm executable when running as the standalone daemon rocmd asked an engine to do something by re-executing its own executable as `__engine-stdio `. That verb is served by rocm. Embedded as `rocm daemon` the daemon is rocm and it works, but the standalone rocmd binary shipped beside rocm answered every such request with "unrecognized subcommand '__engine-stdio'" and exit status 2. The request was made by the stop tool (`rocmd sandbox-tool stop_server`, `rocmd mcp-server`), by the supervisor's readiness wait and by the server-recover watcher's healthcheck, so from the standalone daemon the engine `Stop` always came back as an error and no healthcheck ever succeeded; for an engine whose stop decides confirmation, the stop could never be confirmed. When the running executable is exactly `rocmd`, the request now runs the `rocm` beside it (rocm_core::sibling_binary_path), and says so if there is none. Anything else, the embedded daemon, a test binary or a renamed copy, keeps using its own executable as before. The `service-stop-02` scenario drives the standalone rocmd and now also asserts that the engine `Stop` request was answered. Run against the previous commit's rocmd it fails with the engine_stop error above; with this change it passes, along with the rest of the service-stop scenarios. The choice of executable is a pure function with its own unit test that also runs on Windows. Signed-off-by: Jussi Elo --- apps/rocmd/src/engine_ipc.rs | 79 ++++++++++++++++++- docs/architecture.md | 4 +- docs/testing.md | 3 +- .../features/service_stop.feature | 1 + .../tests/e2e/service_stop_steps.rs | 18 +++++ 5 files changed, 98 insertions(+), 7 deletions(-) diff --git a/apps/rocmd/src/engine_ipc.rs b/apps/rocmd/src/engine_ipc.rs index e238d45c02..d0fcfa29ed 100644 --- a/apps/rocmd/src/engine_ipc.rs +++ b/apps/rocmd/src/engine_ipc.rs @@ -110,8 +110,9 @@ pub(crate) fn ensure_public_service_has_endpoint_key( /// Ask an engine a question about a managed service, through the stdio client /// `rocmd` shares with `rocm`. /// -/// The engine is reached by re-executing this process's own executable as -/// `__engine-stdio `, the verb `rocm` serves for its built-in engines. +/// The engine is reached by running the `rocm` executable as +/// `__engine-stdio `, the verb `rocm` serves for its built-in engines +/// ([`rocm_executable`] says which executable that is). pub(crate) fn engine_request( paths: &AppPaths, engine: &str, @@ -124,11 +125,39 @@ where R: DeserializeOwned, { let envelope = engine_request_envelope(method, request)?; - let engine_binary = - std::env::current_exe().context("failed to resolve current rocm executable path")?; + let engine_binary = rocm_executable()?; engine_request_via(&engine_binary, paths, engine, service_id, &envelope) } +/// The `rocm` executable that serves `__engine-stdio`. +/// +/// `rocmd` runs in two shapes. Embedded, as `rocm daemon`, this process +/// is `rocm` and its own executable is the answer. Run as the standalone +/// `rocmd` binary shipped beside `rocm`, its own executable has no such verb +/// (its parser answers "unrecognized subcommand"), so every engine request +/// failed; there the answer is the `rocm` next to it. +fn rocm_executable() -> Result { + let current = + std::env::current_exe().context("failed to resolve current rocm executable path")?; + rocm_executable_from(current, || rocm_core::sibling_binary_path("rocm")) +} + +/// [`rocm_executable`] given this process's executable and the way to find a +/// sibling `rocm`, so the choice can be tested without either. +fn rocm_executable_from( + current: PathBuf, + sibling_rocm: impl FnOnce() -> Result, +) -> Result { + // Exactly `rocmd` (or `rocmd.exe`). A renamed or test binary is not known to + // lack the verb, and has no `rocm` beside it to prefer. + if current.file_stem().is_some_and(|stem| stem == "rocmd") { + return sibling_rocm().context( + "the standalone rocmd needs the rocm executable beside it to reach an engine", + ); + } + Ok(current) +} + /// [`engine_request`] with the executable that serves `__engine-stdio` named by /// the caller, so a test can point it at a stand-in. fn engine_request_via( @@ -347,6 +376,48 @@ mod tests { // `rocmd` chooses: the verb, the environment and the stderr handling. The // other caller's choices are pinned in `apps/rocm/src/engine_ipc.rs`. + #[test] + fn the_standalone_rocmd_reaches_engines_through_the_rocm_beside_it() { + let beside = || Ok(PathBuf::from("/opt/rocm-cli/bin/rocm")); + + // The standalone binary has no `__engine-stdio` verb, so it must not + // re-execute itself. + for standalone in ["/opt/rocm-cli/bin/rocmd", "C:/rocm-cli/bin/rocmd.exe"] { + assert_eq!( + rocm_executable_from(PathBuf::from(standalone), beside).unwrap(), + Path::new("/opt/rocm-cli/bin/rocm"), + "{standalone}" + ); + } + + // Embedded as `rocm daemon`, this process is `rocm`; a test binary + // or a renamed copy has no `rocm` beside it to prefer. + for own in [ + "/opt/rocm-cli/bin/rocm", + "C:/rocm-cli/bin/rocm.exe", + "/work/target/debug/deps/rocmd-0f3a9c1d", + "/opt/rocm-cli/bin/rocm-0.3", + ] { + let chosen = rocm_executable_from(PathBuf::from(own), || { + panic!("{own} has no use for a sibling") + }) + .unwrap(); + assert_eq!(chosen, Path::new(own)); + } + + // A standalone `rocmd` with no `rocm` beside it fails with a message that + // says why, rather than running something that cannot serve the verb. + let error = rocm_executable_from(PathBuf::from("/opt/rocm-cli/bin/rocmd"), || { + anyhow::bail!("unable to locate sibling binary rocm") + }) + .unwrap_err(); + assert_eq!( + format!("{error:#}"), + "the standalone rocmd needs the rocm executable beside it to reach an engine: \ + unable to locate sibling binary rocm" + ); + } + #[test] fn rocmd_runs_the_child_on_its_own_paths_keeps_stderr_and_asks_for_no_progress() { let (_root, paths) = temp_app_paths("stdio-options"); diff --git a/docs/architecture.md b/docs/architecture.md index 7c1e12b11a..c748089e5e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -145,7 +145,7 @@ Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): - [`cli.rs`](../apps/rocmd/src/cli.rs) — the `Cli`/`Command` clap definitions, the bridge-snapshot diagnostic printer, the pretty-JSON printer for the subcommands that print a result (`print_json`), and the top-level dispatch in `run_cli`/`run_bin_cli`/`run_from_args`. - [`common.rs`](../apps/rocmd/src/common.rs) — helpers the other modules call into: the read-only `rocm update` message, the `rocm`-subprocess capture wrapper (`run_rocm_capture_for_paths`) the sandbox and MCP clusters both call (it only chooses the child's `ROCM_CLI_*_DIR` environment; `CommandCapture` and the spawn itself live in [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), shared with `rocm`'s chat tool bridge, and the bounded-execution half in [`process.rs`](../crates/rocm-core/src/process.rs)), and small arg/JSON-payload utilities (`payload_string`, for example, is read by the [`watchers/cache_warm.rs`](../apps/rocmd/src/watchers/cache_warm.rs), [`watchers/driver_upgrade.rs`](../apps/rocmd/src/watchers/driver_upgrade.rs) and [`watchers/gpu_thermal_protect.rs`](../apps/rocmd/src/watchers/gpu_thermal_protect.rs) handlers and by [`webhook.rs`](../apps/rocmd/src/webhook.rs)). - [`daemon.rs`](../apps/rocmd/src/daemon.rs) — the foreground `rocmd run` loop (`run_daemon`: evaluate the watchers each tick, dispatch local webhook events, write the runtime state, stop on a signal; with no watcher enabled it exits by itself, deciding that under the autostart lock and publishing "stopped" and clearing its spawn claim before releasing it — `record_daemon_stop`, `release_own_autostart_claim`) and `rocmd status` (`print_status`), with the watcher tick interval, which `LoopHooks` makes injectable for tests. It uses [`watchers.rs`](../apps/rocmd/src/watchers.rs), [`webhook.rs`](../apps/rocmd/src/webhook.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs); [`cli.rs`](../apps/rocmd/src/cli.rs) calls it. -- [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) — talking to an engine and guarding its endpoint key: `engine_request` (the stdio request to an engine process, run through the same client `rocm` uses, [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs); `rocmd` chooses what to run, `__engine-stdio ` of its own executable, and what to ask of the client: its own app paths, the service's endpoint key file when it holds a usable key, and stderr kept for the error rather than forwarded), the healthcheck call and its readiness and recoverability checks, and the endpoint-key helpers `apply_endpoint_key_env` and `ensure_public_service_has_endpoint_key`. It uses no other module of the crate; [`service.rs`](../apps/rocmd/src/service.rs), [`supervise.rs`](../apps/rocmd/src/supervise.rs) and [`watchers/server_recover.rs`](../apps/rocmd/src/watchers/server_recover.rs) call it. +- [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) — talking to an engine and guarding its endpoint key: `engine_request` (the stdio request to an engine process, run through the same client `rocm` uses, [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs); `rocmd` chooses what to run, `__engine-stdio ` of the `rocm` executable (its own when embedded as `rocm daemon`, the `rocm` beside it when it is the standalone `rocmd`, which has no such verb), and what to ask of the client: its own app paths, the service's endpoint key file when it holds a usable key, and stderr kept for the error rather than forwarded), the healthcheck call and its readiness and recoverability checks, and the endpoint-key helpers `apply_endpoint_key_env` and `ensure_public_service_has_endpoint_key`. It uses no other module of the crate; [`service.rs`](../apps/rocmd/src/service.rs), [`supervise.rs`](../apps/rocmd/src/supervise.rs) and [`watchers/server_recover.rs`](../apps/rocmd/src/watchers/server_recover.rs) call it. - [`host_snapshot.rs`](../apps/rocmd/src/host_snapshot.rs) — the host snapshot: `build_bridge_snapshot` (the bridge snapshot printed by `rocmd bridge-snapshot` and returned by the `bridge_snapshot` MCP tool, assembled from the examine summary, the GPU snapshot, the automation runtime state and recent events, the engine inventory and the managed services), the amd-smi GPU snapshot (`gather_gpu_snapshot_for_config`, which honours the telemetry setting, over time-bounded amd-smi probes) and `bridge_engine_inventory`. It uses only [`persistence.rs`](../apps/rocmd/src/persistence.rs); [`cli.rs`](../apps/rocmd/src/cli.rs), [`mcp.rs`](../apps/rocmd/src/mcp.rs) and [`watchers.rs`](../apps/rocmd/src/watchers.rs) call it. - [`mcp.rs`](../apps/rocmd/src/mcp.rs) — the MCP stdio server, tool schema table, tool dispatch, and the result-shaping helpers behind the MCP tools (the `rocm` subprocess itself is spawned through [`common.rs`](../apps/rocmd/src/common.rs), via `run_rocm_capture` here). The argument handling the dispatch calls is in [`mcp_argv.rs`](../apps/rocmd/src/mcp_argv.rs). - [`mcp_argv.rs`](../apps/rocmd/src/mcp_argv.rs) — the argument handling for the MCP tools that spawn `rocm`: `normalized_rocm_command_args` and the read-only allowlist `ensure_rocm_command_is_read_only` behind the `rocm_command` tool, and the argv builders for `install_sdk`, `install_engine`, `launch_server` and `watcher_enable` (`build_install_sdk_args`, `build_install_engine_args`, `build_launch_server_args`, `build_watcher_enable_args`, with the public-bind and system-prefix acknowledgement checks they apply). Functions over the tool's JSON arguments: it uses no other module of the crate, spawns nothing and writes nothing, and [`mcp.rs`](../apps/rocmd/src/mcp.rs) is its only caller. Its own `mod tests` holds the allowlist and argv-builder tests. @@ -304,7 +304,7 @@ The other three dashboard crates are described at crate level here, not module b A contract surface: verify all impacted engines after any protocol change here (see [`AGENTS.md`](../AGENTS.md)). -- [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) — the client side of the engine stdio protocol, the one implementation `rocm` and `rocmd` both run an engine child through: the request line, the child's environment (`EngineStdioOptions`: app paths, engine environments root, endpoint key file, progress request), stderr handling (kept for the error, optionally forwarded as it arrives), the SIGPIPE guard around the write, what a failing exit status or an unusable answer is reported as, and `decode_engine_response`. Each caller still chooses which program to run: `rocm` the resolved engine binary with `stdio` (or the built-in handler, with no child), `rocmd` its own executable with `__engine-stdio `. It adds no wire format; the envelope types are the ones in [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs). +- [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) — the client side of the engine stdio protocol, the one implementation `rocm` and `rocmd` both run an engine child through: the request line, the child's environment (`EngineStdioOptions`: app paths, engine environments root, endpoint key file, progress request), stderr handling (kept for the error, optionally forwarded as it arrives), the SIGPIPE guard around the write, what a failing exit status or an unusable answer is reported as, and `decode_engine_response`. Each caller still chooses which program to run: `rocm` the resolved engine binary with `stdio` (or the built-in handler, with no child), `rocmd` the `rocm` executable with `__engine-stdio `. It adds no wire format; the envelope types are the ones in [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs). - [`fake_engine.rs`](../crates/rocm-engine-protocol/src/fake_engine.rs) — test-only, behind the `test-support` feature the `rocm` and `rocmd` dev-dependencies turn on: a shell-script stand-in for an engine child that records its argv, environment and stdin and answers (or crashes) as a test asks, so both callers' tests run against the same fake. - [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs) — the whole crate: the protocol and recipe-contract version constants, the request/response envelopes and the per-method request and response types for the `EngineMethod` calls (`detect`, `install`, `capabilities`, `resolve_model`, `launch`, `healthcheck`, `endpoint`, `stop`, `logs`), the `DevicePolicy` and `GpuSelection` types with the GPU-visibility helpers engines apply to a child process, engine-plugin binary naming and discovery, and the endpoint API key file carrier (`ENDPOINT_API_KEY_FILE_ENV`, `resolve_endpoint_api_key`, `is_public_bind_host`) that `rocm` and `rocmd` share, and `optional_arg`, the `flag value` argument helper both binaries build child command lines with. diff --git a/docs/testing.md b/docs/testing.md index 6518078c5e..b0d8689ead 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1817,7 +1817,8 @@ unconfirmed rather than having its PIDs cleared. `features/service_stop.feature` (`service-stop-02`, Linux, mock lane) drives the real `rocmd sandbox-tool stop_server` against a live recorded process and checks the confirmed report against the process, the record and -the endpoint key file. Run it with +the endpoint key file, and that the standalone `rocmd` got an answer to its +engine `Stop` request (it reaches the engine through the `rocm` beside it). Run it with `ROCM_CLI_BINARY=target/debug/rocm ROCM_CLI_ROCMD_BINARY=target/debug/rocmd cargo test -p e2e-cucumber --test e2e -- -n "service-stop-02"`. `service-stop-03`, `-04` and `-05` (Linux, mock lane) cover `rocm services diff --git a/tests/e2e-cucumber/features/service_stop.feature b/tests/e2e-cucumber/features/service_stop.feature index cae0e9859a..61edcb0009 100644 --- a/tests/e2e-cucumber/features/service_stop.feature +++ b/tests/e2e-cucumber/features/service_stop.feature @@ -59,6 +59,7 @@ Feature: Stopping a managed service Given a managed server whose recorded process is running When the daemon's stop_server tool stops that server Then the tool reports the server stopped, naming the process it stopped + And the tool asked the engine to stop and the engine answered And the recorded process is no longer running And the server's record reads stopped and names no process And the server's endpoint key file is gone diff --git a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs index 31df329d2b..405fdcc557 100644 --- a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs @@ -311,6 +311,24 @@ async fn tool_reports_stopped(world: &mut E2eWorld) { ); } +/// The standalone `rocmd` reaches an engine through the `rocm` beside it (it has +/// no engine verb of its own), so its `Stop` request is answered rather than +/// failing with the standalone binary's "unrecognized subcommand". The report +/// carries the engine's answer or the error that took its place. +#[then("the tool asked the engine to stop and the engine answered")] +async fn engine_answered_the_stop(world: &mut E2eWorld) { + let stdout = world.cli_output.as_deref().unwrap_or_default(); + let report: serde_json::Value = + serde_json::from_str(stdout).unwrap_or_else(|error| panic!("{error}: {stdout}")); + let engine_stop = &report["result"]["engine_stop"]; + assert_eq!(engine_stop["attempted"], true, "{report}"); + assert!( + engine_stop.get("error").is_none(), + "the engine's answer was an error instead: {report}" + ); + assert!(engine_stop["stopped"].is_boolean(), "{report}"); +} + #[then("the recorded process is no longer running")] async fn recorded_process_gone(world: &mut E2eWorld) { let process = world From 8d58bf036d2b4148d7c8c940e300ca5d557678ac Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 05:39:45 +0000 Subject: [PATCH 04/38] fix(engine-protocol): guard the stderr forwarding writes against SIGPIPE `rocm` runs with SIGPIPE at its default action. When it forwards an engine child's progress to a stderr whose reader has gone (a closed pipe), the write delivers the signal and ends the CLI before `write_all` can return, so discarding the result did not protect the request. Both forwarding writes in `collect_stderr` (a line of progress, and the note recorded when reading the child's stderr fails) now go through one helper that runs the write inside `with_sigpipe_ignored`, which turns the signal into the error that is then discarded. `with_sigpipe_ignored` is now safe to overlap. It saved and restored the disposition per call, so two overlapping regions (the stderr forwarder of one request next to the stdin write of another in the daemon) could restore "ignored" over a default action, or the default under a region still open. It remembers the disposition in force before the first region and restores it when the last one ends, and restores on unwind. The re-executed test installs SIG_DFL in a fresh process and forwards into a real pipe whose reader is closed, once per write; the existing `Closed` mock returns the error itself and cannot raise the signal. A second re-executed test covers nested and interleaved regions. Signed-off-by: Jussi Elo --- crates/rocm-core/src/sigpipe.rs | 132 ++++++++++++++++-- .../rocm-engine-protocol/src/stdio_client.rs | 67 ++++++++- docs/architecture.md | 4 +- docs/testing.md | 7 +- 4 files changed, 195 insertions(+), 15 deletions(-) diff --git a/crates/rocm-core/src/sigpipe.rs b/crates/rocm-core/src/sigpipe.rs index 07c5f8b44d..4ed357c587 100644 --- a/crates/rocm-core/src/sigpipe.rs +++ b/crates/rocm-core/src/sigpipe.rs @@ -19,17 +19,55 @@ /// (Rust's startup default) this changes nothing. The disposition is /// process-wide, so keep `f` to the write itself. Windows has no SIGPIPE and /// runs `f` as is. +/// +/// Guarded regions may overlap, on one thread or on several (the stderr +/// forwarder of one engine request runs while another request writes to its +/// child). The disposition in force before the first region is remembered once +/// and put back when the last region ends, so an overlap can neither restore +/// "ignored" over a default action nor leave "ignored" behind. #[cfg(unix)] -#[allow(unsafe_code)] // libc FFI pub fn with_sigpipe_ignored(f: impl FnOnce() -> T) -> T { - // SAFETY: reading/installing a signal disposition is always safe. - let previous = unsafe { libc::signal(libc::SIGPIPE, libc::SIG_IGN) }; - let result = f(); - // SAFETY: restoring the disposition captured above is always safe. - unsafe { - libc::signal(libc::SIGPIPE, previous); + let _guard = unix::IgnoreGuard::enter(); + f() +} + +#[cfg(unix)] +#[allow(unsafe_code)] // libc FFI +mod unix { + use std::sync::{Mutex, PoisonError}; + + /// How many guarded regions are open, and the disposition that was in force + /// before the first of them. + static OPEN: Mutex<(usize, libc::sighandler_t)> = Mutex::new((0, libc::SIG_DFL)); + + /// Holds SIGPIPE ignored from [`IgnoreGuard::enter`] until it is dropped, so + /// a panic inside the guarded region still restores the disposition. + pub(super) struct IgnoreGuard; + + impl IgnoreGuard { + pub(super) fn enter() -> Self { + let mut open = OPEN.lock().unwrap_or_else(PoisonError::into_inner); + if open.0 == 0 { + // SAFETY: reading/installing a signal disposition is always safe. + open.1 = unsafe { libc::signal(libc::SIGPIPE, libc::SIG_IGN) }; + } + open.0 += 1; + Self + } + } + + impl Drop for IgnoreGuard { + fn drop(&mut self) { + let mut open = OPEN.lock().unwrap_or_else(PoisonError::into_inner); + open.0 -= 1; + if open.0 == 0 { + // SAFETY: restoring the disposition captured by the first guard. + unsafe { + libc::signal(libc::SIGPIPE, open.1); + } + } + } } - result } #[cfg(not(unix))] @@ -44,6 +82,84 @@ mod tests { use std::os::unix::process::ExitStatusExt as _; use std::process::{Command, ExitStatus, Stdio}; + /// The disposition SIGPIPE has now (read by installing the same one back). + #[allow(unsafe_code)] // libc::signal + fn current_disposition() -> libc::sighandler_t { + // SAFETY: reading/installing a signal disposition is always safe. + let current = unsafe { libc::signal(libc::SIGPIPE, libc::SIG_IGN) }; + // SAFETY: putting back what was just read. + unsafe { + libc::signal(libc::SIGPIPE, current); + } + current + } + + /// Guarded regions that overlap, nested on one thread or interleaved across + /// two, leave SIGPIPE ignored for as long as any of them is open and put the + /// original default action back only when the last one closes. With a guard + /// that saved and restored per call, the region that ended first would + /// restore "ignored" over the second's `SIG_DFL` capture, or restore the + /// default under the region still open. Runs in a re-executed child, since + /// the disposition is process-wide and the harness leaves it ignored. + #[test] + #[allow(unsafe_code)] // libc::signal, to install the default action in the child + fn overlapping_guards_restore_the_original_disposition_once() { + if std::env::var_os("ROCM_TEST_SIGPIPE_OVERLAP_ARM").is_some() { + use std::sync::Barrier; + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + + // Nested on one thread. + with_sigpipe_ignored(|| { + with_sigpipe_ignored(|| {}); + assert_eq!( + current_disposition(), + libc::SIG_IGN, + "the inner region ended while the outer one was open" + ); + }); + assert_eq!(current_disposition(), libc::SIG_DFL, "nested"); + + // Two threads whose regions overlap, the first to enter leaving first. + let entered = Barrier::new(2); + let first_left = Barrier::new(2); + std::thread::scope(|scope| { + scope.spawn(|| { + with_sigpipe_ignored(|| { + entered.wait(); + }); + first_left.wait(); + }); + scope.spawn(|| { + with_sigpipe_ignored(|| { + entered.wait(); + first_left.wait(); + assert_eq!( + current_disposition(), + libc::SIG_IGN, + "the first region ended while this one was open" + ); + }); + }); + }); + assert_eq!(current_disposition(), libc::SIG_DFL, "interleaved"); + return; + } + + let status = Command::new(std::env::current_exe().expect("current test executable")) + .args([ + "sigpipe::tests::overlapping_guards_restore_the_original_disposition_once", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_SIGPIPE_OVERLAP_ARM", "1") + .status() + .expect("re-exec the test arm in a child process"); + assert!(status.success(), "{status:?}"); + } + /// A write to a dead child's stdin under the default SIGPIPE action is /// fatal, and the same write inside [`with_sigpipe_ignored`] returns an /// error instead. diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 0a247bda9a..4b7d2a08c6 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -188,8 +188,11 @@ where /// Read `stderr` to the end line by line, optionally writing each line to /// `forward` (this process's stderr, in production) as it arrives, and return /// everything read. A failed forward is ignored: a closed terminal must not -/// cost the request its answer. A read error ends the collection and is itself -/// recorded as the last line. +/// cost the request its answer. That holds for the signal as well as the error: +/// under SIGPIPE's default action (`rocm` runs that way) a write to a closed +/// pipe ends the whole process before `write_all` can return, so each forward +/// runs inside [`with_sigpipe_ignored`] (see [`forward_best_effort`]). A read error ends the +/// collection and is itself recorded as the last line. fn collect_stderr(stderr: impl Read, mut forward: Option) -> Vec { let mut reader = io::BufReader::new(stderr); let mut collected = Vec::new(); @@ -200,14 +203,14 @@ fn collect_stderr(stderr: impl Read, mut forward: Option) -> Vec Ok(0) => break, Ok(_) => { if let Some(sink) = forward.as_mut() { - let _ = sink.write_all(&line); + forward_best_effort(sink, &line); } collected.extend_from_slice(&line); } Err(error) => { let message = format!("failed to read engine progress: {error}\n"); if let Some(sink) = forward.as_mut() { - let _ = sink.write_all(message.as_bytes()); + forward_best_effort(sink, message.as_bytes()); } collected.extend_from_slice(message.as_bytes()); break; @@ -217,6 +220,14 @@ fn collect_stderr(stderr: impl Read, mut forward: Option) -> Vec collected } +/// Write `bytes` to the forwarding sink and drop the result: a sink that has +/// gone away must not cost the request its answer. SIGPIPE is ignored for the +/// write, so a closed pipe comes back as the error that is then discarded +/// rather than as a signal that kills the process. +fn forward_best_effort(sink: &mut impl Write, bytes: &[u8]) { + let _ = with_sigpipe_ignored(|| sink.write_all(bytes)); +} + /// Turn an engine's answer into the response type the caller asked for: the /// engine's own error when it reported one, otherwise the decoded `data`. pub fn decode_engine_response(envelope: EngineResponseEnvelope) -> Result @@ -581,6 +592,54 @@ mod tests { ); } + /// Progress forwarded to a stderr whose reader has gone away is dropped, not + /// fatal, also where SIGPIPE is at its default action (`rocm` runs that + /// way). The `Closed` sink above cannot show this: it returns the error + /// itself, where the real failure is a signal that ends the process before + /// any error is returned. So this re-executes itself in a fresh process + /// that installs `SIG_DFL` and forwards into one end of a real pipe whose + /// other end is closed. Two arms, one per forwarding write in + /// `collect_stderr`: a line of the child's stderr, and the note recorded + /// when reading that stderr fails. + #[test] + #[allow(unsafe_code)] // libc::signal, to install the default action in the child + fn forwarding_stderr_to_a_closed_pipe_is_not_fatal_under_sig_dfl() { + if let Some(arm) = std::env::var_os("ROCM_TEST_CLIENT_FORWARD_ARM") { + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + let (reader, writer) = io::pipe().unwrap(); + drop(reader); + let collected = if arm == "line" { + collect_stderr(&b"progress\n"[..], Some(writer)) + } else { + collect_stderr(Fails, Some(writer)) + }; + // Surviving is the point; the text is still kept for the error. + assert!(!collected.is_empty()); + return; + } + + for arm in ["line", "read-error"] { + let status = Command::new(std::env::current_exe().unwrap()) + .args([ + "stdio_client::tests::child::forwarding_stderr_to_a_closed_pipe_is_not_fatal_under_sig_dfl", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_CLIENT_FORWARD_ARM", arm) + .status() + .unwrap(); + assert_eq!( + status.signal(), + None, + "{arm}: SIGPIPE killed the process instead of the write failing: {status:?}" + ); + assert!(status.success(), "{arm}: {status:?}"); + } + } + /// A request written to an engine that has already exited comes back as an /// ordinary error, also where SIGPIPE is at its default action. Re-executes /// this test in a fresh process that installs `SIG_DFL` first, because the diff --git a/docs/architecture.md b/docs/architecture.md index c748089e5e..acd24d31e8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -235,7 +235,7 @@ Further `rocm-core` modules, each with its own doc comment except where noted: - [`report.rs`](../crates/rocm-core/src/report.rs) — the content of a Doctor report and the rule deciding whether one may exist; it sends nothing. - [`report_delivery.rs`](../crates/rocm-core/src/report_delivery.rs) — how a report leaves the machine: building its link and deciding whether a browser may be started. - [`runtime.rs`](../crates/rocm-core/src/runtime.rs) — the host-platform layer: `RuntimePlatform`/`RuntimeHost`, OS-specific executable and Python-environment names (including `runtime_python_env_root_for_executable`, the one rule for finding an environment's folder from its interpreter's path: only an interpreter directly inside a `bin` or `Scripts` folder, in any case, has one, and everything else gets `None`; the adopt command, the ComfyUI runtime environment and the vLLM launch all call it rather than keeping a copy, since the engine crate cannot reach `apps/rocm`), default config/data/cache directories, path-text and path-list helpers, and `install_root_has_payload`, the one definition of a non-empty install root (dot-entries such as the runtime marker do not count; a path that does not exist holds nothing; a root that cannot be listed is an error) that tarball extraction, update repair detection and activation validation in `apps/rocm` all ask (no module doc comment). -- [`sigpipe.rs`](../crates/rocm-core/src/sigpipe.rs) — `with_sigpipe_ignored`, which turns a write to a pipe whose reader has gone into an ordinary error instead of a signal; `rocm` runs with SIGPIPE at its default action, so the engine stdio client wraps its write to the child in it. +- [`sigpipe.rs`](../crates/rocm-core/src/sigpipe.rs) — `with_sigpipe_ignored`, which turns a write to a pipe whose reader has gone into an ordinary error instead of a signal; `rocm` runs with SIGPIPE at its default action, so the engine stdio client wraps its write to the child, and each write forwarding the child's stderr to its own, in it. Guarded regions may overlap, across threads too; the disposition in force before the first is put back when the last ends. - [`terminal.rs`](../crates/rocm-core/src/terminal.rs) — one ECMA-48 walk over untrusted terminal output, shared by every caller that has to quote or re-render it (`serve_summary.rs`, `vllm/startup_diagnostics.rs`, `vllm_oom.rs`, `diagnose/engine.rs`). - [`user_path.rs`](../crates/rocm-core/src/user_path.rs) — removing the install directory from the per-user Windows `Path` that the installer added it to. - [`uv.rs`](../crates/rocm-core/src/uv.rs) — acquiring the `uv` package manager into the managed cache, plus the argument/environment helpers `apps/rocm` and the vLLM engine share. @@ -304,7 +304,7 @@ The other three dashboard crates are described at crate level here, not module b A contract surface: verify all impacted engines after any protocol change here (see [`AGENTS.md`](../AGENTS.md)). -- [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) — the client side of the engine stdio protocol, the one implementation `rocm` and `rocmd` both run an engine child through: the request line, the child's environment (`EngineStdioOptions`: app paths, engine environments root, endpoint key file, progress request), stderr handling (kept for the error, optionally forwarded as it arrives), the SIGPIPE guard around the write, what a failing exit status or an unusable answer is reported as, and `decode_engine_response`. Each caller still chooses which program to run: `rocm` the resolved engine binary with `stdio` (or the built-in handler, with no child), `rocmd` the `rocm` executable with `__engine-stdio `. It adds no wire format; the envelope types are the ones in [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs). +- [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) — the client side of the engine stdio protocol, the one implementation `rocm` and `rocmd` both run an engine child through: the request line, the child's environment (`EngineStdioOptions`: app paths, engine environments root, endpoint key file, progress request), stderr handling (kept for the error, optionally forwarded as it arrives), the SIGPIPE guard around the write and around each forward of the child's stderr, what a failing exit status or an unusable answer is reported as, and `decode_engine_response`. Each caller still chooses which program to run: `rocm` the resolved engine binary with `stdio` (or the built-in handler, with no child), `rocmd` the `rocm` executable with `__engine-stdio `. It adds no wire format; the envelope types are the ones in [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs). - [`fake_engine.rs`](../crates/rocm-engine-protocol/src/fake_engine.rs) — test-only, behind the `test-support` feature the `rocm` and `rocmd` dev-dependencies turn on: a shell-script stand-in for an engine child that records its argv, environment and stdin and answers (or crashes) as a test asks, so both callers' tests run against the same fake. - [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs) — the whole crate: the protocol and recipe-contract version constants, the request/response envelopes and the per-method request and response types for the `EngineMethod` calls (`detect`, `install`, `capabilities`, `resolve_model`, `launch`, `healthcheck`, `endpoint`, `stop`, `logs`), the `DevicePolicy` and `GpuSelection` types with the GPU-visibility helpers engines apply to a child process, engine-plugin binary naming and discovery, and the endpoint API key file carrier (`ENDPOINT_API_KEY_FILE_ENV`, `resolve_endpoint_api_key`, `is_public_bind_host`) that `rocm` and `rocmd` share, and `optional_arg`, the `flag value` argument helper both binaries build child command lines with. diff --git a/docs/testing.md b/docs/testing.md index b0d8689ead..990e3e4711 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -224,7 +224,12 @@ grew its own private copy of the conversation would fail those tests. The SIGPIPE tests re-execute the test binary in a fresh process with SIGPIPE at its default action, which is how `rocm` runs, because the test harness leaves it ignored and changing it process-wide would race sibling tests. A write that is -not guarded kills that process, and the parent test reports the signal. +not guarded kills that process, and the parent test reports the signal. Two +writes are covered that way: the request to the child's stdin, and the forwarding +of the child's stderr to this process's stderr (a line of progress, and the note +recorded when reading it fails). The forwarding test writes into a real pipe whose +reader is closed, because a mock sink that returns `BrokenPipe` itself never +raises the signal. ### Advised commands must parse From 52ef4ee543939f5158cd2e33d96226f2386e462e Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 05:41:34 +0000 Subject: [PATCH 05/38] fix(engine-protocol): quote what the fake engine puts in shell literals `FakeEngine` built its script by interpolating the canned response and the scratch directory into single-quoted shell literals. A valid JSON response with an apostrophe (`can't stop`) ended the literal early, so the fake exited with a syntax error instead of printing the envelope, and a `TMPDIR` containing an apostrophe broke the `dir=` line the same way. Both values now go through one `shell_quote` (close the literal, emit an escaped quote, reopen it), and `FakeEngine::new_in` lets a test choose the scratch base without changing the process-wide TMPDIR. Tests cover the quoting helper, the response and the scratch directory against the script itself, and the apostrophe response through the real client. Signed-off-by: Jussi Elo --- .../rocm-engine-protocol/src/fake_engine.rs | 77 ++++++++++++++++++- .../rocm-engine-protocol/src/stdio_client.rs | 13 ++++ docs/testing.md | 4 +- 3 files changed, 89 insertions(+), 5 deletions(-) diff --git a/crates/rocm-engine-protocol/src/fake_engine.rs b/crates/rocm-engine-protocol/src/fake_engine.rs index 276e9bdb6c..dc90a42449 100644 --- a/crates/rocm-engine-protocol/src/fake_engine.rs +++ b/crates/rocm-engine-protocol/src/fake_engine.rs @@ -22,6 +22,14 @@ use std::sync::atomic::{AtomicU32, Ordering}; static NEXT_ENGINE: AtomicU32 = AtomicU32::new(0); +/// `value` as a single-quoted POSIX shell word. Inside single quotes nothing is +/// special except the quote itself, which cannot be escaped there: close the +/// literal, emit an escaped quote, and reopen it (`'\''`). An apostrophe in a +/// response (`can't stop`) or in `TMPDIR` would otherwise end the literal early. +fn shell_quote(value: &str) -> String { + format!("'{}'", value.replace('\'', r"'\''")) +} + pub struct FakeEngine { dir: PathBuf, script: PathBuf, @@ -37,7 +45,13 @@ impl FakeEngine { /// it straight away cannot fail with `ETXTBSY` while other tests on parallel /// threads spawn children of their own. pub fn new(label: &str, reads_stdin: bool, body: &str) -> Self { - let dir = std::env::temp_dir().join(format!( + Self::new_in(&std::env::temp_dir(), label, reads_stdin, body) + } + + /// [`FakeEngine::new`] with its scratch directory created under `base` + /// instead of the system temporary directory. + pub fn new_in(base: &Path, label: &str, reads_stdin: bool, body: &str) -> Self { + let dir = base.join(format!( "rocm-fake-engine-{label}-{}-{}", std::process::id(), NEXT_ENGINE.fetch_add(1, Ordering::Relaxed) @@ -52,8 +66,8 @@ impl FakeEngine { rocm_core::fake_executable::write_fake_executable( &script, format!( - "#!/bin/sh\ndir='{}'\nprintf '%s\\n' \"$@\" > \"$dir/argv\"\nenv > \"$dir/env\"\n{read_stdin}{body}\n", - dir.display() + "#!/bin/sh\ndir={}\nprintf '%s\\n' \"$@\" > \"$dir/argv\"\nenv > \"$dir/env\"\n{read_stdin}{body}\n", + shell_quote(&dir.to_string_lossy()) ), ) .expect("write the fake engine script"); @@ -66,7 +80,7 @@ impl FakeEngine { Self::new( label, true, - &format!("printf '%s' '{response}'\nexit {status}"), + &format!("printf '%s' {}\nexit {status}", shell_quote(response)), ) } @@ -103,3 +117,58 @@ impl Drop for FakeEngine { let _ = fs::remove_dir_all(&self.dir); } } + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write as _; + use std::process::{Command, Stdio}; + + /// Run `engine` the way a client does: request on stdin, answer on stdout. + fn run(engine: &FakeEngine) -> (std::process::ExitStatus, String, String) { + let mut child = Command::new(engine.path()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + child.stdin.take().unwrap().write_all(b"request\n").unwrap(); + let output = child.wait_with_output().unwrap(); + ( + output.status, + String::from_utf8_lossy(&output.stdout).into_owned(), + String::from_utf8_lossy(&output.stderr).into_owned(), + ) + } + + #[test] + fn shell_quote_survives_each_character_a_literal_can_hold() { + assert_eq!(shell_quote("plain"), "'plain'"); + assert_eq!(shell_quote("can't"), r"'can'\''t'"); + assert_eq!(shell_quote("''"), r"''\'''\'''"); + assert_eq!(shell_quote(""), "''"); + } + + #[test] + fn a_response_with_an_apostrophe_is_printed_verbatim() { + let response = r#"{"ok":false,"error":{"code":"x","message":"can't stop"}}"#; + let engine = FakeEngine::answering("fake-apostrophe-response", response, 0); + let (status, stdout, stderr) = run(&engine); + assert!(status.success(), "{status:?}: {stderr}"); + assert_eq!(stdout, response); + assert_eq!(stderr, "", "the script must parse"); + } + + #[test] + fn a_scratch_directory_with_an_apostrophe_still_records_the_launch() { + let base = + std::env::temp_dir().join(format!("rocm-fake-engine-it's-{}", std::process::id())); + let engine = FakeEngine::new_in(&base, "fake-apostrophe-dir", true, "exit 0"); + let (status, _, stderr) = run(&engine); + assert!(status.success(), "{status:?}: {stderr}"); + assert_eq!(stderr, "", "the script must parse"); + assert_eq!(engine.stdin(), "request\n"); + drop(engine); + let _ = fs::remove_dir_all(&base); + } +} diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 4b7d2a08c6..12f200d171 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -488,6 +488,19 @@ mod tests { ); } + #[test] + fn a_response_with_an_apostrophe_reaches_the_caller_intact() { + // The fake writes its answer through a shell literal; an apostrophe + // in valid JSON must not end that literal early. + let engine = FakeEngine::answering( + "client-apostrophe", + r#"{"ok":false,"error":{"code":"stop_failed","message":"can't stop"}}"#, + 0, + ); + let error = run(&engine, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!(error.to_string(), "stop_failed: can't stop"); + } + #[test] fn the_options_reach_the_running_child() { let engine = FakeEngine::answering("client-options", STOPPED, 0); diff --git a/docs/testing.md b/docs/testing.md index 990e3e4711..49c2efe6af 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -229,7 +229,9 @@ writes are covered that way: the request to the child's stdin, and the forwardin of the child's stderr to this process's stderr (a line of progress, and the note recorded when reading it fails). The forwarding test writes into a real pipe whose reader is closed, because a mock sink that returns `BrokenPipe` itself never -raises the signal. +raises the signal. `FakeEngine` quotes the response and its scratch directory +for the shell, so a response such as `can't stop`, or a `TMPDIR` containing an +apostrophe, reaches the client intact. ### Advised commands must parse From 1f7804b1a56569ad2cb004747285469a86e94b95 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 14:36:09 +0000 Subject: [PATCH 06/38] fix(engine-protocol): kill and reap the engine child when its answer cannot be read run_engine_stdio killed and reaped the child when writing the request failed, but a failed read of its stdout returned through `?` first, leaving the child running or as a zombie. The client now runs in the long-lived daemon, so each such failure leaked a process. The read moves into read_response, which kills and waits on the child before returning the error; the stderr collector is not joined on that path because a grandchild holding the pipe could block the join, and the thread ends once the pipe closes. The test reads through a failing reader on a real child and checks that its /proc entry is gone, which a kill without a wait would not achieve. Signed-off-by: Jussi Elo --- .../rocm-engine-protocol/src/stdio_client.rs | 85 +++++++++++++++++-- docs/architecture.md | 2 +- docs/testing.md | 8 ++ 3 files changed, 88 insertions(+), 7 deletions(-) diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 12f200d171..ed7c9c1fd4 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -22,7 +22,7 @@ use std::ffi::OsStr; use std::io::{self, BufRead, Read, Write}; use std::path::Path; -use std::process::{Command, Stdio}; +use std::process::{Child, Command, Stdio}; use std::thread; use anyhow::{Context, Result, bail}; @@ -154,13 +154,11 @@ where let stderr_reader = child.stderr.take().map(|stderr| { thread::spawn(move || collect_stderr(stderr, forward_stderr.then(io::stderr))) }); - let mut stdout = Vec::new(); - child + let stdout_pipe = child .stdout .take() - .context("engine stdio child did not expose stdout")? - .read_to_end(&mut stdout) - .context("failed to read engine stdio response")?; + .context("engine stdio child did not expose stdout")?; + let stdout = read_response(&mut child, stdout_pipe)?; let status = child .wait() .context("failed waiting for engine stdio response")?; @@ -185,6 +183,24 @@ where }) } +/// Read the child's whole answer from `stdout`. +/// +/// A failed read would leave the child running with nobody waiting on it, and +/// this client lives in the long-lived daemon, so the child is killed and +/// reaped before the error is returned, as it is when the request cannot be +/// written. The stderr collector is not joined on that path: a grandchild that +/// inherited the pipe could hold it open, and the thread ends by itself once +/// the pipe closes. +fn read_response(child: &mut Child, mut stdout: impl Read) -> Result> { + let mut response = Vec::new(); + if let Err(error) = stdout.read_to_end(&mut response) { + let _ = child.kill(); + let _ = child.wait(); + return Err(anyhow::Error::new(error).context("failed to read engine stdio response")); + } + Ok(response) +} + /// Read `stderr` to the end line by line, optionally writing each line to /// `forward` (this process's stderr, in production) as it arrives, and return /// everything read. A failed forward is ignored: a closed terminal must not @@ -590,6 +606,63 @@ mod tests { ); } + /// An answer whose read fails kills the child and reaps it, so a daemon + /// that hits this repeatedly collects neither a running engine nor a + /// zombie per failure. A zombie keeps its `/proc` entry until it is + /// waited on, so the entry being gone shows the reap and not only the + /// kill (hence Linux only). + #[cfg(target_os = "linux")] + #[test] + fn a_failed_read_of_the_answer_kills_and_reaps_the_child() { + let mut child = Command::new("sleep") + .arg("60") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let pid = child.id(); + let proc_entry = format!("/proc/{pid}"); + assert!(Path::new(&proc_entry).exists(), "the child is running"); + + let error = read_response(&mut child, Fails).unwrap_err(); + + assert_eq!(error.to_string(), "failed to read engine stdio response"); + assert_eq!( + format!("{error:#}"), + "failed to read engine stdio response: pipe went away" + ); + assert!( + !Path::new(&proc_entry).exists(), + "the child must be killed and reaped, not left running or as a zombie" + ); + // The status was collected by the reap, so it is still reported. + assert_eq!( + child.try_wait().unwrap().and_then(|status| status.signal()), + Some(libc::SIGKILL) + ); + } + + /// The success path of the same helper hands the bytes back and leaves + /// the child alone. + #[test] + fn a_successful_read_of_the_answer_leaves_the_child_to_the_caller() { + let mut child = Command::new("sleep") + .arg("60") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + + let bytes = read_response(&mut child, &b"{\"ok\":true}"[..]).unwrap(); + + assert_eq!(bytes, b"{\"ok\":true}"); + assert!(child.try_wait().unwrap().is_none(), "still running"); + child.kill().unwrap(); + child.wait().unwrap(); + } + #[test] fn a_program_that_cannot_start_is_named_in_the_error() { let error = run_engine_stdio( diff --git a/docs/architecture.md b/docs/architecture.md index acd24d31e8..dda42c54fa 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -304,7 +304,7 @@ The other three dashboard crates are described at crate level here, not module b A contract surface: verify all impacted engines after any protocol change here (see [`AGENTS.md`](../AGENTS.md)). -- [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) — the client side of the engine stdio protocol, the one implementation `rocm` and `rocmd` both run an engine child through: the request line, the child's environment (`EngineStdioOptions`: app paths, engine environments root, endpoint key file, progress request), stderr handling (kept for the error, optionally forwarded as it arrives), the SIGPIPE guard around the write and around each forward of the child's stderr, what a failing exit status or an unusable answer is reported as, and `decode_engine_response`. Each caller still chooses which program to run: `rocm` the resolved engine binary with `stdio` (or the built-in handler, with no child), `rocmd` the `rocm` executable with `__engine-stdio `. It adds no wire format; the envelope types are the ones in [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs). +- [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) — the client side of the engine stdio protocol, the one implementation `rocm` and `rocmd` both run an engine child through: the request line, the child's environment (`EngineStdioOptions`: app paths, engine environments root, endpoint key file, progress request), stderr handling (kept for the error, optionally forwarded as it arrives), the SIGPIPE guard around the write and around each forward of the child's stderr, killing and reaping the child when its request cannot be written or its answer cannot be read, what a failing exit status or an unusable answer is reported as, and `decode_engine_response`. Each caller still chooses which program to run: `rocm` the resolved engine binary with `stdio` (or the built-in handler, with no child), `rocmd` the `rocm` executable with `__engine-stdio `. It adds no wire format; the envelope types are the ones in [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs). - [`fake_engine.rs`](../crates/rocm-engine-protocol/src/fake_engine.rs) — test-only, behind the `test-support` feature the `rocm` and `rocmd` dev-dependencies turn on: a shell-script stand-in for an engine child that records its argv, environment and stdin and answers (or crashes) as a test asks, so both callers' tests run against the same fake. - [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs) — the whole crate: the protocol and recipe-contract version constants, the request/response envelopes and the per-method request and response types for the `EngineMethod` calls (`detect`, `install`, `capabilities`, `resolve_model`, `launch`, `healthcheck`, `endpoint`, `stop`, `logs`), the `DevicePolicy` and `GpuSelection` types with the GPU-visibility helpers engines apply to a child process, engine-plugin binary naming and discovery, and the endpoint API key file carrier (`ENDPOINT_API_KEY_FILE_ENV`, `resolve_endpoint_api_key`, `is_public_bind_host`) that `rocm` and `rocmd` share, and `optional_arg`, the `flag value` argument helper both binaries build child command lines with. diff --git a/docs/testing.md b/docs/testing.md index 49c2efe6af..94feb26b1f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -233,6 +233,14 @@ raises the signal. `FakeEngine` quotes the response and its scratch directory for the shell, so a response such as `can't stop`, or a `TMPDIR` containing an apostrophe, reaches the client intact. +A child whose answer cannot be read is killed and reaped before the error is +returned, as it is when the request cannot be written, so the long-lived daemon +does not collect a running engine or a zombie per failure. +`a_failed_read_of_the_answer_kills_and_reaps_the_child` (Linux only) runs the +read step on a real child with a reader that fails and checks that the child's +`/proc` entry is gone: a killed child that was never waited on would keep it as a +zombie. + ### Advised commands must parse `apps/rocm/src/advised_commands.rs` checks the `rocm`/`rocmd` commands that the From 40942ffc32a8d42fd3892ac524da01c5d80c8972 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 20:39:13 +0000 Subject: [PATCH 07/38] test(core): enter the two overlapping SIGPIPE guards in a fixed order The interleaved half of overlapping_guards_restore_the_original_disposition_once only made sure both guards were open before the first left, not that the first thread to be spawned entered first. If the other closure entered first, a guard that saved and restored the disposition per call passed both assertions. The second thread now waits on a barrier that the first reaches from inside its guard, so the first region always opens, and closes, before the second opens its own; with a per-call save and restore in place of the shared counter the test fails on every run (five of five). Signed-off-by: Jussi Elo --- crates/rocm-core/src/sigpipe.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/crates/rocm-core/src/sigpipe.rs b/crates/rocm-core/src/sigpipe.rs index 4ed357c587..68192204b1 100644 --- a/crates/rocm-core/src/sigpipe.rs +++ b/crates/rocm-core/src/sigpipe.rs @@ -122,17 +122,25 @@ mod tests { }); assert_eq!(current_disposition(), libc::SIG_DFL, "nested"); - // Two threads whose regions overlap, the first to enter leaving first. + // Two threads whose regions overlap, the first to enter leaving + // first. The second thread waits until the first is inside its + // region before it enters its own, so the order is the same on + // every run: were it left to the scheduler, a per-call save and + // restore would pass whenever the "second" closure happened to run + // first. + let first_in = Barrier::new(2); let entered = Barrier::new(2); let first_left = Barrier::new(2); std::thread::scope(|scope| { scope.spawn(|| { with_sigpipe_ignored(|| { + first_in.wait(); entered.wait(); }); first_left.wait(); }); scope.spawn(|| { + first_in.wait(); with_sigpipe_ignored(|| { entered.wait(); first_left.wait(); From 06371bbda3fdb4c44b9369d0111653401e4d4095 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 07:10:17 +0000 Subject: [PATCH 08/38] test(e2e): drive the daemon's stop tool against an engine that exits non-zero or prints something unreadable The engine client shared by rocm and rocmd changed what the standalone daemon's tools report when an engine child ends badly: an answer printed before a failing exit is now decoded, and a parse failure carries the child's stderr. Only service-stop-02 exercised the daemon against an engine, and only a clean one. Two scenarios in a new engine_ipc.feature copy the real rocmd into a scratch directory next to a stand-in rocm, which is the executable the standalone daemon runs for an engine request. The stand-in records its argv and the request it was sent and then answers and exits non-zero (engine-ipc-01), or prints something that is not an answer, writes to stderr and exits non-zero (engine-ipc-02). Each checks what the stand-in itself recorded, the message in the stop_server report, and the server's process, record and endpoint key together: for the server used here (vllm) the engine's Stop is advisory, so the stop completes in both and the report says why the request did or did not succeed. The steps live in service_stop_steps.rs with the planting helpers they reuse. The scenarios use their own feature key so they cannot collide with the numbering of service_stop.feature. Reverting the exit-status rule, dropping stderr from the parse failure, or making the standalone daemon run itself turns the scenarios red. docs/testing.md describes them. Signed-off-by: Jussi Elo --- docs/testing.md | 18 ++ .../e2e-cucumber/features/engine_ipc.feature | 47 +++++ .../tests/e2e/service_stop_steps.rs | 165 +++++++++++++++++- tests/e2e-cucumber/tests/feature_naming.rs | 1 + 4 files changed, 230 insertions(+), 1 deletion(-) create mode 100644 tests/e2e-cucumber/features/engine_ipc.feature diff --git a/docs/testing.md b/docs/testing.md index 94feb26b1f..775663d186 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1849,6 +1849,24 @@ a hooks build, as `cargo xtask e2e` makes: `cargo build -p rocm -p rocmd --features rocm/e2e-test-hooks`, then the command above with `-n "service-stop"`. +`features/engine_ipc.feature` (`engine-ipc-01` and `-02`, Linux, mock lane) +covers how the daemon reads an engine child's answer, through the same +`stop_server` tool. Each scenario copies the real `rocmd` into a scratch +directory and puts a stand-in `rocm` shell script beside it, which is the +executable the standalone daemon runs for an engine request; the stand-in +records its argv and the request it was sent, and prints what the scenario +says. `engine-ipc-01` has the engine answer the stop and then exit non-zero: the +report carries the engine's answer (not an error built from the exit status). +`engine-ipc-02` has it print something that is not an answer, write to stderr +and exit non-zero: the report's engine error is the parse failure with that +stderr. Both check the report together with the server's process, record and +endpoint key, and that the stand-in was run once as `__engine-stdio vllm` with +one `stop` request line. The server is a `vllm` one, for which the engine's +`Stop` is advisory, so in `-02` the stop still completes; an engine whose stop +decides the confirmation (`aim`) is not staged here, because teardown would +then run the real engine's stop. Run them with a hooks build and `-n +"engine-ipc"`. + Manual restricted-tool smoke: ```bash diff --git a/tests/e2e-cucumber/features/engine_ipc.feature b/tests/e2e-cucumber/features/engine_ipc.feature new file mode 100644 index 0000000000..a061425d42 --- /dev/null +++ b/tests/e2e-cucumber/features/engine_ipc.feature @@ -0,0 +1,47 @@ +Feature: How the daemon reads an engine's answer + + # `rocmd` asks an engine to do things (here: stop a server) by running + # `rocm __engine-stdio ` and reading what the child prints. The + # standalone `rocmd` binary has no such verb of its own and runs the `rocm` + # beside it, which is what makes these scenarios stageable from outside: each + # one copies the real `rocmd` into a scratch directory and puts a stand-in + # `rocm` next to it, so the daemon's own tool reaches a child whose answer the + # scenario controls. The stand-in records the argv and the request it was + # given and prints what the scenario says. + # + # service-stop-02 covers the engine that answers cleanly; the two below are the + # other ways an engine child ends. In each, the engine's message in the tool's + # report is checked together with what the stop did to the server: its process, + # its record and its endpoint key. The server is a `vllm` one, for which the + # engine's `Stop` is advisory (the recorded process decides whether the stop + # is confirmed), so the stop going through is part of what is shown, not a + # side effect. + # + # Linux-only, for the same reason as service-stop-02: the recorded process + # carries the start-time token `/proc` gives it, and the stand-in is a shell + # script. No GPU and no real engine are involved, so these run on the mock lane. + + @id:engine-ipc-reads-an-answer-printed-before-a-failing-exit + @requires-os:linux + Scenario: engine-ipc-01 - An engine that answers the stop and then exits non-zero is read as having answered it + Given a managed server whose recorded process is running + And the engine answers the stop request and then exits with a failing status + When the standalone daemon's stop_server tool stops that server + Then the engine received one stop request for that server over stdio + And the tool reports the engine's answer to the stop, not the status it exited with + And the recorded process is no longer running + And the server's record reads stopped and names no process + And the server's endpoint key file is gone + + @id:engine-ipc-reports-an-unreadable-answer-with-the-engines-stderr + @requires-os:linux + Scenario: engine-ipc-02 - An engine that prints something unreadable and fails reports its stderr, and the stop still completes + Given a managed server whose recorded process is running + And the engine prints something that is not an answer, complains on stderr and exits with a failing status + When the standalone daemon's stop_server tool stops that server + Then the engine received one stop request for that server over stdio + And the tool reports the stop request failed, with the engine's stderr + And the tool still reports the server stopped + And the recorded process is no longer running + And the server's record reads stopped and names no process + And the server's endpoint key file is gone diff --git a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs index 405fdcc557..72519539b3 100644 --- a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs @@ -222,6 +222,72 @@ async fn next_stop_unverified(world: &mut E2eWorld) { script_next_stop_outcome(world, "unverified"); } +// ── Given: an engine child the scenario controls ──────────────────── + +/// The standalone daemon's file name: `rocmd`, with the platform's suffix. +const STANDALONE_ROCMD: &str = if cfg!(windows) { "rocmd.exe" } else { "rocmd" }; +/// What the stand-in engine launcher records, one line per time it is run. +const ENGINE_INVOCATIONS: &str = "engine-invocations"; +/// What the stand-in engine launcher read on its stdin, over every run. +const ENGINE_REQUESTS: &str = "engine-requests"; + +/// The directory holding the standalone daemon copy and the stand-in `rocm`. +fn standalone_dir(world: &E2eWorld) -> PathBuf { + world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .join("standalone-daemon") +} + +/// Plant a copy of the real `rocmd` with a stand-in `rocm` beside it. +/// +/// A `rocmd` named exactly that reaches an engine through the `rocm` next to it, +/// so the engine child the daemon runs for its `Stop` request is `behaviour`: a +/// shell script that first records its argv and what it read on stdin (into +/// [`ENGINE_INVOCATIONS`] and [`ENGINE_REQUESTS`] beside it), then does what the +/// scenario says. The copy is a plain file, not a link, because the daemon looks +/// for its sibling next to where its executable really is. +fn install_standalone_daemon(world: &E2eWorld, behaviour: &str) { + let dir = standalone_dir(world); + std::fs::create_dir_all(&dir).expect("failed to create the standalone daemon dir"); + let rocmd = std::fs::read(crate::rocmd_binary()).expect("failed to read rocmd"); + rocm_core::fake_executable::write_fake_executable(&dir.join(STANDALONE_ROCMD), rocmd) + .expect("failed to plant the standalone daemon"); + let script = format!( + "#!/bin/sh\n\ + dir=$(dirname \"$0\")\n\ + echo \"$*\" >> \"$dir/{ENGINE_INVOCATIONS}\"\n\ + cat >> \"$dir/{ENGINE_REQUESTS}\"\n\ + {behaviour}\n" + ); + rocm_core::fake_executable::write_fake_executable(&dir.join("rocm"), script) + .expect("failed to plant the stand-in rocm"); +} + +#[given("the engine answers the stop request and then exits with a failing status")] +async fn engine_answers_then_fails(world: &mut E2eWorld) { + install_standalone_daemon( + world, + "echo '{\"ok\":true,\"data\":{\"stopped\":true,\"graceful\":true}}'\n\ + echo 'stand-in engine: exiting with a failing status after answering' >&2\n\ + exit 3", + ); +} + +#[given( + "the engine prints something that is not an answer, complains on stderr and exits with a failing status" +)] +async fn engine_prints_garbage_and_fails(world: &mut E2eWorld) { + install_standalone_daemon( + world, + "printf 'this is not an answer'\n\ + echo 'engine exploded: no container runtime' >&2\n\ + exit 1", + ); +} + // ── When ─────────────────────────────────────────────────────────── /// Runs the pinned, directly-runnable form `serve_summary.rs`'s `stop` row @@ -239,7 +305,25 @@ async fn stop_with_yes(world: &mut E2eWorld) { #[when("the daemon's stop_server tool stops that server")] async fn stop_server_tool(world: &mut E2eWorld) { - let mut command = Command::new(crate::rocmd_binary()); + run_stop_server_tool(world, crate::rocmd_binary()); +} + +/// The same tool, run from the copy of `rocmd` that has a stand-in `rocm` beside +/// it (see [`install_standalone_daemon`]), so the engine's `Stop` request reaches +/// a child the scenario controls. +#[when("the standalone daemon's stop_server tool stops that server")] +async fn standalone_stop_server_tool(world: &mut E2eWorld) { + let rocmd = standalone_dir(world).join(STANDALONE_ROCMD); + assert!( + rocmd.is_file(), + "no standalone daemon was installed: {}", + rocmd.display() + ); + run_stop_server_tool(world, rocmd); +} + +fn run_stop_server_tool(world: &mut E2eWorld, rocmd: PathBuf) { + let mut command = Command::new(rocmd); command.args(["sandbox-tool", "stop_server", "--service-id", SERVICE_ID]); world.isolate_cmd(&mut command); let output = command.output().expect("failed to run rocmd"); @@ -329,6 +413,85 @@ async fn engine_answered_the_stop(world: &mut E2eWorld) { assert!(engine_stop["stopped"].is_boolean(), "{report}"); } +/// The stop report `rocmd sandbox-tool stop_server` printed, with the tool's own +/// exit status checked first: both engine endings below leave the stop itself +/// succeeding. +fn tool_report(world: &E2eWorld) -> serde_json::Value { + let stdout = world.cli_output.as_deref().unwrap_or_default(); + assert_eq!( + world.cli_rc, + Some(0), + "rocmd failed:\n{stdout}\n{}", + world.cli_stderr.as_deref().unwrap_or_default() + ); + serde_json::from_str(stdout).unwrap_or_else(|error| panic!("{error}: {stdout}")) +} + +/// The stand-in engine was run once, as `__engine-stdio vllm`, and was sent one +/// line: an engine `stop` request for the planted server. Reads what the +/// stand-in itself recorded, so it holds for the child the daemon ran, not for +/// what the daemon says it ran. +#[then("the engine received one stop request for that server over stdio")] +async fn engine_received_one_stop_request(world: &mut E2eWorld) { + let dir = standalone_dir(world); + let invocations = std::fs::read_to_string(dir.join(ENGINE_INVOCATIONS)) + .expect("the stand-in engine was never run"); + assert_eq!( + invocations, + format!("__engine-stdio {ENGINE}\n"), + "the daemon must run the rocm beside it once, as `__engine-stdio `" + ); + let requests = std::fs::read_to_string(dir.join(ENGINE_REQUESTS)) + .expect("the stand-in engine recorded no request"); + assert_eq!( + requests.lines().count(), + 1, + "one request, one line: {requests:?}" + ); + let request: serde_json::Value = serde_json::from_str(&requests) + .unwrap_or_else(|error| panic!("the request is not JSON ({error}): {requests:?}")); + assert_eq!(request["method"], "stop", "{request}"); + assert_eq!(request["payload"]["service_id"], SERVICE_ID, "{request}"); +} + +/// What the engine printed before it exited non-zero is what the report carries: +/// a stop it answered, with no error taken from the exit status. +#[then("the tool reports the engine's answer to the stop, not the status it exited with")] +async fn engine_answer_survives_a_failing_exit(world: &mut E2eWorld) { + let report = tool_report(world); + assert_eq!( + report["result"]["engine_stop"], + serde_json::json!({"attempted": true, "stopped": true, "graceful": true}), + "{report}" + ); + assert_eq!(report["status"], "stopped", "{report}"); + assert_eq!(report["result"]["stopped"], true, "{report}"); +} + +/// An answer that cannot be read is reported as that, with what the engine said +/// on stderr, rather than as an exit status. +#[then("the tool reports the stop request failed, with the engine's stderr")] +async fn unreadable_answer_reported_with_stderr(world: &mut E2eWorld) { + let report = tool_report(world); + let engine_stop = &report["result"]["engine_stop"]; + assert_eq!(engine_stop["attempted"], true, "{report}"); + assert_eq!( + engine_stop["error"], + "failed to parse engine response envelope; stderr: engine exploded: no container runtime", + "{report}" + ); + assert!(engine_stop.get("stopped").is_none(), "{report}"); +} + +/// For a `vllm` server the engine's `Stop` is advisory: the recorded process +/// decides, so a failed request does not leave the server unconfirmed. +#[then("the tool still reports the server stopped")] +async fn tool_still_reports_stopped(world: &mut E2eWorld) { + let report = tool_report(world); + assert_eq!(report["status"], "stopped", "{report}"); + assert_eq!(report["result"]["stopped"], true, "{report}"); +} + #[then("the recorded process is no longer running")] async fn recorded_process_gone(world: &mut E2eWorld) { let process = world diff --git a/tests/e2e-cucumber/tests/feature_naming.rs b/tests/e2e-cucumber/tests/feature_naming.rs index 230f791c8b..405ec0384c 100644 --- a/tests/e2e-cucumber/tests/feature_naming.rs +++ b/tests/e2e-cucumber/tests/feature_naming.rs @@ -33,6 +33,7 @@ const FEATURE_KEYS: &[(&str, &str)] = &[ ("diagnose.feature", "diagnose"), ("download_progress_pty.feature", "download-progress"), ("driver_install.feature", "driver-install"), + ("engine_ipc.feature", "engine-ipc"), ("engine_shell.feature", "engine-shell"), ("examine.feature", "examine"), ("install_lifecycle.feature", "lifecycle"), From b648ab7b773d9a5202f21bf601d69d32ef581822 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 12:59:10 +0000 Subject: [PATCH 09/38] fix(rocmd): recognise the standalone daemon after its executable was replaced On Linux, current_exe() of a process whose executable file has since been replaced or removed ends in " (deleted)", which is the usual state of a long-lived daemon right after an in-place package upgrade. The standalone check compared the file stem against "rocmd", so "rocmd (deleted)" no longer matched, the daemon ran that name as the engine launcher, and every engine request failed to spawn with an error that had nothing to do with the cause. The marker is now stripped from the file name first, by a pure helper, so the daemon still finds the rocm beside it. The same strip applies to the path an embedded rocm daemon runs itself by, which would otherwise name a file that does not exist. A standalone rocmd shipped under another name is still not recognised: it runs itself, and answers the engine request with "unrecognized subcommand". The comment says so. The test runs on every platform with injected paths and fails when the strip is removed. Signed-off-by: Jussi Elo --- apps/rocmd/src/engine_ipc.rs | 64 +++++++++++++++++++++++++++++++++++- 1 file changed, 63 insertions(+), 1 deletion(-) diff --git a/apps/rocmd/src/engine_ipc.rs b/apps/rocmd/src/engine_ipc.rs index d0fcfa29ed..ff69c4cd63 100644 --- a/apps/rocmd/src/engine_ipc.rs +++ b/apps/rocmd/src/engine_ipc.rs @@ -148,8 +148,11 @@ fn rocm_executable_from( current: PathBuf, sibling_rocm: impl FnOnce() -> Result, ) -> Result { + let current = without_deleted_marker(current); // Exactly `rocmd` (or `rocmd.exe`). A renamed or test binary is not known to - // lack the verb, and has no `rocm` beside it to prefer. + // lack the verb, and has no `rocm` beside it to prefer: it runs itself, so a + // standalone `rocmd` shipped under another name still answers an engine + // request with "unrecognized subcommand". if current.file_stem().is_some_and(|stem| stem == "rocmd") { return sibling_rocm().context( "the standalone rocmd needs the rocm executable beside it to reach an engine", @@ -158,6 +161,28 @@ fn rocm_executable_from( Ok(current) } +/// What Linux appends to the name `current_exe()` reports when the running +/// executable's file has been replaced or removed since the process started, +/// which is what an in-place upgrade of a long-lived daemon does. +const DELETED_MARKER: &str = " (deleted)"; + +/// `path` without the [`DELETED_MARKER`] on its file name. The marker is not +/// part of any file's name, and left in place it hides the `rocmd` the name check +/// looks for and names a file that cannot be run; the path without it is where +/// the upgraded executable now is. +fn without_deleted_marker(path: PathBuf) -> PathBuf { + let stripped = path + .file_name() + .and_then(|name| name.to_str()) + .and_then(|name| name.strip_suffix(DELETED_MARKER)) + .filter(|name| !name.is_empty()) + .map(str::to_owned); + match stripped { + Some(name) => path.with_file_name(name), + None => path, + } +} + /// [`engine_request`] with the executable that serves `__engine-stdio` named by /// the caller, so a test can point it at a stand-in. fn engine_request_via( @@ -418,6 +443,43 @@ mod tests { ); } + /// After an in-place upgrade Linux reports the running executable as + /// ` (deleted)`. The daemon is long-lived, so that is the usual state + /// of a standalone `rocmd` right after its package is upgraded: it must + /// still be recognised, and an embedded `rocm` must be run by a path that + /// exists. + #[test] + fn an_executable_replaced_while_running_is_still_told_apart_by_its_name() { + let beside = || Ok(PathBuf::from("/usr/bin/rocm")); + + let chosen = + rocm_executable_from(PathBuf::from("/usr/bin/rocmd (deleted)"), beside).unwrap(); + assert_eq!(chosen, Path::new("/usr/bin/rocm")); + + // Embedded `rocm daemon`: run the file now at that path, not the + // marker-suffixed name, which no file has. + let chosen = rocm_executable_from(PathBuf::from("/usr/bin/rocm (deleted)"), || { + panic!("an embedded rocm has no use for a sibling") + }) + .unwrap(); + assert_eq!(chosen, Path::new("/usr/bin/rocm")); + + // The marker is only ever a suffix of the file name, and only that + // exact one; nothing else is touched. + for untouched in [ + "/opt/(deleted)/rocm", + "/opt/rocm-cli/bin/rocm (deleted) extra", + "/opt/rocm-cli/bin/ (deleted)", + "/opt/rocm-cli/bin/rocm", + ] { + assert_eq!( + without_deleted_marker(PathBuf::from(untouched)), + Path::new(untouched), + "{untouched}" + ); + } + } + #[test] fn rocmd_runs_the_child_on_its_own_paths_keeps_stderr_and_asks_for_no_progress() { let (_root, paths) = temp_app_paths("stdio-options"); From 3e292d6dd74991b86ede6cfc9ac49ab72a9bfa85 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 13:02:31 +0000 Subject: [PATCH 10/38] fix(engine-protocol): name the exit status when a failing child's answer cannot be parsed A child that exited with a failing status after printing something that is not a response envelope was reported as "failed to parse engine response envelope" with its stderr, which dropped the status that the old rocmd always reported for a failing child. The status is often why the answer was cut short, so the parse failure now carries it, worded as the failing-exit error already is: "failed to parse engine response envelope (engine stdio process exited with status ...); stderr: ...". A child that exited cleanly adds nothing, and a failing child that printed nothing is still reported by its status alone. The wording of the exit note comes from one function, so the two messages cannot drift. The unit test pins all three cases against the fake engine. The engine-ipc-02 scenario now expects the new text in the stop_server report, still together with what the stop did to the process, record and key, and docs/testing.md describes it. Signed-off-by: Jussi Elo --- .../rocm-engine-protocol/src/stdio_client.rs | 77 ++++++++++++++++--- docs/testing.md | 12 ++- .../e2e-cucumber/features/engine_ipc.feature | 2 +- .../tests/e2e/service_stop_steps.rs | 11 ++- 4 files changed, 83 insertions(+), 19 deletions(-) diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index ed7c9c1fd4..2a269696db 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -22,7 +22,7 @@ use std::ffi::OsStr; use std::io::{self, BufRead, Read, Write}; use std::path::Path; -use std::process::{Child, Command, Stdio}; +use std::process::{Child, Command, ExitStatus, Stdio}; use std::thread; use anyhow::{Context, Result, bail}; @@ -166,21 +166,34 @@ where .map(|reader| reader.join().unwrap_or_default()) .unwrap_or_default(); + let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); if !status.success() && stdout.is_empty() { - let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); + let exited = exit_note(status); if stderr.is_empty() { - bail!("engine stdio process exited with status {status}"); + bail!("{exited}"); } - bail!("engine stdio process exited with status {status}: {stderr}"); + bail!("{exited}: {stderr}"); } - serde_json::from_slice(&stdout).with_context(|| { - let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); - if stderr.is_empty() { - "failed to parse engine response envelope".to_owned() - } else { - format!("failed to parse engine response envelope; stderr: {stderr}") - } - }) + serde_json::from_slice(&stdout).with_context(|| unparseable_answer_message(status, &stderr)) +} + +/// How a failing exit is worded wherever the status is reported. +fn exit_note(status: ExitStatus) -> String { + format!("engine stdio process exited with status {status}") +} + +/// The error for an answer that is not a response envelope. A child that also +/// exited with a failing status says so, because that is often why the answer is +/// cut short; the stderr it wrote follows when there is any. +fn unparseable_answer_message(status: ExitStatus, stderr: &str) -> String { + let mut message = "failed to parse engine response envelope".to_owned(); + if !status.success() { + message.push_str(&format!(" ({})", exit_note(status))); + } + if !stderr.is_empty() { + message.push_str(&format!("; stderr: {stderr}")); + } + message } /// Read the child's whole answer from `stdout`. @@ -606,6 +619,46 @@ mod tests { ); } + /// A child that fails and also leaves something that is not an envelope on + /// stdout is reported with its exit status (the status says the answer was + /// cut short, where the parse error alone would not), as a child that + /// printed nothing is. The same unreadable answer from a child that exited + /// cleanly carries no status, so the note is the condition's own and not + /// added to every parse failure. + #[test] + fn an_unparseable_answer_from_a_failing_child_names_the_exit_status() { + let crashed = FakeEngine::new( + "client-garbage-crash", + true, + "printf 'not json'\necho oops >&2\nexit 4", + ); + let error = run(&crashed, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to parse engine response envelope \ + (engine stdio process exited with status exit status: 4); stderr: oops" + ); + + let silent = FakeEngine::new( + "client-garbage-crash-quiet", + true, + "printf 'not json'\nexit 5", + ); + let error = run(&silent, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to parse engine response envelope \ + (engine stdio process exited with status exit status: 5)" + ); + + let clean = FakeEngine::new("client-garbage-clean", true, "printf 'not json'\nexit 0"); + let error = run(&clean, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to parse engine response envelope" + ); + } + /// An answer whose read fails kills the child and reaps it, so a daemon /// that hits this repeatedly collects neither a running engine nor a /// zombie per failure. A zombie keeps its `/proc` entry until it is diff --git a/docs/testing.md b/docs/testing.md index 775663d186..5d787f01dc 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -221,6 +221,14 @@ choose: the program and verb, the app paths, and whether stderr is forwarded (se `apps/rocm/src/engine_ipc.rs` and `apps/rocmd/src/engine_ipc.rs`). A caller that grew its own private copy of the conversation would fail those tests. +An answer that is not a response envelope is reported as that, and when the +child also exited with a failing status the error says so, because the status is +often why the answer was cut short (a clean exit adds nothing). A failing child +that printed nothing is reported by its status alone. Both carry the child's +stderr. `an_unparseable_answer_from_a_failing_child_names_the_exit_status` pins +the three cases against the fake, and `engine-ipc-02` below shows the same +message in a `stop_server` report next to what the stop did. + The SIGPIPE tests re-execute the test binary in a fresh process with SIGPIPE at its default action, which is how `rocm` runs, because the test harness leaves it ignored and changing it process-wide would race sibling tests. A write that is @@ -1858,8 +1866,8 @@ records its argv and the request it was sent, and prints what the scenario says. `engine-ipc-01` has the engine answer the stop and then exit non-zero: the report carries the engine's answer (not an error built from the exit status). `engine-ipc-02` has it print something that is not an answer, write to stderr -and exit non-zero: the report's engine error is the parse failure with that -stderr. Both check the report together with the server's process, record and +and exit non-zero: the report's engine error is the parse failure, naming the +exit status the engine ended with, followed by its stderr. Both check the report together with the server's process, record and endpoint key, and that the stand-in was run once as `__engine-stdio vllm` with one `stop` request line. The server is a `vllm` one, for which the engine's `Stop` is advisory, so in `-02` the stop still completes; an engine whose stop diff --git a/tests/e2e-cucumber/features/engine_ipc.feature b/tests/e2e-cucumber/features/engine_ipc.feature index a061425d42..96eb773636 100644 --- a/tests/e2e-cucumber/features/engine_ipc.feature +++ b/tests/e2e-cucumber/features/engine_ipc.feature @@ -40,7 +40,7 @@ Feature: How the daemon reads an engine's answer And the engine prints something that is not an answer, complains on stderr and exits with a failing status When the standalone daemon's stop_server tool stops that server Then the engine received one stop request for that server over stdio - And the tool reports the stop request failed, with the engine's stderr + And the tool reports the stop request failed, with the engine's exit status and stderr And the tool still reports the server stopped And the recorded process is no longer running And the server's record reads stopped and names no process diff --git a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs index 72519539b3..59f512a25c 100644 --- a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs @@ -468,16 +468,19 @@ async fn engine_answer_survives_a_failing_exit(world: &mut E2eWorld) { assert_eq!(report["result"]["stopped"], true, "{report}"); } -/// An answer that cannot be read is reported as that, with what the engine said -/// on stderr, rather than as an exit status. -#[then("the tool reports the stop request failed, with the engine's stderr")] +/// An answer that cannot be read is reported as that, together with the status +/// the engine exited with and what it said on stderr: the status says the answer +/// was cut short, the stderr why. +#[then("the tool reports the stop request failed, with the engine's exit status and stderr")] async fn unreadable_answer_reported_with_stderr(world: &mut E2eWorld) { let report = tool_report(world); let engine_stop = &report["result"]["engine_stop"]; assert_eq!(engine_stop["attempted"], true, "{report}"); assert_eq!( engine_stop["error"], - "failed to parse engine response envelope; stderr: engine exploded: no container runtime", + "failed to parse engine response envelope \ + (engine stdio process exited with status exit status: 1); \ + stderr: engine exploded: no container runtime", "{report}" ); assert!(engine_stop.get("stopped").is_none(), "{report}"); From 87c0d2f3bff88f8584bfb6eaef7462536100a955 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 13:04:44 +0000 Subject: [PATCH 11/38] fix(engine-protocol): report what an engine said on stderr when its request cannot be written A request whose write failed (the engine exited, or closed its stdin, before reading it) killed and reaped the child but dropped its stderr, although the engine's own message is usually the reason it was not listening and the error was meant to let the caller report why. The child is still killed and waited for, and its stderr is now read and appended to the error: "failed to write engine request: ; stderr: ". The kill is not made conditional on try_wait showing an exit first, because the pipe closes while the process is still exiting, so a check right after the failed write can see it running and would discard the very text this adds. The read of the stderr is bounded by a short grace: a grandchild that inherited the pipe keeps it open past the exit, and this error path must not block on it; what had arrived by then is reported. Tests: the whole client against a fake that exits without reading (the stderr must be in the error), the step on a real child that is still running (killed and reaped, by its /proc entry; the test fails instead of hanging when the kill is removed), and on a child whose grandchild holds the pipe (returns within the grace, with what had been written). Removing the stderr read, the kill, or the bound each turns one of them red. Signed-off-by: Jussi Elo --- .../rocm-engine-protocol/src/stdio_client.rs | 196 +++++++++++++++++- docs/architecture.md | 2 +- docs/testing.md | 14 ++ 3 files changed, 203 insertions(+), 9 deletions(-) diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 2a269696db..104685b9b6 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -23,7 +23,9 @@ use std::ffi::OsStr; use std::io::{self, BufRead, Read, Write}; use std::path::Path; use std::process::{Child, Command, ExitStatus, Stdio}; +use std::sync::mpsc; use std::thread; +use std::time::{Duration, Instant}; use anyhow::{Context, Result, bail}; use rocm_core::{AppPaths, ENGINE_ENVS_ROOT_ENV, with_sigpipe_ignored}; @@ -136,16 +138,15 @@ where // SIGPIPE (`rocm` runs that way, and so does the daemon hosted in it); // ignored, the write returns EPIPE and fails as an ordinary error. let written = with_sigpipe_ignored(|| { - serde_json::to_writer(&mut stdin, envelope).context("failed to write engine request")?; - stdin.write_all(b"\n")?; - Ok::<(), anyhow::Error>(()) + serde_json::to_writer(&mut stdin, envelope).map_err(io::Error::from)?; + stdin.write_all(b"\n") }); if let Err(error) = written { - // The engine will never get its request, so do not leave it running or - // unreaped: a long-lived caller would collect one zombie per failure. - let _ = child.kill(); - let _ = child.wait(); - return Err(error); + return Err(abandon_unwritten_request( + child, + &error, + EXITED_STDERR_GRACE, + )); } // End of the request: the engine reads to EOF before it answers. drop(stdin); @@ -196,6 +197,69 @@ fn unparseable_answer_message(status: ExitStatus, stderr: &str) -> String { message } +/// How long the stderr of a child that was killed, or had exited, is waited for +/// when its request could not be written. +const EXITED_STDERR_GRACE: Duration = Duration::from_secs(1); + +/// The request could not be written to `child`'s stdin, so it will never answer: +/// do not leave it running or unreaped (a long-lived caller would collect one +/// zombie per failure), and report why it was not listening. +/// +/// The child is killed (nothing to do for one that has already exited) and +/// waited for, then whatever it wrote to stderr is added to the error: an engine +/// that exits at once usually says why there, and that is the explanation the +/// caller wants. The kill does not wait on `try_wait` showing an exit first, +/// because the pipe closes while the process is still exiting, so a check made +/// right after the failed write can still see it running. The read is bounded +/// by `grace`: a grandchild that inherited the pipe keeps it open past the exit, +/// and this error path must not block on it. +fn abandon_unwritten_request( + mut child: Child, + write_error: &io::Error, + grace: Duration, +) -> anyhow::Error { + let _ = child.kill(); + let _ = child.wait(); + let stderr = child + .stderr + .take() + .map(|stderr| read_available(stderr, grace)) + .unwrap_or_default(); + let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); + let mut message = format!("failed to write engine request: {write_error}"); + if !stderr.is_empty() { + message.push_str(&format!("; stderr: {stderr}")); + } + anyhow::anyhow!(message) +} + +/// Read `source` to its end, or for at most `grace` if it does not end, and +/// return what arrived. The reading thread is left to finish by itself when the +/// pipe closes. +fn read_available(mut source: impl Read + Send + 'static, grace: Duration) -> Vec { + let (sender, receiver) = mpsc::channel::>(); + thread::spawn(move || { + let mut buffer = [0_u8; 4096]; + loop { + match source.read(&mut buffer) { + Ok(0) | Err(_) => break, + Ok(n) => { + if sender.send(buffer[..n].to_vec()).is_err() { + break; + } + } + } + } + }); + let deadline = Instant::now() + grace; + let mut collected = Vec::new(); + while let Ok(chunk) = receiver.recv_timeout(deadline.saturating_duration_since(Instant::now())) + { + collected.extend(chunk); + } + collected +} + /// Read the child's whole answer from `stdout`. /// /// A failed read would leave the child running with nobody waiting on it, and @@ -659,6 +723,122 @@ mod tests { ); } + /// Runs `work` on its own thread and fails the test if it has not finished + /// within `limit`, so a step that should not block turns red instead of + /// hanging the run. + fn finishes_within( + limit: Duration, + work: impl FnOnce() -> T + Send + 'static, + ) -> T { + let (sender, receiver) = mpsc::channel(); + thread::spawn(move || { + let _ = sender.send(work()); + }); + receiver + .recv_timeout(limit) + .unwrap_or_else(|_| panic!("did not finish within {limit:?}")) + } + + /// A request that cannot be written because the engine is gone reports + /// what the engine said on stderr, not only that the write failed: the + /// engine's own words are the explanation. The request is larger than a + /// pipe buffer, so the write reaches the closed read end. + #[test] + fn a_request_to_an_engine_that_exited_reports_what_it_said_on_stderr() { + let engine = FakeEngine::new("client-no-listen", false, "echo 'no thanks' >&2\nexit 3"); + let envelope = + engine_request_envelope(EngineMethod::Stop, &json!({"blob": "x".repeat(1 << 20)})) + .unwrap(); + let error = run_engine_stdio( + engine.path(), + ["stdio"], + &EngineStdioOptions::default(), + &envelope, + ) + .expect_err("the engine never read its request"); + let message = error.to_string(); + assert!( + message.starts_with("failed to write engine request: "), + "{message}" + ); + assert!(message.ends_with("; stderr: no thanks"), "{message}"); + } + + /// The child of a failed write is killed and reaped when it is still + /// running, which a failed write does not show it to have stopped. + /// Linux-only for the `/proc` entry, which a zombie keeps until waited on. + #[cfg(target_os = "linux")] + #[test] + fn a_request_that_cannot_be_written_kills_and_reaps_a_child_that_is_still_running() { + let child = Command::new("sleep") + .arg("60") + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let proc_entry = format!("/proc/{}", child.id()); + assert!(Path::new(&proc_entry).exists(), "the child is running"); + + let error = finishes_within(Duration::from_secs(10), move || { + abandon_unwritten_request( + child, + &io::Error::from(io::ErrorKind::BrokenPipe), + Duration::from_secs(1), + ) + }); + + assert!( + error + .to_string() + .starts_with("failed to write engine request: "), + "{error}" + ); + assert!( + !error.to_string().contains("stderr"), + "a child that said nothing adds no stderr: {error}" + ); + assert!( + !Path::new(&proc_entry).exists(), + "the child must be killed and reaped, not left running or as a zombie" + ); + } + + /// A grandchild that holds the child's stderr open past its exit cannot + /// make the error path wait for it: what had been written is reported + /// and the rest is not waited for. + #[test] + fn the_stderr_of_a_failed_write_is_not_waited_for_past_the_grace() { + let mut child = Command::new("sh") + .args(["-c", "echo said >&2; sleep 5 & exit 3"]) + .stdin(Stdio::piped()) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + // The engine has said its piece and exited before the failure is + // handled, as it has in the case this bounds. + while child.try_wait().unwrap().is_none() { + thread::sleep(Duration::from_millis(5)); + } + let started = Instant::now(); + + let error = finishes_within(Duration::from_secs(4), move || { + abandon_unwritten_request( + child, + &io::Error::from(io::ErrorKind::BrokenPipe), + Duration::from_millis(300), + ) + }); + + assert!( + started.elapsed() < Duration::from_secs(4), + "waited for the grandchild: {:?}", + started.elapsed() + ); + assert!(error.to_string().ends_with("; stderr: said"), "{error}"); + } + /// An answer whose read fails kills the child and reaps it, so a daemon /// that hits this repeatedly collects neither a running engine nor a /// zombie per failure. A zombie keeps its `/proc` entry until it is diff --git a/docs/architecture.md b/docs/architecture.md index 033836b568..75ead91d02 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -305,7 +305,7 @@ The other three dashboard crates are described at crate level here, not module b A contract surface: verify all impacted engines after any protocol change here (see [`AGENTS.md`](../AGENTS.md)). -- [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) — the client side of the engine stdio protocol, the one implementation `rocm` and `rocmd` both run an engine child through: the request line, the child's environment (`EngineStdioOptions`: app paths, engine environments root, endpoint key file, progress request), stderr handling (kept for the error, optionally forwarded as it arrives), the SIGPIPE guard around the write and around each forward of the child's stderr, killing and reaping the child when its request cannot be written or its answer cannot be read, what a failing exit status or an unusable answer is reported as, and `decode_engine_response`. Each caller still chooses which program to run: `rocm` the resolved engine binary with `stdio` (or the built-in handler, with no child), `rocmd` the `rocm` executable with `__engine-stdio `. It adds no wire format; the envelope types are the ones in [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs). +- [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) — the client side of the engine stdio protocol, the one implementation `rocm` and `rocmd` both run an engine child through: the request line, the child's environment (`EngineStdioOptions`: app paths, engine environments root, endpoint key file, progress request), stderr handling (kept for the error, optionally forwarded as it arrives), the SIGPIPE guard around the write and around each forward of the child's stderr, killing and reaping the child when its request cannot be written (the error then carries what it wrote to stderr) or its answer cannot be read, what a failing exit status or an unusable answer is reported as, and `decode_engine_response`. Each caller still chooses which program to run: `rocm` the resolved engine binary with `stdio` (or the built-in handler, with no child), `rocmd` the `rocm` executable with `__engine-stdio `. It adds no wire format; the envelope types are the ones in [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs). - [`fake_engine.rs`](../crates/rocm-engine-protocol/src/fake_engine.rs) — test-only, behind the `test-support` feature the `rocm` and `rocmd` dev-dependencies turn on: a shell-script stand-in for an engine child that records its argv, environment and stdin and answers (or crashes) as a test asks, so both callers' tests run against the same fake. - [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs) — the whole crate: the protocol and recipe-contract version constants, the request/response envelopes and the per-method request and response types for the `EngineMethod` calls (`detect`, `install`, `capabilities`, `resolve_model`, `launch`, `healthcheck`, `endpoint`, `stop`, `logs`), the `DevicePolicy` and `GpuSelection` types with the GPU-visibility helpers engines apply to a child process, engine-plugin binary naming and discovery, and the endpoint API key file carrier (`ENDPOINT_API_KEY_FILE_ENV`, `resolve_endpoint_api_key`, `is_public_bind_host`) that `rocm` and `rocmd` share, and `optional_arg`, the `flag value` argument helper both binaries build child command lines with. diff --git a/docs/testing.md b/docs/testing.md index 5d787f01dc..896abe71b8 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -249,6 +249,20 @@ read step on a real child with a reader that fails and checks that the child's `/proc` entry is gone: a killed child that was never waited on would keep it as a zombie. +A request that cannot be written (the engine exited, or closed its stdin, before +reading it) is reported with what the engine wrote to stderr, since that is +usually the explanation, after the child has been killed and reaped. The child is +killed whether or not a `try_wait` would show it gone, because its pipe closes +while it is still exiting; the read of its stderr is bounded (a grandchild that +holds the pipe cannot hold the error path). `a_request_to_an_engine_that_exited_reports_what_it_said_on_stderr` +runs the whole client against a fake that exits without reading; +`a_request_that_cannot_be_written_kills_and_reaps_a_child_that_is_still_running` +(Linux only) checks the `/proc` entry of one that is still running, and +`the_stderr_of_a_failed_write_is_not_waited_for_past_the_grace` one whose +grandchild keeps the pipe open. The last two run the step on a real child with an +injected write error, because whether a real write fails before the child has +closed its stdin is a race. + ### Advised commands must parse `apps/rocm/src/advised_commands.rs` checks the `rocm`/`rocmd` commands that the From ace9d9fa9af81e91cc60eb9a42e945b6c0815a3d Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 13:05:37 +0000 Subject: [PATCH 12/38] docs(engine-protocol): state that waiting for a child's pipes after it exits is unbounded After the child exits, the stdio client reads its stdout to the end and joins its stderr collector, so an engine that leaves a grandchild holding either pipe makes the request wait for that grandchild. The clients this one replaced had the same exposure (rocm read stdout to the end and then joined its stderr thread; rocmd used wait_with_output), so this is not a regression, but the client now lives in the long-lived daemon. Bounding it exactly would mean reading both pipes on their own threads, waiting for the child itself without a limit and for the pipes only up to a deadline after it exits, and deciding what to report when the deadline passes (a complete answer, a partial one, an error). That is a timeout policy for an engine contract that has none, and it reorders the kill-on-failed-read handling, so it is not folded into this change. The module comment, docs/testing.md and the architecture entry now say precisely what is and is not bounded. The one wait this client adds itself, the stderr of a child whose request could not be written, is bounded. Signed-off-by: Jussi Elo --- crates/rocm-engine-protocol/src/stdio_client.rs | 16 ++++++++++++++++ docs/architecture.md | 2 +- docs/testing.md | 8 ++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 104685b9b6..2126d6d303 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -18,6 +18,22 @@ //! //! The request and the child's lifetime are not bounded by a timeout: an engine //! install legitimately runs for minutes, and no caller has ever set one. +//! +//! Nor is the wait for the pipes after the child has exited, and that is a known +//! limit rather than an oversight. The answer is read to the end of the child's +//! stdout, and the stderr collector is joined, so an engine that leaves a +//! grandchild holding either pipe open makes the request wait for that grandchild +//! to exit. Both clients this one replaces behaved the same way (`rocm` read +//! stdout to the end then joined its stderr thread; `rocmd` used +//! `wait_with_output`), so nothing new is exposed to it, but the daemon is long +//! lived and a hung request holds its caller. Bounding it exactly would mean +//! reading both pipes on their own threads, waiting for the child itself without a +//! limit and for the pipes only up to a deadline after it exits, and choosing what +//! to report when the deadline passes (an answer already complete? a partial +//! one?). That is a timeout policy for an engine contract nothing here sets, so +//! this change leaves it to be decided on its own. The one wait this client adds +//! itself, for the stderr of a child whose request could not be written, is +//! bounded by `EXITED_STDERR_GRACE`. use std::ffi::OsStr; use std::io::{self, BufRead, Read, Write}; diff --git a/docs/architecture.md b/docs/architecture.md index 75ead91d02..6ff1d4d412 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -305,7 +305,7 @@ The other three dashboard crates are described at crate level here, not module b A contract surface: verify all impacted engines after any protocol change here (see [`AGENTS.md`](../AGENTS.md)). -- [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) — the client side of the engine stdio protocol, the one implementation `rocm` and `rocmd` both run an engine child through: the request line, the child's environment (`EngineStdioOptions`: app paths, engine environments root, endpoint key file, progress request), stderr handling (kept for the error, optionally forwarded as it arrives), the SIGPIPE guard around the write and around each forward of the child's stderr, killing and reaping the child when its request cannot be written (the error then carries what it wrote to stderr) or its answer cannot be read, what a failing exit status or an unusable answer is reported as, and `decode_engine_response`. Each caller still chooses which program to run: `rocm` the resolved engine binary with `stdio` (or the built-in handler, with no child), `rocmd` the `rocm` executable with `__engine-stdio `. It adds no wire format; the envelope types are the ones in [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs). +- [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) — the client side of the engine stdio protocol, the one implementation `rocm` and `rocmd` both run an engine child through: the request line, the child's environment (`EngineStdioOptions`: app paths, engine environments root, endpoint key file, progress request), stderr handling (kept for the error, optionally forwarded as it arrives), the SIGPIPE guard around the write and around each forward of the child's stderr, killing and reaping the child when its request cannot be written (the error then carries what it wrote to stderr) or its answer cannot be read, what a failing exit status or an unusable answer is reported as, and `decode_engine_response`. It sets no timeout, and does not bound the wait for the child's pipes after it exits (a grandchild holding one makes the request wait); its module comment states that limit. Each caller still chooses which program to run: `rocm` the resolved engine binary with `stdio` (or the built-in handler, with no child), `rocmd` the `rocm` executable with `__engine-stdio `. It adds no wire format; the envelope types are the ones in [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs). - [`fake_engine.rs`](../crates/rocm-engine-protocol/src/fake_engine.rs) — test-only, behind the `test-support` feature the `rocm` and `rocmd` dev-dependencies turn on: a shell-script stand-in for an engine child that records its argv, environment and stdin and answers (or crashes) as a test asks, so both callers' tests run against the same fake. - [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs) — the whole crate: the protocol and recipe-contract version constants, the request/response envelopes and the per-method request and response types for the `EngineMethod` calls (`detect`, `install`, `capabilities`, `resolve_model`, `launch`, `healthcheck`, `endpoint`, `stop`, `logs`), the `DevicePolicy` and `GpuSelection` types with the GPU-visibility helpers engines apply to a child process, engine-plugin binary naming and discovery, and the endpoint API key file carrier (`ENDPOINT_API_KEY_FILE_ENV`, `resolve_endpoint_api_key`, `is_public_bind_host`) that `rocm` and `rocmd` share, and `optional_arg`, the `flag value` argument helper both binaries build child command lines with. diff --git a/docs/testing.md b/docs/testing.md index 896abe71b8..cd4eaf7d69 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -263,6 +263,14 @@ grandchild keeps the pipe open. The last two run the step on a real child with a injected write error, because whether a real write fails before the child has closed its stdin is a race. +One wait is deliberately not bounded and has no test: after the child exits, the +answer is read to the end of its stdout and its stderr collector is joined, so an +engine that leaves a grandchild holding either pipe makes the request wait for +it. The clients this one replaced had the same exposure, and bounding it would +mean reading both pipes on their own threads and choosing what to report when a +deadline passes, which is a timeout policy the engine contract does not have +today. The module comment in `stdio_client.rs` states the limit and its tradeoff. + ### Advised commands must parse `apps/rocm/src/advised_commands.rs` checks the `rocm`/`rocmd` commands that the From 9d46e16a1a1f523d9f7b17fa1a01d5da37dffa26 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 13:06:53 +0000 Subject: [PATCH 13/38] test(engine-protocol): pin that stderr forwarding guards each write and not the reads between them Forwarding the child's stderr guards each write against SIGPIPE separately, which flips the process-wide disposition twice per forwarded line. Guarding once around the whole forwarding loop would remove that, but the loop spends nearly all its time blocked in reads (an engine install is quiet for long stretches), so the disposition would stay ignored for the whole run: every other thread's write to a closed stdout would then return an error instead of ending rocm quietly, which is what the default action is for. Guarding only after a failed write is not possible, since under the default action a write to a closed pipe ends the process before it can fail. The cost of the per-line guard, two signal calls beside a write, is negligible. So the code stays as it is and a test pins the scope: a fresh process with the default action runs the collector over a reader and a sink that record the disposition seen inside every read and every write, and the test requires ignored inside each write and the default action inside each read and after the loop. Wrapping the whole collector in the guard turns it red. Signed-off-by: Jussi Elo --- .../rocm-engine-protocol/src/stdio_client.rs | 96 +++++++++++++++++++ docs/testing.md | 6 +- 2 files changed, 101 insertions(+), 1 deletion(-) diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 2126d6d303..1a580bd5c9 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -975,6 +975,102 @@ mod tests { } } + /// Whether SIGPIPE is ignored right now, read by installing the same + /// disposition back. + #[allow(unsafe_code)] // libc::signal + fn sigpipe_is_ignored() -> bool { + // SAFETY: reading/installing a signal disposition is always safe. + let current = unsafe { libc::signal(libc::SIGPIPE, libc::SIG_IGN) }; + // SAFETY: putting back what was just read. + unsafe { + libc::signal(libc::SIGPIPE, current); + } + current == libc::SIG_IGN + } + + /// Forwarding guards each write on its own and never holds SIGPIPE ignored + /// across the reads between them. The reads block for as long as the engine + /// is quiet (an install runs for minutes), and the disposition is + /// process-wide, so a guard around the whole loop would leave every other + /// thread's write to a closed stdout returning an error for that whole time + /// instead of ending `rocm` quietly. Each line costs one pair of + /// `signal` calls, which is nothing next to the write itself. A fresh + /// process with `SIG_DFL` records the disposition seen inside every read + /// and every write, and after the loop. + #[test] + #[allow(unsafe_code)] // libc::signal, to install the default action in the child + fn forwarding_guards_each_write_and_never_the_reads_between_them() { + use std::cell::RefCell; + use std::rc::Rc; + + type Seen = Rc>>; + + struct Source { + lines: Vec<&'static [u8]>, + seen: Seen, + } + impl Read for Source { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + self.seen.borrow_mut().push(("read", sigpipe_is_ignored())); + if self.lines.is_empty() { + return Ok(0); + } + let line = self.lines.remove(0); + buffer[..line.len()].copy_from_slice(line); + Ok(line.len()) + } + } + struct Sink(Seen); + impl Write for Sink { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.borrow_mut().push(("write", sigpipe_is_ignored())); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + if std::env::var_os("ROCM_TEST_CLIENT_GUARD_SCOPE_ARM").is_some() { + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + let seen: Seen = Rc::default(); + let source = Source { + lines: vec![b"one\n", b"two\n", b"three\n"], + seen: Rc::clone(&seen), + }; + collect_stderr(source, Some(Sink(Rc::clone(&seen)))); + let seen = seen.borrow(); + let writes = seen.iter().filter(|(what, _)| *what == "write").count(); + assert_eq!(writes, 3, "{seen:?}"); + for (what, ignored) in seen.iter() { + assert_eq!( + *ignored, + *what == "write", + "SIGPIPE must be ignored inside each write and only there: {seen:?}" + ); + } + assert!( + !sigpipe_is_ignored(), + "the default action is back after the loop" + ); + return; + } + + let status = Command::new(std::env::current_exe().unwrap()) + .args([ + "stdio_client::tests::child::forwarding_guards_each_write_and_never_the_reads_between_them", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_CLIENT_GUARD_SCOPE_ARM", "1") + .status() + .unwrap(); + assert!(status.success(), "{status:?}"); + } + /// A request written to an engine that has already exited comes back as an /// ordinary error, also where SIGPIPE is at its default action. Re-executes /// this test in a fresh process that installs `SIG_DFL` first, because the diff --git a/docs/testing.md b/docs/testing.md index cd4eaf7d69..e398d4d85b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -237,7 +237,11 @@ writes are covered that way: the request to the child's stdin, and the forwardin of the child's stderr to this process's stderr (a line of progress, and the note recorded when reading it fails). The forwarding test writes into a real pipe whose reader is closed, because a mock sink that returns `BrokenPipe` itself never -raises the signal. `FakeEngine` quotes the response and its scratch directory +raises the signal. Each forwarding write is guarded on its own, never the loop +around it, because the reads between lines block for as long as the engine is +quiet and the disposition is process-wide; a third re-executed test, +`forwarding_guards_each_write_and_never_the_reads_between_them`, records the +disposition inside every read and every write and after the loop. `FakeEngine` quotes the response and its scratch directory for the shell, so a response such as `can't stop`, or a `TMPDIR` containing an apostrophe, reaches the client intact. From c9e7b98704fa5ad213bed647666922cc97ff9d45 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 13:27:22 +0000 Subject: [PATCH 14/38] style(engine-protocol): build the error messages without appending a format! to a String clippy's format_push_string, denied in CI, flags the three message builders added by the review fixes. They now rebuild the message with format! instead of push_str(&format!(..)); the text is unchanged and the existing tests pin it. Signed-off-by: Jussi Elo --- crates/rocm-engine-protocol/src/stdio_client.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 1a580bd5c9..e6f75d1ddd 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -205,10 +205,10 @@ fn exit_note(status: ExitStatus) -> String { fn unparseable_answer_message(status: ExitStatus, stderr: &str) -> String { let mut message = "failed to parse engine response envelope".to_owned(); if !status.success() { - message.push_str(&format!(" ({})", exit_note(status))); + message = format!("{message} ({})", exit_note(status)); } if !stderr.is_empty() { - message.push_str(&format!("; stderr: {stderr}")); + message = format!("{message}; stderr: {stderr}"); } message } @@ -244,7 +244,7 @@ fn abandon_unwritten_request( let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); let mut message = format!("failed to write engine request: {write_error}"); if !stderr.is_empty() { - message.push_str(&format!("; stderr: {stderr}")); + message = format!("{message}; stderr: {stderr}"); } anyhow::anyhow!(message) } From 30504f15cc809ccf32441872bebe56ccd5b57c94 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 14:33:42 +0000 Subject: [PATCH 15/38] fix(engine-protocol): stop collecting a failed write's stderr at the deadline even with output queued The bounded stderr read added for a request that cannot be written computed each receive's timeout from the remaining time. recv_timeout with an expired timeout still returns a message that is already queued, so a backlog, or a descendant that keeps writing to the pipe, could keep the loop collecting past the grace period for as long as the queue stayed non-empty, delaying the daemon's write error. The collection is now collect_until, which compares the deadline with the clock before every receive and receives nothing once it has passed, returning what had been collected. The one-second bound is unchanged. The other recv_timeout calls in the engine client and rocmd are a single fixed-duration receive in a test helper and in tests, not loops derived from a deadline. Tests run the collection directly on every platform: a channel pre-filled with 100000 messages and an already-expired deadline must collect nothing and return promptly (it collected all 1.6 MB with the previous loop), and a producer that keeps sending must not hold the collection past a short deadline. Signed-off-by: Jussi Elo --- .../rocm-engine-protocol/src/stdio_client.rs | 71 +++++++++++++++++-- docs/testing.md | 8 ++- 2 files changed, 74 insertions(+), 5 deletions(-) diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index e6f75d1ddd..9cd6125f6f 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -267,11 +267,26 @@ fn read_available(mut source: impl Read + Send + 'static, grace: Duration) -> Ve } } }); - let deadline = Instant::now() + grace; + collect_until(&receiver, Instant::now() + grace) +} + +/// Gather what arrives on `receiver` until it closes or `deadline` passes. +/// +/// The deadline is compared with the clock before every receive, and nothing is +/// received once it has passed: `recv_timeout` with an expired timeout still +/// hands back a message that is already queued, so a backlog, or a descendant +/// that keeps writing, would otherwise keep this collecting past the bound. +fn collect_until(receiver: &mpsc::Receiver>, deadline: Instant) -> Vec { let mut collected = Vec::new(); - while let Ok(chunk) = receiver.recv_timeout(deadline.saturating_duration_since(Instant::now())) - { - collected.extend(chunk); + loop { + let now = Instant::now(); + if now >= deadline { + break; + } + match receiver.recv_timeout(deadline - now) { + Ok(chunk) => collected.extend(chunk), + Err(_) => break, + } } collected } @@ -552,6 +567,54 @@ mod tests { ); } + /// Output already queued when the deadline has passed is not collected: + /// `recv_timeout(Duration::ZERO)` would still return it, one message after + /// another, for as long as the queue stays non-empty. + #[test] + fn collecting_stops_at_the_deadline_even_with_output_queued() { + let (sender, receiver) = mpsc::channel::>(); + for _ in 0..100_000 { + sender.send(vec![b'x'; 16]).unwrap(); + } + let started = Instant::now(); + let collected = collect_until(&receiver, Instant::now()); + assert!( + collected.is_empty(), + "collected {} bytes after the deadline", + collected.len() + ); + assert!(started.elapsed() < Duration::from_secs(1)); + + // Before the deadline, queued output is collected, and a closed channel + // ends the collection early. + drop(sender); + let collected = collect_until(&receiver, Instant::now() + Duration::from_secs(30)); + assert_eq!(collected.len(), 100_000 * 16); + } + + /// A producer that keeps writing (a descendant holding the pipe) cannot hold + /// the collection past its deadline. + #[test] + fn collecting_returns_at_the_deadline_while_output_keeps_arriving() { + let (sender, receiver) = mpsc::channel::>(); + let producer = thread::spawn(move || { + let until = Instant::now() + Duration::from_secs(3); + while Instant::now() < until { + if sender.send(vec![b'y'; 1024]).is_err() { + break; + } + thread::sleep(Duration::from_millis(1)); + } + }); + let started = Instant::now(); + let collected = collect_until(&receiver, Instant::now() + Duration::from_millis(100)); + let elapsed = started.elapsed(); + drop(receiver); + producer.join().unwrap(); + assert!(elapsed < Duration::from_secs(2), "took {elapsed:?}"); + assert!(!collected.is_empty(), "what arrived in time is kept"); + } + #[test] fn forwarded_stderr_is_written_out_and_still_collected() { let mut shown = Vec::new(); diff --git a/docs/testing.md b/docs/testing.md index 2a26ed82a7..c23e2a348e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -265,7 +265,13 @@ runs the whole client against a fake that exits without reading; `the_stderr_of_a_failed_write_is_not_waited_for_past_the_grace` one whose grandchild keeps the pipe open. The last two run the step on a real child with an injected write error, because whether a real write fails before the child has -closed its stdin is a race. +closed its stdin is a race. The bound compares the deadline with the clock +before every receive and receives nothing once it has passed, because a receive +with an expired timeout still returns output that is already queued, which would +let a backlog or a descendant that keeps writing hold the error path past it; +`collecting_stops_at_the_deadline_even_with_output_queued` and +`collecting_returns_at_the_deadline_while_output_keeps_arriving` run that +collection directly, on every platform. One wait is deliberately not bounded and has no test: after the child exits, the answer is read to the end of its stdout and its stderr collector is joined, so an From f15031862ffb3cd365f99fd6205e541a9f33f4b0 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 15:08:05 +0000 Subject: [PATCH 16/38] test(engine-protocol): show that forward_stderr sends the child's stderr to this process's stderr The forward_stderr option took effect in exactly one expression, forward_stderr.then(io::stderr), and no test ran the client with it set, so replacing it with None left every test green although it is what shows a user an engine install's progress. The public entry point now hands the sink to a private function that takes it, so a test can pass a Vec-backed writer and see the child's stderr arrive as it is written. A second test runs the public entry point in a re-executed copy of the test binary and reads that process's real stderr from the parent: the child's line is there with the option and absent without it. The second test turns red when the forwarding is replaced with None. docs/testing.md describes both. Signed-off-by: Jussi Elo --- .../rocm-engine-protocol/src/stdio_client.rs | 114 +++++++++++++++++- docs/testing.md | 9 ++ 2 files changed, 119 insertions(+), 4 deletions(-) diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 9cd6125f6f..fc3b07e38e 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -137,6 +137,30 @@ pub fn run_engine_stdio( options: &EngineStdioOptions<'_>, envelope: &EngineRequestEnvelope, ) -> Result +where + I: IntoIterator, + S: AsRef, +{ + run_engine_stdio_forwarding( + program, + args, + options, + envelope, + options.forward_stderr.then(io::stderr), + ) +} + +/// [`run_engine_stdio`] with the sink the child's stderr is forwarded to chosen +/// by the caller (`None` forwards nothing), so a test can see what is forwarded +/// without capturing this process's own stderr. [`run_engine_stdio`] is the one +/// place the `forward_stderr` option becomes a sink. +fn run_engine_stdio_forwarding( + program: &Path, + args: I, + options: &EngineStdioOptions<'_>, + envelope: &EngineRequestEnvelope, + forward: Option, +) -> Result where I: IntoIterator, S: AsRef, @@ -167,10 +191,10 @@ where // End of the request: the engine reads to EOF before it answers. drop(stdin); - let forward_stderr = options.forward_stderr; - let stderr_reader = child.stderr.take().map(|stderr| { - thread::spawn(move || collect_stderr(stderr, forward_stderr.then(io::stderr))) - }); + let stderr_reader = child + .stderr + .take() + .map(|stderr| thread::spawn(move || collect_stderr(stderr, forward))); let stdout_pipe = child .stdout .take() @@ -990,6 +1014,88 @@ mod tests { ); } + /// A writer that keeps what it is given where the test can read it. + #[derive(Clone, Default)] + struct SharedSink(std::sync::Arc>>); + impl Write for SharedSink { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + /// What the running child writes to its stderr reaches the forwarding + /// sink as it arrives. (That nothing is forwarded without the option is + /// the `off` arm of the test below.) + #[test] + fn a_childs_stderr_reaches_the_forwarding_sink_when_there_is_one() { + let progress = "downloading 1/2\ndownloading 2/2\n"; + let body = format!("printf '%s' '{}' >&2\nprintf '%s' '{STOPPED}'", progress); + let engine = FakeEngine::new("client-forward-sink", true, &body); + + let sink = SharedSink::default(); + let answer = run_engine_stdio_forwarding( + engine.path(), + ["stdio"], + &EngineStdioOptions::default(), + &stop_request(), + Some(sink.clone()), + ) + .unwrap(); + assert!(answer.ok); + assert_eq!(sink.0.lock().unwrap().as_slice(), progress.as_bytes()); + } + + /// The public entry point sends the child's stderr to THIS process's + /// stderr when `forward_stderr` is set, and does not when it is not. That + /// is the one place the option takes effect, and it is what shows a user + /// an engine install's progress. The test binary re-executes itself so the + /// parent can read the stderr the arm writes. + #[test] + fn forward_stderr_sends_the_childs_stderr_to_this_processs_stderr() { + const LINE: &str = "engine progress line for the terminal"; + if let Some(arm) = std::env::var_os("ROCM_TEST_CLIENT_FORWARD_OPTION_ARM") { + let engine = FakeEngine::new( + "client-forward-option", + true, + &format!("echo '{LINE}' >&2\nprintf '%s' '{STOPPED}'"), + ); + let options = EngineStdioOptions { + forward_stderr: arm == "on", + ..EngineStdioOptions::default() + }; + run_engine_stdio(engine.path(), ["stdio"], &options, &stop_request()).unwrap(); + return; + } + + for (arm, expected) in [("on", true), ("off", false)] { + let output = Command::new(std::env::current_exe().unwrap()) + .args([ + "stdio_client::tests::child::forward_stderr_sends_the_childs_stderr_to_this_processs_stderr", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_CLIENT_FORWARD_OPTION_ARM", arm) + .output() + .unwrap(); + assert!(output.status.success(), "{arm}: {:?}", output.status); + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!( + stderr.contains(LINE), + expected, + "forward_stderr {arm}: the child's stderr {} this process's stderr:\n{stderr}", + if expected { + "must reach" + } else { + "must not reach" + } + ); + } + } + /// Progress forwarded to a stderr whose reader has gone away is dropped, not /// fatal, also where SIGPIPE is at its default action (`rocm` runs that /// way). The `Closed` sink above cannot show this: it returns the error diff --git a/docs/testing.md b/docs/testing.md index c23e2a348e..b4b2d9bdc4 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -245,6 +245,15 @@ disposition inside every read and every write and after the loop. `FakeEngine` q for the shell, so a response such as `can't stop`, or a `TMPDIR` containing an apostrophe, reaches the client intact. +The `forward_stderr` option, which is what shows a user an engine install's +progress, is tested at both ends: the client runs against a private entry point +that takes the sink (`a_childs_stderr_reaches_the_forwarding_sink_when_there_is_one`, +a `Vec`-backed writer), and the public entry point is run in a re-executed test +binary whose own stderr the parent reads +(`forward_stderr_sends_the_childs_stderr_to_this_processs_stderr`: the child's +line is there with the option and absent without it). Replacing the one +`forward_stderr.then(io::stderr)` with `None` turns the second red. + A child whose answer cannot be read is killed and reaped before the error is returned, as it is when the request cannot be written, so the long-lived daemon does not collect a running engine or a zombie per failure. From a44f2d337e91729d8a4ac096f1463f476c34cea1 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 15:10:03 +0000 Subject: [PATCH 17/38] fix(engine-protocol): word a failing exit from its code or signal, not from ExitStatus's own text ExitStatus already displays as "exit status: N" on Unix ("exit code: N" on Windows), so the failing-exit messages read "exited with status exit status: N", and the wording was now pinned in several tests. The note is built from status.code() ("engine stdio process exited with code N"), and for a child ended by a signal on Unix from the signal ("was terminated by signal N"), with a neutral fallback when there is neither, so the text no longer differs between platforms. The unit tests, rocmd's engine_ipc test and the engine-ipc-02 scenario now expect the new wording, and docs/testing.md describes it. A new test shows a child killed by a signal is reported by the signal, with and without a half-written answer; it turns red when the signal branch is removed. The other "exit status" phrases in the repository belong to unrelated code (the vLLM startup diagnostics, the Lemonade diagnostics, amd-smi) and are left alone. Signed-off-by: Jussi Elo --- apps/rocmd/src/engine_ipc.rs | 2 +- .../rocm-engine-protocol/src/stdio_client.rs | 53 +++++++++++++++---- docs/testing.md | 5 +- .../tests/e2e/service_stop_steps.rs | 2 +- 4 files changed, 50 insertions(+), 12 deletions(-) diff --git a/apps/rocmd/src/engine_ipc.rs b/apps/rocmd/src/engine_ipc.rs index ff69c4cd63..242a8348ab 100644 --- a/apps/rocmd/src/engine_ipc.rs +++ b/apps/rocmd/src/engine_ipc.rs @@ -613,7 +613,7 @@ mod tests { let crash = FakeEngine::new("rocmd-crash", true, "echo boom >&2\nexit 3"); assert_eq!( request(&crash, &paths).unwrap_err().to_string(), - "engine stdio process exited with status exit status: 3: boom" + "engine stdio process exited with code 3: boom" ); } diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index fc3b07e38e..6dbb9dccab 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -218,9 +218,22 @@ where serde_json::from_slice(&stdout).with_context(|| unparseable_answer_message(status, &stderr)) } -/// How a failing exit is worded wherever the status is reported. +/// How a failing exit is worded wherever the status is reported. Built from the +/// exit code (or the signal that ended the process) rather than from the status's +/// own `Display`, which already says "exit status: N" on Unix and "exit code: N" +/// on Windows. fn exit_note(status: ExitStatus) -> String { - format!("engine stdio process exited with status {status}") + if let Some(code) = status.code() { + return format!("engine stdio process exited with code {code}"); + } + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt as _; + if let Some(signal) = status.signal() { + return format!("engine stdio process was terminated by signal {signal}"); + } + } + "engine stdio process ended without an exit code".to_owned() } /// The error for an answer that is not a response envelope. A child that also @@ -737,15 +750,12 @@ mod tests { let error = run(&engine, &EngineStdioOptions::default()).unwrap_err(); assert_eq!( error.to_string(), - "engine stdio process exited with status exit status: 3: boom" + "engine stdio process exited with code 3: boom" ); let silent = FakeEngine::new("client-silent-crash", true, "exit 4"); let error = run(&silent, &EngineStdioOptions::default()).unwrap_err(); - assert_eq!( - error.to_string(), - "engine stdio process exited with status exit status: 4" - ); + assert_eq!(error.to_string(), "engine stdio process exited with code 4"); } #[test] @@ -786,6 +796,31 @@ mod tests { ); } + /// A child ended by a signal has no exit code; it is reported by the signal, + /// with or without a half-written answer, and never as a code of its own + /// invention. + #[test] + fn a_child_ended_by_a_signal_is_reported_by_the_signal() { + let killed = FakeEngine::new("client-killed", true, "kill -KILL $$"); + let error = run(&killed, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "engine stdio process was terminated by signal 9" + ); + + let cut_short = FakeEngine::new( + "client-killed-mid-answer", + true, + "printf '{\"ok\"'\nkill -KILL $$", + ); + let error = run(&cut_short, &EngineStdioOptions::default()).unwrap_err(); + assert_eq!( + error.to_string(), + "failed to parse engine response envelope \ + (engine stdio process was terminated by signal 9)" + ); + } + /// A child that fails and also leaves something that is not an envelope on /// stdout is reported with its exit status (the status says the answer was /// cut short, where the parse error alone would not), as a child that @@ -803,7 +838,7 @@ mod tests { assert_eq!( error.to_string(), "failed to parse engine response envelope \ - (engine stdio process exited with status exit status: 4); stderr: oops" + (engine stdio process exited with code 4); stderr: oops" ); let silent = FakeEngine::new( @@ -815,7 +850,7 @@ mod tests { assert_eq!( error.to_string(), "failed to parse engine response envelope \ - (engine stdio process exited with status exit status: 5)" + (engine stdio process exited with code 5)" ); let clean = FakeEngine::new("client-garbage-clean", true, "printf 'not json'\nexit 0"); diff --git a/docs/testing.md b/docs/testing.md index b4b2d9bdc4..1e891b7784 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -224,7 +224,10 @@ grew its own private copy of the conversation would fail those tests. An answer that is not a response envelope is reported as that, and when the child also exited with a failing status the error says so, because the status is often why the answer was cut short (a clean exit adds nothing). A failing child -that printed nothing is reported by its status alone. Both carry the child's +that printed nothing is reported by its status alone. The status is worded from +the exit code (`exited with code N`) or, for a child ended by a signal on Unix, +the signal (`was terminated by signal N`), not from `ExitStatus`'s own text, which +already reads `exit status: N` and would double up (`a_child_ended_by_a_signal_is_reported_by_the_signal`). Both carry the child's stderr. `an_unparseable_answer_from_a_failing_child_names_the_exit_status` pins the three cases against the fake, and `engine-ipc-02` below shows the same message in a `stop_server` report next to what the stop did. diff --git a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs index 59f512a25c..250774a8f3 100644 --- a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs @@ -479,7 +479,7 @@ async fn unreadable_answer_reported_with_stderr(world: &mut E2eWorld) { assert_eq!( engine_stop["error"], "failed to parse engine response envelope \ - (engine stdio process exited with status exit status: 1); \ + (engine stdio process exited with code 1); \ stderr: engine exploded: no container runtime", "{report}" ); From bef146995af917da9fac833ef4f6bcbeff90d2c8 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 15:11:11 +0000 Subject: [PATCH 18/38] docs(engine-protocol): state the stalled-sink SIGPIPE limit, and narrow the drift claim to behaviour Three statements made more or less than the code and tests support. The stdio client's module comment now says that the SIGPIPE guard around a forwarded stderr write is held for as long as the write blocks, so a stalled (rather than closed) sink leaves SIGPIPE ignored process-wide for that time, and a concurrent print to a closed pipe on another thread fails with an error instead of ending the process quietly. It is accepted, with the reason. docs/testing.md said a caller with a private copy of the conversation would fail the shared tests. It would fail them only if its behaviour differed, and a copy that behaves the same passes, so the sentence now says the tests catch behavioural drift and nothing here rules out a second copy; a structural check would read the callers' files from outside the crate, which prepush's cross-crate-read rule only accepts with a widening-rule entry. The producer test for the bounded stderr collection is timing-based and also passes with the loop it was written about, so its comment now says it is a secondary guard and points to the deterministic test that covers the fix. Signed-off-by: Jussi Elo --- crates/rocm-engine-protocol/src/stdio_client.rs | 17 ++++++++++++++++- docs/testing.md | 12 ++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 6dbb9dccab..c989f24176 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -34,6 +34,17 @@ //! this change leaves it to be decided on its own. The one wait this client adds //! itself, for the stderr of a child whose request could not be written, is //! bounded by `EXITED_STDERR_GRACE`. +//! +//! Forwarding the child's stderr guards each write against SIGPIPE on its own +//! (see `forward_best_effort`), and the guard is held for as long as that write +//! blocks. If the sink is stalled rather than closed (a terminal that has stopped +//! reading), SIGPIPE therefore stays ignored process-wide for that time, and a +//! concurrent `print!` to a closed pipe on another thread fails with an error +//! (which `print!` turns into a panic) instead of ending the process quietly. The +//! guard is not held across the reads between lines, so this needs a stalled +//! sink, not just a quiet engine. It is accepted: a guard that could be taken +//! back mid-write does not exist, and without one a closed sink would end +//! `rocm` before the request could report why. use std::ffi::OsStr; use std::io::{self, BufRead, Read, Write}; @@ -630,7 +641,11 @@ mod tests { } /// A producer that keeps writing (a descendant holding the pipe) cannot hold - /// the collection past its deadline. + /// the collection past its deadline. A secondary guard: it is timing-based + /// and also passes with the loop that let queued output bypass the deadline, + /// because that producer sleeps between messages. The fix is covered by + /// `collecting_stops_at_the_deadline_even_with_output_queued` above, which is + /// deterministic. #[test] fn collecting_returns_at_the_deadline_while_output_keeps_arriving() { let (sender, receiver) = mpsc::channel::>(); diff --git a/docs/testing.md b/docs/testing.md index 1e891b7784..091b3a49c5 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -218,8 +218,16 @@ builds, `decode_engine_response`, stderr collection) run on every platform, and the ones that start the fake are `#[cfg(unix)]`. The `rocm` and `rocmd` callers each run their own entry point against the same fake and pin only what they choose: the program and verb, the app paths, and whether stderr is forwarded (see -`apps/rocm/src/engine_ipc.rs` and `apps/rocmd/src/engine_ipc.rs`). A caller that -grew its own private copy of the conversation would fail those tests. +`apps/rocm/src/engine_ipc.rs` and `apps/rocmd/src/engine_ipc.rs`). These tests +catch behavioural drift: a caller whose own copy of the conversation behaved +differently from the shared client in what they pin (the framing, the +environment, the exit and answer handling, the guards) would fail them, but a +copy that behaves the same passes them, so nothing here rules out a second +copy existing. A structural check would have to read the callers' files from +outside this crate, a cross-crate read that `cargo xtask prepush` only accepts +with an entry in its widening rule +(`every_cross_crate_read_is_covered_by_the_widening_rule`), which is not worth +adding for this, so the guarantee is for behaviour only. An answer that is not a response envelope is reported as that, and when the child also exited with a failing status the error says so, because the status is From 838d9093f62e74496bd75f6ff865556bc2dc108f Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 15:14:14 +0000 Subject: [PATCH 19/38] style(engine-protocol): inline the format argument in the forwarding test clippy's uninlined_format_args, denied in CI, flags the body the forwarding-sink test builds. The text is unchanged. Signed-off-by: Jussi Elo --- crates/rocm-engine-protocol/src/stdio_client.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index c989f24176..23156cd7a5 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -1083,7 +1083,7 @@ mod tests { #[test] fn a_childs_stderr_reaches_the_forwarding_sink_when_there_is_one() { let progress = "downloading 1/2\ndownloading 2/2\n"; - let body = format!("printf '%s' '{}' >&2\nprintf '%s' '{STOPPED}'", progress); + let body = format!("printf '%s' '{progress}' >&2\nprintf '%s' '{STOPPED}'"); let engine = FakeEngine::new("client-forward-sink", true, &body); let sink = SharedSink::default(); From 8486362b7f1bc87715ff8c265abef8c1e63cac92 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 15:47:24 +0000 Subject: [PATCH 20/38] fix(rocmd): recognise the standalone daemon by its whole file name The check for the standalone rocmd compared the file stem with "rocmd", so on Linux a rocmd.bak or rocmd.old was taken for the daemon and told to reach engines through a sibling rocm, while the comment and the PR text said exactly rocmd (rocmd.exe on Windows). The pure helper now compares the whole file name after the " (deleted)" marker has been stripped: exactly rocmd, and rocmd.exe (in any case) only where the platform is Windows, which is a parameter so both spellings are tested on every platform. The test covers rocmd, rocmd (deleted), rocmd.exe and ROCMD.EXE as Windows names, and rocmd.bak, rocmd.old, rocmd.bak (deleted), a rocmd.exe on a non-Windows platform and a versioned name as not the daemon. It turns red with the file-stem comparison. Signed-off-by: Jussi Elo --- apps/rocmd/src/engine_ipc.rs | 75 +++++++++++++++++++++++++----------- 1 file changed, 52 insertions(+), 23 deletions(-) diff --git a/apps/rocmd/src/engine_ipc.rs b/apps/rocmd/src/engine_ipc.rs index 242a8348ab..1ab1016e19 100644 --- a/apps/rocmd/src/engine_ipc.rs +++ b/apps/rocmd/src/engine_ipc.rs @@ -139,21 +139,30 @@ where fn rocm_executable() -> Result { let current = std::env::current_exe().context("failed to resolve current rocm executable path")?; - rocm_executable_from(current, || rocm_core::sibling_binary_path("rocm")) + rocm_executable_from(current, cfg!(windows), || { + rocm_core::sibling_binary_path("rocm") + }) } -/// [`rocm_executable`] given this process's executable and the way to find a -/// sibling `rocm`, so the choice can be tested without either. +/// [`rocm_executable`] given this process's executable, whether it runs on +/// Windows, and the way to find a sibling `rocm`, so the choice can be tested +/// without any of them. fn rocm_executable_from( current: PathBuf, + windows: bool, sibling_rocm: impl FnOnce() -> Result, ) -> Result { let current = without_deleted_marker(current); - // Exactly `rocmd` (or `rocmd.exe`). A renamed or test binary is not known to - // lack the verb, and has no `rocm` beside it to prefer: it runs itself, so a - // standalone `rocmd` shipped under another name still answers an engine - // request with "unrecognized subcommand". - if current.file_stem().is_some_and(|stem| stem == "rocmd") { + // The whole file name is `rocmd`, or `rocmd.exe` on Windows: not `rocmd.bak`, + // not `rocmd-0.3`. A renamed or test binary is not known to lack the verb, and + // has no `rocm` beside it to prefer: it runs itself, so a standalone `rocmd` + // shipped under another name still answers an engine request with + // "unrecognized subcommand". + if current + .file_name() + .and_then(|name| name.to_str()) + .is_some_and(|name| is_standalone_rocmd_name(name, windows)) + { return sibling_rocm().context( "the standalone rocmd needs the rocm executable beside it to reach an engine", ); @@ -161,6 +170,13 @@ fn rocm_executable_from( Ok(current) } +/// Whether `name`, a file name, is the standalone daemon's: exactly `rocmd`, plus +/// `rocmd.exe` (in any case) where `windows` says the platform names executables +/// that way. The platform is a parameter so both spellings are tested everywhere. +fn is_standalone_rocmd_name(name: &str, windows: bool) -> bool { + name == "rocmd" || (windows && name.eq_ignore_ascii_case("rocmd.exe")) +} + /// What Linux appends to the name `current_exe()` reports when the running /// executable's file has been replaced or removed since the process started, /// which is what an in-place upgrade of a long-lived daemon does. @@ -407,32 +423,45 @@ mod tests { // The standalone binary has no `__engine-stdio` verb, so it must not // re-execute itself. - for standalone in ["/opt/rocm-cli/bin/rocmd", "C:/rocm-cli/bin/rocmd.exe"] { + for (standalone, windows) in [ + ("/opt/rocm-cli/bin/rocmd", false), + ("/opt/rocm-cli/bin/rocmd", true), + ("C:/rocm-cli/bin/rocmd.exe", true), + ("C:/rocm-cli/bin/ROCMD.EXE", true), + ] { assert_eq!( - rocm_executable_from(PathBuf::from(standalone), beside).unwrap(), + rocm_executable_from(PathBuf::from(standalone), windows, beside).unwrap(), Path::new("/opt/rocm-cli/bin/rocm"), - "{standalone}" + "{standalone} (windows: {windows})" ); } - // Embedded as `rocm daemon`, this process is `rocm`; a test binary - // or a renamed copy has no `rocm` beside it to prefer. - for own in [ - "/opt/rocm-cli/bin/rocm", - "C:/rocm-cli/bin/rocm.exe", - "/work/target/debug/deps/rocmd-0f3a9c1d", - "/opt/rocm-cli/bin/rocm-0.3", + // Embedded as `rocm daemon`, this process is `rocm`; a test binary, a + // renamed copy or a backup of the daemon has no `rocm` beside it to + // prefer, and only the whole name `rocmd` counts: a file stem would also + // match `rocmd.bak`. `rocmd.exe` is the daemon's name only on Windows. + for (own, windows) in [ + ("/opt/rocm-cli/bin/rocm", false), + ("C:/rocm-cli/bin/rocm.exe", true), + ("/work/target/debug/deps/rocmd-0f3a9c1d", false), + ("/opt/rocm-cli/bin/rocm-0.3", false), + ("/opt/rocm-cli/bin/rocmd.bak", false), + ("/opt/rocm-cli/bin/rocmd.old", false), + ("C:/rocm-cli/bin/rocmd.bak", true), + ("/opt/rocm-cli/bin/rocmd.exe", false), + ("/opt/rocm-cli/bin/rocmd.bak (deleted)", false), ] { - let chosen = rocm_executable_from(PathBuf::from(own), || { + let chosen = rocm_executable_from(PathBuf::from(own), windows, || { panic!("{own} has no use for a sibling") }) .unwrap(); - assert_eq!(chosen, Path::new(own)); + let expected = own.strip_suffix(" (deleted)").unwrap_or(own); + assert_eq!(chosen, Path::new(expected), "{own} (windows: {windows})"); } // A standalone `rocmd` with no `rocm` beside it fails with a message that // says why, rather than running something that cannot serve the verb. - let error = rocm_executable_from(PathBuf::from("/opt/rocm-cli/bin/rocmd"), || { + let error = rocm_executable_from(PathBuf::from("/opt/rocm-cli/bin/rocmd"), false, || { anyhow::bail!("unable to locate sibling binary rocm") }) .unwrap_err(); @@ -453,12 +482,12 @@ mod tests { let beside = || Ok(PathBuf::from("/usr/bin/rocm")); let chosen = - rocm_executable_from(PathBuf::from("/usr/bin/rocmd (deleted)"), beside).unwrap(); + rocm_executable_from(PathBuf::from("/usr/bin/rocmd (deleted)"), false, beside).unwrap(); assert_eq!(chosen, Path::new("/usr/bin/rocm")); // Embedded `rocm daemon`: run the file now at that path, not the // marker-suffixed name, which no file has. - let chosen = rocm_executable_from(PathBuf::from("/usr/bin/rocm (deleted)"), || { + let chosen = rocm_executable_from(PathBuf::from("/usr/bin/rocm (deleted)"), false, || { panic!("an embedded rocm has no use for a sibling") }) .unwrap(); From e399137d53f3b1ae6b813d1c548f7a4d3b53991f Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 15:54:21 +0000 Subject: [PATCH 21/38] test(rocm): compare a child's inherited environment with the ambient one at the same instant The env-inheritance tests of the rocm engine client compared the variables a child inherited with std::env::var read afterwards. Other tests in the module set ROCM_CLI_* process-wide through with_scoped_builtin_engine_env, which holds BUILTIN_ENGINE_ENV_LOCK, while these tests did not, so under parallel test threads the child could start while the variables were scoped and the read see them restored, or the reverse. The tests now start the child and read the ambient value inside one region holding the same lock. A new test runs the comparison while another thread keeps scoping the variables and fails if the lock is removed (it did, on the first round). The same pattern was looked for in the other tests this change added: rocmd has no test or helper that sets process environment, and the engine client's own tests compare against values the test set explicitly, so nothing else needed it. Signed-off-by: Jussi Elo --- apps/rocm/src/engine_ipc.rs | 95 +++++++++++++++++++++++++++++-------- 1 file changed, 76 insertions(+), 19 deletions(-) diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index db883adcf5..8be6929242 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -475,6 +475,18 @@ mod tests { ) } + /// Run `f` holding the lock that serialises every test that sets + /// `ROCM_CLI_*` process-wide (`with_scoped_builtin_engine_env`), so that + /// what a child inherits when `f` starts it and what [`ambient`] reads + /// inside `f` come from the same instant. + fn under_env_lock(f: impl FnOnce() -> R) -> R { + let lock = BUILTIN_ENGINE_ENV_LOCK.get_or_init(|| Mutex::new(())); + let _guard = lock + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + f() + } + fn ambient(key: &str) -> Option { std::env::var(key).ok() } @@ -515,16 +527,60 @@ mod tests { #[test] fn the_child_inherits_the_callers_environment_when_no_app_paths_are_given() { - let engine = FakeEngine::answering("rocm-no-app-paths", STOPPED, 0); - request(&engine, None, None, false).unwrap(); - - for key in [ + const KEYS: [&str; 3] = [ "ROCM_CLI_CONFIG_DIR", "ROCM_CLI_DATA_DIR", "ROCM_CLI_CACHE_DIR", - ] { - assert_eq!(engine.env(key), ambient(key), "{key}"); + ]; + let engine = FakeEngine::answering("rocm-no-app-paths", STOPPED, 0); + let expected = under_env_lock(|| { + request(&engine, None, None, false).unwrap(); + KEYS.map(ambient) + }); + + for (key, expected) in KEYS.into_iter().zip(expected) { + assert_eq!(engine.env(key), expected, "{key}"); + } + } + + /// The comparison of a child's inherited environment with the ambient one + /// holds while another thread scopes `ROCM_CLI_*` process-wide, which + /// other tests of this module do: the spawn and the read happen under the + /// same lock. Without the lock the child can start while the variables + /// are scoped and the read see them restored (or the other way round). + #[test] + fn the_inherited_environment_is_compared_at_the_instant_the_child_starts() { + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + + let (root, paths) = crate::tests::test_paths("engine-ipc-env-race"); + let done = Arc::new(AtomicBool::new(false)); + let flipper = { + let done = Arc::clone(&done); + let paths = paths.clone(); + std::thread::spawn(move || { + while !done.load(Ordering::Relaxed) { + with_scoped_builtin_engine_env(Some(&paths), None, || { + std::thread::sleep(std::time::Duration::from_millis(2)); + }); + // Outside the lock, so the test's own turn is not starved. + std::thread::sleep(std::time::Duration::from_millis(2)); + } + }) + }; + + for round in 0..25 { + let engine = FakeEngine::answering("rocm-env-race", STOPPED, 0); + let expected = under_env_lock(|| { + request(&engine, None, None, false).unwrap(); + ambient("ROCM_CLI_DATA_DIR") + }); + assert_eq!(engine.env("ROCM_CLI_DATA_DIR"), expected, "round {round}"); } + + done.store(true, Ordering::Relaxed); + flipper.join().unwrap(); + let _ = fs::remove_dir_all(root); } #[test] @@ -538,11 +594,11 @@ mod tests { ); let without = FakeEngine::answering("rocm-no-env-root", STOPPED, 0); - request(&without, None, None, false).unwrap(); - assert_eq!( - without.env("ROCM_CLI_ENGINE_ENVS_ROOT"), + let expected = under_env_lock(|| { + request(&without, None, None, false).unwrap(); ambient("ROCM_CLI_ENGINE_ENVS_ROOT") - ); + }); + assert_eq!(without.env("ROCM_CLI_ENGINE_ENVS_ROOT"), expected); } #[test] @@ -555,11 +611,11 @@ mod tests { ); let quiet = FakeEngine::answering("rocm-progress-off", STOPPED, 0); - request(&quiet, None, None, false).unwrap(); - assert_eq!( - quiet.env("ROCM_ENGINE_PROGRESS_STDERR"), + let expected = under_env_lock(|| { + request(&quiet, None, None, false).unwrap(); ambient("ROCM_ENGINE_PROGRESS_STDERR") - ); + }); + assert_eq!(quiet.env("ROCM_ENGINE_PROGRESS_STDERR"), expected); } #[test] @@ -568,11 +624,12 @@ mod tests { // install, resolve-model, stop), so this client has no service to find // a key for. `rocmd`'s healthcheck is the call that threads one. let engine = FakeEngine::answering("rocm-no-key", STOPPED, 0); - request(&engine, None, None, false).unwrap(); - assert_eq!( - engine.env(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV), - ambient(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV) - ); + let key = rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV; + let expected = under_env_lock(|| { + request(&engine, None, None, false).unwrap(); + ambient(key) + }); + assert_eq!(engine.env(key), expected); } #[test] From 6785a7ce826c09777716d8bd40325a9ec7f9a15e Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 16:10:28 +0000 Subject: [PATCH 22/38] test(engine-protocol): cover the stdio client's remaining branches so the crate holds its coverage floor The coverage gate failed for rocm-engine-protocol (97.78% against a floor of 98.37%) because the client added to the crate brought lines the tests did not run. The floor is not re-blessed; the lines are covered. Added: a test for the neutral wording of a status that has neither an exit code nor a signal (a stopped process, Unix only), and one for the bounded read's left-behind reader thread, which must end when its source finally answers and nobody is listening (the source never reaches its end, so a thread that kept reading would never end; it fails if the break is removed). The test sinks became one closure-backed writer whose flush is exercised once, and assertion messages no longer build their arguments on lines that only run when an assertion fails. The grandchild test waits for its child instead of polling it. A run of cargo llvm-cov over the protocol, rocmd and rocm tests now measures the crate at 98.5% lines (1454 of 1476), with one source line below it uncovered by a nondeterministic poll that this change also removes. Signed-off-by: Jussi Elo --- .../rocm-engine-protocol/src/stdio_client.rs | 154 ++++++++++++------ 1 file changed, 104 insertions(+), 50 deletions(-) diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 23156cd7a5..d7638e19fb 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -584,6 +584,68 @@ mod tests { ); } + /// A status that is neither an exit nor a signal (a stopped process, on + /// Unix) has neither a code nor a signal to name, and is reported as that. + #[cfg(unix)] + #[test] + fn a_status_with_no_code_and_no_signal_gets_the_neutral_wording() { + use std::os::unix::process::ExitStatusExt as _; + // Stopped by SIGSTOP: WIFSTOPPED, which is neither WIFEXITED nor WIFSIGNALED. + let stopped = ExitStatus::from_raw(0x137f); + assert_eq!(stopped.code(), None); + assert_eq!(stopped.signal(), None); + assert_eq!( + exit_note(stopped), + "engine stdio process ended without an exit code" + ); + } + + /// A source that does not answer until released, then never stops answering, + /// and says when it is dropped. + struct Gated { + release: mpsc::Receiver<()>, + dropped: mpsc::Sender<()>, + released: bool, + } + impl Read for Gated { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + if !self.released { + self.release.recv().unwrap(); + self.released = true; + } + buffer[..4].copy_from_slice(b"late"); + Ok(4) + } + } + impl Drop for Gated { + fn drop(&mut self) { + let _ = self.dropped.send(()); + } + } + + /// Once the bounded read has given up, the reading thread that is still + /// blocked is not left spinning: when its source finally answers it finds + /// nobody listening and ends (it drops the source, which the test sees). The + /// source never reaches its end, so a thread that kept reading after the + /// failed send would never end. + #[test] + fn the_reader_left_behind_by_a_bounded_read_ends_when_nobody_is_listening() { + let (release, released) = mpsc::channel(); + let (dropped, dropped_signal) = mpsc::channel(); + let source = Gated { + release: released, + dropped, + released: false, + }; + let collected = read_available(source, Duration::from_millis(50)); + assert!(collected.is_empty()); + + release.send(()).unwrap(); + dropped_signal + .recv_timeout(Duration::from_secs(10)) + .expect("the abandoned reader thread did not end"); + } + /// A stderr whose reads fail. struct Fails; impl Read for Fails { @@ -592,17 +654,25 @@ mod tests { } } - /// A sink that refuses every write, like a closed terminal. - struct Closed; - impl Write for Closed { - fn write(&mut self, _: &[u8]) -> io::Result { - Err(io::Error::from(io::ErrorKind::BrokenPipe)) + /// A sink that hands every write to a closure: one that refuses them is a + /// closed terminal, one that records them shows what was forwarded. + struct FnSink(F); + impl io::Result> Write for FnSink { + fn write(&mut self, bytes: &[u8]) -> io::Result { + (self.0)(bytes) } fn flush(&mut self) -> io::Result<()> { Ok(()) } } + #[test] + fn a_closure_sink_writes_through_and_flushes_nothing() { + let mut sink = FnSink(|bytes: &[u8]| Ok(bytes.len())); + assert_eq!(sink.write(b"abc").unwrap(), 3); + sink.flush().unwrap(); + } + #[test] fn stderr_is_collected_whole_and_a_read_error_is_recorded() { let collected = collect_stderr(&b"first\nsecond"[..], None::>); @@ -626,11 +696,8 @@ mod tests { } let started = Instant::now(); let collected = collect_until(&receiver, Instant::now()); - assert!( - collected.is_empty(), - "collected {} bytes after the deadline", - collected.len() - ); + let after_deadline = collected.len(); + assert_eq!(after_deadline, 0, "collected bytes after the deadline"); assert!(started.elapsed() < Duration::from_secs(1)); // Before the deadline, queued output is collected, and a closed channel @@ -675,7 +742,12 @@ mod tests { assert_eq!(collected, b"step 1\nstep 2\n"); // A sink that refuses writes (a closed terminal) costs nothing. - let collected = collect_stderr(&b"still kept\n"[..], Some(Closed)); + let collected = collect_stderr( + &b"still kept\n"[..], + Some(FnSink(|_: &[u8]| { + Err(io::Error::from(io::ErrorKind::BrokenPipe)) + })), + ); assert_eq!(collected, b"still kept\n"); } @@ -971,9 +1043,7 @@ mod tests { .unwrap(); // The engine has said its piece and exited before the failure is // handled, as it has in the case this bounds. - while child.try_wait().unwrap().is_none() { - thread::sleep(Duration::from_millis(5)); - } + child.wait().unwrap(); let started = Instant::now(); let error = finishes_within(Duration::from_secs(4), move || { @@ -984,10 +1054,10 @@ mod tests { ) }); + let waited = started.elapsed(); assert!( - started.elapsed() < Duration::from_secs(4), - "waited for the grandchild: {:?}", - started.elapsed() + waited < Duration::from_secs(4), + "waited for the grandchild: {waited:?}" ); assert!(error.to_string().ends_with("; stderr: said"), "{error}"); } @@ -1064,19 +1134,6 @@ mod tests { ); } - /// A writer that keeps what it is given where the test can read it. - #[derive(Clone, Default)] - struct SharedSink(std::sync::Arc>>); - impl Write for SharedSink { - fn write(&mut self, bytes: &[u8]) -> io::Result { - self.0.lock().unwrap().extend_from_slice(bytes); - Ok(bytes.len()) - } - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } - } - /// What the running child writes to its stderr reaches the forwarding /// sink as it arrives. (That nothing is forwarded without the option is /// the `off` arm of the test below.) @@ -1086,17 +1143,24 @@ mod tests { let body = format!("printf '%s' '{progress}' >&2\nprintf '%s' '{STOPPED}'"); let engine = FakeEngine::new("client-forward-sink", true, &body); - let sink = SharedSink::default(); + let forwarded = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let sink = { + let forwarded = std::sync::Arc::clone(&forwarded); + FnSink(move |bytes: &[u8]| { + forwarded.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + }) + }; let answer = run_engine_stdio_forwarding( engine.path(), ["stdio"], &EngineStdioOptions::default(), &stop_request(), - Some(sink.clone()), + Some(sink), ) .unwrap(); assert!(answer.ok); - assert_eq!(sink.0.lock().unwrap().as_slice(), progress.as_bytes()); + assert_eq!(forwarded.lock().unwrap().as_slice(), progress.as_bytes()); } /// The public entry point sends the child's stderr to THIS process's @@ -1136,12 +1200,7 @@ mod tests { assert_eq!( stderr.contains(LINE), expected, - "forward_stderr {arm}: the child's stderr {} this process's stderr:\n{stderr}", - if expected { - "must reach" - } else { - "must not reach" - } + "forward_stderr {arm}: the child's stderr on this process's stderr:\n{stderr}" ); } } @@ -1239,16 +1298,6 @@ mod tests { Ok(line.len()) } } - struct Sink(Seen); - impl Write for Sink { - fn write(&mut self, bytes: &[u8]) -> io::Result { - self.0.borrow_mut().push(("write", sigpipe_is_ignored())); - Ok(bytes.len()) - } - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } - } if std::env::var_os("ROCM_TEST_CLIENT_GUARD_SCOPE_ARM").is_some() { // SAFETY: installing a signal disposition is always safe. @@ -1260,7 +1309,12 @@ mod tests { lines: vec![b"one\n", b"two\n", b"three\n"], seen: Rc::clone(&seen), }; - collect_stderr(source, Some(Sink(Rc::clone(&seen)))); + let sink_seen = Rc::clone(&seen); + let sink = FnSink(move |bytes: &[u8]| { + sink_seen.borrow_mut().push(("write", sigpipe_is_ignored())); + Ok(bytes.len()) + }); + collect_stderr(source, Some(sink)); let seen = seen.borrow(); let writes = seen.iter().filter(|(what, _)| *what == "write").count(); assert_eq!(writes, 3, "{seen:?}"); From 80db099bbf60f7219b6fb369d70d6cabadfd8535 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 16:11:21 +0000 Subject: [PATCH 23/38] docs(engine-protocol): state that an answer printed before a failed request write is not read If the engine answers and exits without draining a request larger than a pipe buffer, the write fails and the child is killed with its stdout unread, so the error carries its stderr and not its answer, which departs from the rule that an answer on stdout wins over the exit status. Every engine here (lemonade, vllm, aim, and the rocm __engine-stdio verb that serves the built-in ones) reads stdin to the end before it answers, so reaching this takes an engine that breaks the protocol. Decoding such an answer would mean waiting, up to a deadline, for the child to finish printing it and to exit before deciding it is complete, which is not worth doing for an engine that has stopped reading. The module comment and docs/testing.md now state the limit. Signed-off-by: Jussi Elo --- crates/rocm-engine-protocol/src/stdio_client.rs | 11 +++++++++++ docs/testing.md | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index d7638e19fb..3653fa1b4e 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -35,6 +35,17 @@ //! itself, for the stderr of a child whose request could not be written, is //! bounded by `EXITED_STDERR_GRACE`. //! +//! A request that cannot be written is the other place the answer-wins rule does +//! not apply: if the engine answered and exited without draining a request larger +//! than a pipe buffer, the write fails, the child is killed and its stdout is +//! never read, so the error carries its stderr and not its answer. Every engine +//! here (`lemonade`, `vllm`, `aim`, and the `rocm __engine-stdio` verb that serves +//! the built-in ones) reads stdin to the end before it answers, so reaching this +//! needs an engine that breaks the protocol. Decoding such an answer would mean +//! waiting, up to a deadline, for the child to finish printing it and to exit +//! before deciding it is complete, and an engine that has stopped reading is no +//! engine to wait for, so it is left as a documented limit. +//! //! Forwarding the child's stderr guards each write against SIGPIPE on its own //! (see `forward_best_effort`), and the guard is held for as long as that write //! blocks. If the sink is stalled rather than closed (a terminal that has stopped diff --git a/docs/testing.md b/docs/testing.md index 091b3a49c5..7f53d9e2a9 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -285,7 +285,7 @@ runs the whole client against a fake that exits without reading; `the_stderr_of_a_failed_write_is_not_waited_for_past_the_grace` one whose grandchild keeps the pipe open. The last two run the step on a real child with an injected write error, because whether a real write fails before the child has -closed its stdin is a race. The bound compares the deadline with the clock +closed its stdin is a race. The answer of an engine that exits without draining a request larger than a pipe buffer is not read on this path (every engine here reads stdin to the end before answering, so that needs one that breaks the protocol); the module comment in `stdio_client.rs` states the limit. The bound compares the deadline with the clock before every receive and receives nothing once it has passed, because a receive with an expired timeout still returns output that is already queued, which would let a backlog or a descendant that keeps writing hold the error path past it; From 2dd24d1a8aa6b2f27bffc5779c0a7494a8920923 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 16:14:40 +0000 Subject: [PATCH 24/38] style(rocm): drop a redundant clone in the environment test clippy's redundant_clone, denied in CI, flags the app paths cloned for the flipper thread of the new environment test, which does not use them afterwards. Behaviour is unchanged. Signed-off-by: Jussi Elo --- apps/rocm/src/engine_ipc.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index 8be6929242..233dc59671 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -557,7 +557,6 @@ mod tests { let done = Arc::new(AtomicBool::new(false)); let flipper = { let done = Arc::clone(&done); - let paths = paths.clone(); std::thread::spawn(move || { while !done.load(Ordering::Relaxed) { with_scoped_builtin_engine_env(Some(&paths), None, || { From 042f8cc1d9776b7fd1de1b90bcec977547bca066 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 16:37:36 +0000 Subject: [PATCH 25/38] docs(core): say what the SIGPIPE guard's restore can and cannot round-trip The guard reads and puts back the disposition with signal(), which reports only the handler. A disposition installed with sigaction() that carries flags or a signal mask would come back without them. This process only ever sets SIG_DFL (rocm at startup) and SIG_IGN (the guard) for SIGPIPE, so the doc comment now says so instead of claiming the previous disposition is restored in general. Signed-off-by: Jussi Elo --- crates/rocm-core/src/sigpipe.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/crates/rocm-core/src/sigpipe.rs b/crates/rocm-core/src/sigpipe.rs index 68192204b1..2a28b0a223 100644 --- a/crates/rocm-core/src/sigpipe.rs +++ b/crates/rocm-core/src/sigpipe.rs @@ -12,7 +12,7 @@ //! the child exited, so code that writes to a child's stdin wraps the write in //! [`with_sigpipe_ignored`]. -/// Run `f` with SIGPIPE ignored, restoring the previous disposition afterwards. +/// Run `f` with SIGPIPE ignored, putting the previous disposition back afterwards. /// /// A write that reaches a closed pipe inside `f` then returns an `EPIPE` error /// instead of delivering a signal. Where the disposition was already ignored @@ -20,6 +20,12 @@ /// process-wide, so keep `f` to the write itself. Windows has no SIGPIPE and /// runs `f` as is. /// +/// The disposition is read and put back with `signal()`, which reports only the +/// handler (`SIG_DFL`, `SIG_IGN` or a function address). One that was installed +/// with `sigaction()` and carries flags or a signal mask would come back without +/// them. That is not a case here: this process only ever sets `SIG_DFL` (`rocm` +/// at startup) and `SIG_IGN` (this guard) for SIGPIPE. +/// /// Guarded regions may overlap, on one thread or on several (the stderr /// forwarder of one engine request runs while another request writes to its /// child). The disposition in force before the first region is remembered once From 813357af6a5580517188816b6e29b49e39ad91e1 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 16:38:59 +0000 Subject: [PATCH 26/38] refactor(core): name the three ROCM_CLI_*_DIR variables once AppPaths::env_vars and AppPaths::discover each spelled the ROCM_CLI_CONFIG_DIR, ROCM_CLI_DATA_DIR and ROCM_CLI_CACHE_DIR names as literals, with a test that scanned the source text to keep them in step, unlike ENGINE_ENVS_ROOT_ENV which is a constant. Each name is now one constant (CONFIG_DIR_ENV, DATA_DIR_ENV, CACHE_DIR_ENV) used by both, so they cannot disagree, and the source-text scan is gone. The test that remains pins the public names against the paths they carry, since a child process and a user's shell see those spellings. No behaviour change. Signed-off-by: Jussi Elo --- crates/rocm-core/src/lib.rs | 4 +++- crates/rocm-core/src/paths.rs | 34 ++++++++++++++++++---------------- docs/architecture.md | 2 +- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 7ea509d427..a2dd5d57ca 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -176,7 +176,9 @@ pub use net::{ http_get_text_with_auth, http_get_with_auth, http_post_json_with_auth, parse_http_endpoint, read_http_response_bounded, write_all_tcp_stream, }; -pub use paths::{AppPaths, ENGINE_ENVS_ROOT_ENV, engine_plugin_dirs}; +pub use paths::{ + AppPaths, CACHE_DIR_ENV, CONFIG_DIR_ENV, DATA_DIR_ENV, ENGINE_ENVS_ROOT_ENV, engine_plugin_dirs, +}; pub use proc_lifecycle::{ IdentityState, KillScope, ProcessIdentity, TerminationOutcome, identity_state, process_start_ticks, terminate_verified, diff --git a/crates/rocm-core/src/paths.rs b/crates/rocm-core/src/paths.rs index 6a1a6473c7..80810bbe71 100644 --- a/crates/rocm-core/src/paths.rs +++ b/crates/rocm-core/src/paths.rs @@ -39,6 +39,14 @@ pub struct AppPaths { /// that must install into a particular root. pub const ENGINE_ENVS_ROOT_ENV: &str = "ROCM_CLI_ENGINE_ENVS_ROOT"; +/// The variable that moves the config root. +pub const CONFIG_DIR_ENV: &str = "ROCM_CLI_CONFIG_DIR"; +/// The variable that moves the data root (and, unless it is set too, the cache +/// root that follows it). +pub const DATA_DIR_ENV: &str = "ROCM_CLI_DATA_DIR"; +/// The variable that moves the cache root. +pub const CACHE_DIR_ENV: &str = "ROCM_CLI_CACHE_DIR"; + impl AppPaths { /// The three `ROCM_CLI_*_DIR` variables that make a child process run on /// these roots, in the order config, data, cache. [`Self::discover`] reads @@ -46,17 +54,17 @@ impl AppPaths { /// parent was given. pub fn env_vars(&self) -> [(&'static str, &Path); 3] { [ - ("ROCM_CLI_CONFIG_DIR", self.config_dir.as_path()), - ("ROCM_CLI_DATA_DIR", self.data_dir.as_path()), - ("ROCM_CLI_CACHE_DIR", self.cache_dir.as_path()), + (CONFIG_DIR_ENV, self.config_dir.as_path()), + (DATA_DIR_ENV, self.data_dir.as_path()), + (CACHE_DIR_ENV, self.cache_dir.as_path()), ] } pub fn discover() -> Result { - let data_dir_override = env_path_override("ROCM_CLI_DATA_DIR"); - let cache_dir_override = env_path_override("ROCM_CLI_CACHE_DIR"); + let data_dir_override = env_path_override(DATA_DIR_ENV); + let cache_dir_override = env_path_override(CACHE_DIR_ENV); let paths = Self::from_overrides(PathOverrides { - config: env_path_override("ROCM_CLI_CONFIG_DIR"), + config: env_path_override(CONFIG_DIR_ENV), data: data_dir_override.clone(), cache: cache_dir_override.clone(), })?; @@ -375,8 +383,11 @@ mod tests { use crate::test_support::temp_app_paths; + /// `env_vars` and `discover` share one constant per name, so they cannot + /// disagree; the names themselves are what a child process and a user's + /// shell see, so they are pinned here. #[test] - fn env_vars_name_the_three_roots_discover_reads_back() { + fn env_vars_pair_each_root_with_its_public_variable_name() { let (_root, paths) = temp_app_paths("env-vars"); assert_eq!( paths.env_vars(), @@ -386,15 +397,6 @@ mod tests { ("ROCM_CLI_CACHE_DIR", paths.cache_dir.as_path()), ] ); - // `discover` reads these same names; a rename in one place must not - // leave the other behind. - let source = include_str!("paths.rs"); - for (name, _) in paths.env_vars() { - assert!( - source.contains(&format!("env_path_override(\"{name}\")")), - "{name} is not read by AppPaths::discover" - ); - } } #[test] diff --git a/docs/architecture.md b/docs/architecture.md index dd52dd21a9..11abf9da89 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -201,7 +201,7 @@ Already-extracted subsystem modules, following the same full-domain-extraction p - [`model_catalog.rs`](../crates/rocm-core/src/model_catalog.rs) — (ROCMAI-152, Phase 7c) the curated model catalog's types, built-in catalog JSON loading, signed external recipe indexes, and artifact cache-marker bookkeeping — see the module's own doc comment for what it owns. - [`model_readiness.rs`](../crates/rocm-core/src/model_readiness.rs) — the `rocm diagnose --model` fit assessment: curated-recipe lookup against a host's measured GPU/RAM. `rocm model --verbose` renders its own fit labels in [`model_catalog_text.rs`](../apps/rocm/src/model_catalog_text.rs), which uses exactly two items from this module: `curated_alternatives`, called directly, and `recipe_display_ref`, reached through a private one-line wrapper of the same name in that file. For a GGUF outside the catalog it also attaches a context-size estimate (the three `gguf*` modules above). - [`net.rs`](../crates/rocm-core/src/net.rs) — (ROCMAI-154) networking primitives: the default local host and port, URL/endpoint formatting and parsing, and the hand-rolled loopback HTTP/TCP client (`http_get_text*`, `http_post_json_with_auth`, `read_http_response_bounded`). It depends on no other service module: the managed-service readiness probes built on it live in [`endpoint_readiness.rs`](../crates/rocm-core/src/endpoint_readiness.rs) and the streaming downloads in [`download.rs`](../crates/rocm-core/src/download.rs). -- [`paths.rs`](../crates/rocm-core/src/paths.rs) — `AppPaths` and its filesystem-location primitives, including `AppPaths::env_vars`, the three `ROCM_CLI_*_DIR` variables that make a child process run on the same roots, and `ENGINE_ENVS_ROOT_ENV`. +- [`paths.rs`](../crates/rocm-core/src/paths.rs) — `AppPaths` and its filesystem-location primitives, including `AppPaths::env_vars`, the three `ROCM_CLI_*_DIR` variables that make a child process run on the same roots, and the constants that name the variables (`CONFIG_DIR_ENV`, `DATA_DIR_ENV`, `CACHE_DIR_ENV`, `ENGINE_ENVS_ROOT_ENV`). - [`placeholder.rs`](../crates/rocm-core/src/placeholder.rs) — the `` placeholder rendered reports print for an empty value: `empty_as_unknown` (returns the value untouched) and `trimmed_or_unknown` (trims first), shared by `apps/rocm`'s driver-install plan and chat tool summaries and by [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs)'s Windows GPU diagnostics, replacing the copy those two `apps/rocm` callers shared in [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) and `host_gpu.rs`'s own trimming copy. - [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs) — the host probes' subprocess and file-read helpers, split out of [`examine.rs`](../crates/rocm-core/src/examine.rs) so `fix.rs`, `process.rs` and the child modules of [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) reach them without importing the module that owns the `Examination` document: `run`/`run_with_env`/`run_raw` (run a command under a timeout; `rc` is `127` when it cannot be spawned and `124` on timeout), `run_utf16le`/`decode_utf16le` (for `wsl.exe`'s UTF-16LE output), `read_text`/`read_host_text`, `which`, the `SHORT`/`MEDIUM` timeouts and `truncate_to_chars`. All are `pub(crate)` except `run_raw`, which stays private to the module. The module itself is private to the crate and is not re-exported. - [`process.rs`](../crates/rocm-core/src/process.rs) — (ROCMAI-154, Phase 7b) process-spawn/terminate primitives, including `FileLock`, plus `run_with_timeout`: it drains both pipes on their own threads from the moment the child starts, so output size cannot stall a command into its own timeout, and it bounds how much of a timed-out child's output the error quotes, because that text reaches persisted automation logs. It is not yet the *only* such helper — [`therock/process.rs`](../apps/rocm/src/therock/process.rs)'s `wait_with_output_bounded`, [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs)'s `run_raw`, and [`host_gpu/capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs)'s `capture_optional_command_candidate_with_timeout` (plus the Windows-only `capture_diagnostic_command` in [`host_gpu/diagnostics.rs`](../crates/rocm-core/src/host_gpu/diagnostics.rs)) still join their reader threads with no time limit, so a grandchild holding the pipe can outlast their timeout; converging them onto `run_with_timeout` is follow-up work to ROCMAI-448 tracked in [#623](https://github.com/ROCm/rocm-cli/issues/623). From d28902e32e219d6b7adcda21a0b3ca73332d7db3 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 16:43:54 +0000 Subject: [PATCH 27/38] test(e2e): assert service-stop-02's whole engine answer and fix the stale comment above it The comment above service-stop-02 said no engine is involved, which stopped being true when the scenario gained a step asserting the daemon's engine Stop request was answered. It now says what is and is not involved. That step only checked that the engine's answer carried a boolean stopped. It now asserts the whole answer. The answer is stopped false, and that is the right one rather than a gap: the planted record names no engine state with a process id, so the real vllm engine has nothing it can confirm stopping (its stop_service answers false when no PID is recorded), while the recorded supervisor, which the daemon signals itself, is what the other steps check. Changing the expectation to true turns the scenario red. Signed-off-by: Jussi Elo --- tests/e2e-cucumber/features/service_stop.feature | 6 +++++- tests/e2e-cucumber/tests/e2e/service_stop_steps.rs | 13 ++++++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/tests/e2e-cucumber/features/service_stop.feature b/tests/e2e-cucumber/features/service_stop.feature index 61edcb0009..b32ef7e933 100644 --- a/tests/e2e-cucumber/features/service_stop.feature +++ b/tests/e2e-cucumber/features/service_stop.feature @@ -41,7 +41,11 @@ Feature: Stopping a managed service # # The recorded process is a real one the scenario starts, recorded with the # kernel's own start-time token for it, so the stop's identity check confirms - # it and goes on to terminate it. No GPU and no engine are involved. + # it and goes on to terminate it. No GPU and no running engine are involved: the + # daemon does ask the engine to stop (a real `vllm` engine, reached through the + # `rocm` beside the standalone `rocmd`), and since the planted record names no + # engine process the engine answers that it stopped nothing, which the scenario + # checks along with the report. # # Linux-only: the token is read from `/proc`, which is where `rocmd` reads it # too; without one the identity check is not exercised at all. diff --git a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs index 250774a8f3..32153a060d 100644 --- a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs @@ -399,6 +399,13 @@ async fn tool_reports_stopped(world: &mut E2eWorld) { /// no engine verb of its own), so its `Stop` request is answered rather than /// failing with the standalone binary's "unrecognized subcommand". The report /// carries the engine's answer or the error that took its place. +/// +/// The answer is `stopped: false`, and that is the right one here, not a gap: the +/// planted record names no engine state with a process id, so the real `vllm` +/// engine has nothing it can confirm stopping (`stop_service` answers `false` +/// when no PID is recorded). The recorded supervisor, which the daemon itself +/// signals, is what the other steps check. Asserting the whole answer, not just +/// that it is a boolean, is what tells it from an engine that was never reached. #[then("the tool asked the engine to stop and the engine answered")] async fn engine_answered_the_stop(world: &mut E2eWorld) { let stdout = world.cli_output.as_deref().unwrap_or_default(); @@ -410,7 +417,11 @@ async fn engine_answered_the_stop(world: &mut E2eWorld) { engine_stop.get("error").is_none(), "the engine's answer was an error instead: {report}" ); - assert!(engine_stop["stopped"].is_boolean(), "{report}"); + assert_eq!( + *engine_stop, + serde_json::json!({"attempted": true, "stopped": false, "graceful": false}), + "{report}" + ); } /// The stop report `rocmd sandbox-tool stop_server` printed, with the tool's own From afd6d79ec9836405750daa1008d02e7a02905982 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 16:44:16 +0000 Subject: [PATCH 28/38] docs(engine-protocol): name the ok:true case of an answer followed by a failing exit The doc comment of run_engine_stdio said an error envelope followed by a failing exit is read as that envelope. It now says the same holds for a success envelope: the child is reported as the success it answered, and the failing exit is neither reported nor logged, by design. No behaviour change; the existing test a_nonzero_exit_that_still_answered_is_decoded_not_discarded covers both envelopes. Signed-off-by: Jussi Elo --- crates/rocm-engine-protocol/src/stdio_client.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 3653fa1b4e..346f33ef74 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -151,7 +151,11 @@ where /// /// A failing exit status is an error only when the child printed no answer: an /// engine that reports its own failure in an error envelope and then exits -/// non-zero is read as that envelope, which says more than the status does. +/// non-zero is read as that envelope, which says more than the status does. The +/// same holds for a success envelope (`ok: true`): a child that answers +/// successfully and then exits non-zero is reported as the success it answered, +/// and the failing exit is not reported or logged anywhere, by design, because +/// the answer is what the caller asked for. /// Every other failure names what went wrong and carries the child's stderr. pub fn run_engine_stdio( program: &Path, From 95f8c243ec1accaf67078640b27ae06d169fa409 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 19:43:54 +0000 Subject: [PATCH 29/38] test(xtask): fail when a second engine stdio client appears The shared stdio client replaced the two copies rocm and rocmd carried, and the behavioural tests of the client and of each caller catch a copy that behaves differently. None of them can see that a second copy exists, so a spawn-write-read helper pasted back into a binary would pass as long as it behaved the same, and would then be free to drift again. A static contract test, in the style of the amd-smi launch contract, scans every Rust file in the tree with comments stripped and fails when a file other than the shared client both pipes a child's stdin and names an engine request or response envelope, which is what an engine stdio client is made of. A caller of run_engine_stdio has neither. The allowlist is bidirectional, each entry carries a reason, and the tests include a negative fixture of the old copy's shape, comment-only mentions, callers and the serving side, which must not match. Pasting the old rocmd copy back into apps/rocmd/src/engine_ipc.rs turns the scan red by name and also turns five of rocmd's behavioural pinning tests red (app paths, crash, unparseable answer, nonzero exit with an answer, SIGPIPE). docs/testing.md describes the check and what it cannot see. Signed-off-by: Jussi Elo --- docs/testing.md | 30 ++- xtask/src/engine_stdio_client_contract.rs | 278 ++++++++++++++++++++++ xtask/src/main.rs | 1 + 3 files changed, 302 insertions(+), 7 deletions(-) create mode 100644 xtask/src/engine_stdio_client_contract.rs diff --git a/docs/testing.md b/docs/testing.md index 05d084189c..38aa720de3 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -221,13 +221,29 @@ choose: the program and verb, the app paths, and whether stderr is forwarded (se `apps/rocm/src/engine_ipc.rs` and `apps/rocmd/src/engine_ipc.rs`). These tests catch behavioural drift: a caller whose own copy of the conversation behaved differently from the shared client in what they pin (the framing, the -environment, the exit and answer handling, the guards) would fail them, but a -copy that behaves the same passes them, so nothing here rules out a second -copy existing. A structural check would have to read the callers' files from -outside this crate, a cross-crate read that `cargo xtask prepush` only accepts -with an entry in its widening rule -(`every_cross_crate_read_is_covered_by_the_widening_rule`), which is not worth -adding for this, so the guarantee is for behaviour only. +environment, the exit and answer handling, the guards) would fail them. A copy +that behaves the same passes them, so a second guard is on the existence of a +second copy: +[engine_stdio_client_contract.rs](../xtask/src/engine_stdio_client_contract.rs) +is a static text scan over every `.rs` file in the tree (comments stripped) that +fails when a file other than the shared client both pipes a child's stdin and +names an engine request or response envelope, which is what an engine stdio +client is made of. A caller that reaches an engine through `run_engine_stdio` +has neither. Pasting the old rocmd copy back into `apps/rocmd/src/engine_ipc.rs` +fails it by name (and, separately, five of rocmd's pinning tests: the +app-paths, crash, unparseable-answer, nonzero-with-answer and SIGPIPE ones). Like +the other allowlists it is bidirectional, so an exemption cannot outlive its +reason. + +```bash +cargo test -p xtask engine_stdio_client_contract +``` + +It is a source-text scan, not a call-graph analysis: a copy that makes its pipe +by hand, or reaches the envelope types under another name, is outside what it +can see, and it does not check that a caller uses the shared client correctly +(the callers' tests do). It closes the cheap, likely regression of a +spawn-write-read helper pasted back into a binary. An answer that is not a response envelope is reported as that, and when the child also exited with a failing status the error says so, because the status is diff --git a/xtask/src/engine_stdio_client_contract.rs b/xtask/src/engine_stdio_client_contract.rs new file mode 100644 index 0000000000..81f2709a12 --- /dev/null +++ b/xtask/src/engine_stdio_client_contract.rs @@ -0,0 +1,278 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Static contract test: only the shared client may send an engine child its +//! request. +//! +//! `rocm` and `rocmd` used to carry their own copies of the JSON-over-stdio +//! request to an engine process, and the two drifted (different environment, +//! different treatment of a failing exit, a SIGPIPE guard in only one). The +//! single client in `crates/rocm-engine-protocol/src/stdio_client.rs` replaced +//! both. The behavioural tests of that client and of each caller catch a copy +//! that *behaves* differently; none of them can see that a second copy exists, +//! because the regression is a new function nobody calls from a test. +//! +//! So this guard is on the ingredient. A client has to put a request on a +//! child's stdin, which takes a piped stdin, and the request is an engine +//! envelope, so a file that does both is a client. [`CLIENT`] is the only file +//! allowed to; a caller reaches an engine through `run_engine_stdio` and has +//! neither a piped stdin nor a need to name an envelope for a child. A second +//! client has to either use the shared one or add itself to [`ALLOWED`] in a +//! reviewed diff that says why, as for [`crate::crate_edges::ALLOWLIST`]. +//! +//! What this does **not** claim: it is a source-text scan, not a call-graph +//! analysis. A copy that builds its stdin another way (a pipe made by hand), or +//! reaches the envelope types under another name, is outside what it can see, +//! and it says nothing about whether a caller then uses the shared client +//! correctly (the callers' own tests pin that). It closes the cheap, likely +//! regression: somebody pasting a spawn-write-read helper back into a binary. +//! +//! Bidirectional, like the other allowlists: an entry that no longer matches +//! fails too, so the exemption does not outlive its reason. + +#[cfg(test)] +mod tests { + use std::path::{Path, PathBuf}; + + /// The one file that is the client. + const CLIENT: &str = "crates/rocm-engine-protocol/src/stdio_client.rs"; + + /// Every file that may both pipe a child's stdin and name an engine + /// envelope, with the reason it is allowed to. + const ALLOWED: &[(&str, &str)] = &[( + CLIENT, + "the shared client: it spawns the engine child, writes the request line \ + to its stdin and reads the answer", + )]; + + /// This file, which the scan must skip: its fixtures quote both ingredients + /// as data, and a text scan cannot tell a fixture string from code. + const SELF_PATH: &str = "xtask/src/engine_stdio_client_contract.rs"; + + /// What names an engine envelope: the request and response types, or the + /// helper that builds a request. + const ENVELOPE_TOKENS: &[&str] = &[ + "EngineRequestEnvelope", + "EngineResponseEnvelope", + "engine_request_envelope", + ]; + + fn repo_root() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .expect("xtask crate has a parent directory") + .to_path_buf() + } + + /// Every `.rs` file in the repository as repo-relative, `/`-separated + /// paths. `target/` and dot-directories are skipped so the verdict never + /// depends on state outside the commit under test (the same two exclusions + /// as the other contract tests here). + fn workspace_rust_sources(root: &Path) -> Vec { + let mut found = Vec::new(); + collect_rust_files(root, root, &mut found); + assert!( + !found.is_empty(), + "found no Rust sources to scan -- the walk is broken, not the tree" + ); + found.sort(); + found + } + + fn collect_rust_files(root: &Path, dir: &Path, found: &mut Vec) { + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + let skip = path + .file_name() + .and_then(|name| name.to_str()) + .is_none_or(|name| name == "target" || name.starts_with('.')); + if skip { + continue; + } + collect_rust_files(root, &path, found); + } else if path.extension().is_some_and(|ext| ext == "rs") + && let Ok(relative) = path.strip_prefix(root) + { + found.push(relative.to_string_lossy().replace('\\', "/")); + } + } + } + + /// `line` with any `//` comment tail removed, so prose that names the + /// ingredients is not read as code. Crude on purpose (it does not track + /// string literals): that fails open only for a pathological line. + fn strip_comment(line: &str) -> &str { + match line.find("//") { + Some(at) => &line[..at], + None => line, + } + } + + /// Whether `text` pipes a child's stdin *and* names an engine envelope, + /// ignoring comments: the two ingredients of an engine stdio client. + fn is_a_client(text: &str) -> bool { + let code: Vec<&str> = text.lines().map(strip_comment).collect(); + let pipes_stdin = code + .iter() + .any(|line| line.contains(".stdin(") && line.contains("piped()")); + let names_envelope = code + .iter() + .any(|line| ENVELOPE_TOKENS.iter().any(|token| line.contains(token))); + pipes_stdin && names_envelope + } + + fn client_files(root: &Path) -> Vec { + workspace_rust_sources(root) + .into_iter() + .filter(|relative| relative != SELF_PATH) + .filter(|relative| { + std::fs::read_to_string(root.join(relative)).is_ok_and(|text| is_a_client(&text)) + }) + .collect() + } + + /// Check a set of client files against [`ALLOWED`] in both directions. + fn check(clients: &[String]) -> Vec { + let unexpected = clients + .iter() + .filter(|file| !ALLOWED.iter().any(|(allowed, _)| allowed == file)) + .map(|file| { + format!( + "`{file}` pipes a child's stdin and names an engine envelope, which is an \ + engine stdio client. Use `rocm_engine_protocol::run_engine_stdio` (in \ + {CLIENT}), or add `{file}` to ALLOWED in \ + xtask/src/engine_stdio_client_contract.rs with the reason a second client \ + is needed" + ) + }); + let stale = ALLOWED + .iter() + .filter(|(allowed, _)| !clients.iter().any(|file| file == allowed)) + .map(|(allowed, _)| { + format!( + "ALLOWED entry `{allowed}` is stale: it no longer pipes a child's stdin and \ + names an engine envelope. Remove it, or the exemption outlives its reason" + ) + }); + unexpected.chain(stale).collect() + } + + #[test] + fn there_is_only_one_engine_stdio_client() { + let violations = check(&client_files(&repo_root())); + assert!( + violations.is_empty(), + "rocm and rocmd share one engine stdio client; a second copy can drift from it \ + (ROCMAI-875):\n{}", + violations.join("\n") + ); + } + + #[test] + fn every_allowed_client_carries_a_reason() { + for (file, reason) in ALLOWED { + assert!( + reason.len() > 40, + "ALLOWED entry `{file}` has no real justification: {reason:?}" + ); + } + } + + /// The shape the two replaced copies had: spawn with a piped stdin, write an + /// envelope to it, read the answer back. + const OLD_STYLE_COPY: &str = " +fn engine_request(engine_binary: &Path, envelope: &EngineRequestEnvelope) -> Result<()> { + let mut command = ProcessCommand::new(engine_binary); + command + .arg(\"__engine-stdio\") + .stdin(Stdio::piped()) + .stdout(Stdio::piped()); + let mut child = command.spawn()?; + serde_json::to_writer(child.stdin.as_mut().unwrap(), envelope)?; + let output = child.wait_with_output()?; + let _answer: EngineResponseEnvelope = serde_json::from_slice(&output.stdout)?; + Ok(()) +} +"; + + #[test] + fn a_pasted_back_copy_of_the_old_client_is_recognised_and_reported() { + assert!(is_a_client(OLD_STYLE_COPY)); + let mut clients: Vec = ALLOWED.iter().map(|(file, _)| (*file).to_owned()).collect(); + clients.push("apps/rocmd/src/engine_ipc.rs".to_owned()); + let violations = check(&clients); + assert_eq!(violations.len(), 1, "{violations:?}"); + assert!(violations[0].contains("apps/rocmd/src/engine_ipc.rs")); + assert!(violations[0].contains("run_engine_stdio")); + } + + #[test] + fn a_caller_of_the_shared_client_is_not_a_client() { + let caller = " +fn engine_request(engine_binary: &Path, envelope: &EngineRequestEnvelope) -> Result { + decode_engine_response(run_engine_stdio(engine_binary, [\"stdio\"], &options, envelope)?) +} +"; + assert!(!is_a_client(caller)); + // Piping stdin for something that is not an engine request is fine. + let unrelated = "let child = Command::new(\"tar\").stdin(Stdio::piped()).spawn()?;"; + assert!(!is_a_client(unrelated)); + // Naming the envelope without a child is fine too (the serving side). + let serving = "fn handle(envelope: EngineRequestEnvelope) -> EngineResponseEnvelope {}"; + assert!(!is_a_client(serving)); + } + + #[test] + fn a_mention_in_a_comment_is_not_a_client() { + let prose = " +// builds an EngineRequestEnvelope and pipes it with .stdin(Stdio::piped()) +/// see EngineResponseEnvelope; the child gets `.stdin(Stdio::piped())` +fn nothing() {} +"; + assert!(!is_a_client(prose)); + } + + #[test] + fn the_client_that_is_allowed_stops_being_allowed_if_it_stops_matching() { + let violations = check(&[]); + assert_eq!(violations.len(), 1); + assert!(violations[0].contains("is stale")); + assert!(violations[0].contains(CLIENT)); + } + + #[test] + fn the_self_exclusion_names_a_file_that_exists_and_would_otherwise_be_reported() { + // A rename would leave SELF_PATH pointing at nothing, and the exclusion + // would silently stop applying. + let root = repo_root(); + let text = std::fs::read_to_string(root.join(SELF_PATH)) + .unwrap_or_else(|e| panic!("SELF_PATH `{SELF_PATH}` does not name a real file: {e}")); + assert!( + text.contains("OLD_STYLE_COPY"), + "`{SELF_PATH}` no longer holds the fixture the exclusion exists for" + ); + } + + #[test] + fn the_walk_reaches_the_client_and_both_callers() { + // Without this the contract could pass by scanning nothing. + let files = workspace_rust_sources(&repo_root()); + for expected in [ + CLIENT, + "apps/rocm/src/engine_ipc.rs", + "apps/rocmd/src/engine_ipc.rs", + "xtask/src/engine_stdio_client_contract.rs", + ] { + assert!( + files.iter().any(|file| file == expected), + "the source walk missed `{expected}`, so the contract is scanning less than \ + the tree" + ); + } + } +} diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 404facf47e..6fe9a79a19 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -22,6 +22,7 @@ mod demos; mod e2e; mod e2e_prewarm; mod e2e_report; +mod engine_stdio_client_contract; mod env_mutation_contract; mod eval_fixtures; mod lychee; From f07a6b560e53dba7b94f915838d23d227706f05a Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 19:51:19 +0000 Subject: [PATCH 30/38] fix(engine-protocol): read the child's stderr from the start so a full stderr cannot deadlock a large request The stderr collector started only after the whole request had been written, the order both replaced clients had. An engine that fills its stderr pipe before it drains a request larger than a pipe buffer then blocks on that write while the client blocks on its own, and neither side reads. The collector now starts, on a thread of its own, right after the child is spawned, and the request is written while it runs. StderrCollector appends each line to a shared buffer as it arrives and is waited for with one optional deadline: the success path waits for the end of the stderr as before, and the failed-write path waits at most its short grace and reports what had arrived, which replaces the chunk-by-chunk receive loop and its helper (a single receive on a completion channel has no queue to drain past an expired deadline). The module comment now lists the three deliberate tradeoffs together (the unbounded wait on pipes held by a grandchild, the answer of an engine that exits without draining a request not being decoded, SIGPIPE staying ignored while a stalled stderr forward blocks) and says why only the stderr half of the first would be cheap to bound. A test runs the client against a fake that writes 300 kB to stderr before reading a 1 MiB request; it deadlocks and fails after 20 seconds when the collector is started after the write. Two tests cover the collector's deadline, and docs/testing.md describes them. Signed-off-by: Jussi Elo --- .../rocm-engine-protocol/src/stdio_client.rs | 307 ++++++++++-------- docs/testing.md | 21 +- 2 files changed, 193 insertions(+), 135 deletions(-) diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 346f33ef74..4093a1cb10 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -56,12 +56,25 @@ //! sink, not just a quiet engine. It is accepted: a guard that could be taken //! back mid-write does not exist, and without one a closed sink would end //! `rocm` before the request could report why. +//! +//! These three are deliberate tradeoffs, not oversights: the wait on pipes held +//! by a grandchild is unbounded, an answer from an engine that exits without +//! draining a large request is not decoded, and SIGPIPE stays ignored +//! process-wide while a stalled stderr forward blocks. Only the first is +//! partly cheap to bound: the stderr side would be one deadline on +//! `StderrCollector::finish`, but stdout is read to its end on the calling thread, +//! so a bound there needs a second reader thread and a decision on what to report +//! when it fires, and bounding one pipe and not the other would be a half +//! measure. The stderr is read from the moment the child starts, on its own +//! thread, so an engine that fills it before draining a large request cannot +//! deadlock the write. use std::ffi::OsStr; use std::io::{self, BufRead, Read, Write}; use std::path::Path; use std::process::{Child, Command, ExitStatus, Stdio}; use std::sync::mpsc; +use std::sync::{Arc, Mutex, PoisonError}; use std::thread; use std::time::{Duration, Instant}; @@ -199,6 +212,14 @@ where .stdin .take() .context("engine stdio child did not expose stdin")?; + // Read the child's stderr from the start, on its own thread: an engine that + // fills the stderr pipe before it has drained a request larger than a pipe + // buffer would otherwise block on that write while this process blocks on + // its own, with nobody reading either side. + let stderr_collector = child + .stderr + .take() + .map(|stderr| StderrCollector::start(stderr, forward)); // Ignore SIGPIPE for the duration of the write. Under the default action, an // engine that exits before reading its stdin would kill this process with // SIGPIPE (`rocm` runs that way, and so does the daemon hosted in it); @@ -210,6 +231,7 @@ where if let Err(error) = written { return Err(abandon_unwritten_request( child, + stderr_collector, &error, EXITED_STDERR_GRACE, )); @@ -217,10 +239,6 @@ where // End of the request: the engine reads to EOF before it answers. drop(stdin); - let stderr_reader = child - .stderr - .take() - .map(|stderr| thread::spawn(move || collect_stderr(stderr, forward))); let stdout_pipe = child .stdout .take() @@ -229,8 +247,8 @@ where let status = child .wait() .context("failed waiting for engine stdio response")?; - let stderr = stderr_reader - .map(|reader| reader.join().unwrap_or_default()) + let stderr = stderr_collector + .map(|collector| collector.finish(None)) .unwrap_or_default(); let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); @@ -294,15 +312,15 @@ const EXITED_STDERR_GRACE: Duration = Duration::from_secs(1); /// and this error path must not block on it. fn abandon_unwritten_request( mut child: Child, + stderr: Option, write_error: &io::Error, grace: Duration, ) -> anyhow::Error { let _ = child.kill(); let _ = child.wait(); - let stderr = child - .stderr - .take() - .map(|stderr| read_available(stderr, grace)) + let deadline = Instant::now() + grace; + let stderr = stderr + .map(|collector| collector.finish(Some(deadline))) .unwrap_or_default(); let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); let mut message = format!("failed to write engine request: {write_error}"); @@ -312,46 +330,55 @@ fn abandon_unwritten_request( anyhow::anyhow!(message) } -/// Read `source` to its end, or for at most `grace` if it does not end, and -/// return what arrived. The reading thread is left to finish by itself when the -/// pipe closes. -fn read_available(mut source: impl Read + Send + 'static, grace: Duration) -> Vec { - let (sender, receiver) = mpsc::channel::>(); - thread::spawn(move || { - let mut buffer = [0_u8; 4096]; - loop { - match source.read(&mut buffer) { - Ok(0) | Err(_) => break, - Ok(n) => { - if sender.send(buffer[..n].to_vec()).is_err() { - break; - } - } - } - } - }); - collect_until(&receiver, Instant::now() + grace) +/// A child's stderr being collected on a thread of its own, from the moment it +/// is started, whatever the main thread is doing meanwhile. +struct StderrCollector { + /// Everything read so far, appended line by line as it arrives. + collected: Arc>>, + /// Receives one message when the stderr has been read to its end. + finished: mpsc::Receiver<()>, } -/// Gather what arrives on `receiver` until it closes or `deadline` passes. -/// -/// The deadline is compared with the clock before every receive, and nothing is -/// received once it has passed: `recv_timeout` with an expired timeout still -/// hands back a message that is already queued, so a backlog, or a descendant -/// that keeps writing, would otherwise keep this collecting past the bound. -fn collect_until(receiver: &mpsc::Receiver>, deadline: Instant) -> Vec { - let mut collected = Vec::new(); - loop { - let now = Instant::now(); - if now >= deadline { - break; +impl StderrCollector { + /// Start reading `stderr`, forwarding each line to `forward` when there is + /// one (see [`collect_stderr_into`]). + fn start( + stderr: impl Read + Send + 'static, + forward: Option, + ) -> Self { + let collected = Arc::new(Mutex::new(Vec::new())); + let (done, finished) = mpsc::channel(); + let sink = Arc::clone(&collected); + thread::spawn(move || { + collect_stderr_into(stderr, forward, &sink); + let _ = done.send(()); + }); + Self { + collected, + finished, } - match receiver.recv_timeout(deadline - now) { - Ok(chunk) => collected.extend(chunk), - Err(_) => break, + } + + /// What was collected, once the stderr has ended, or what has arrived by + /// `deadline` when there is one and the stderr has not ended by then (a + /// grandchild holding the pipe open). With no deadline this waits for the + /// end, which is the unbounded wait the module comment states. + fn finish(&self, deadline: Option) -> Vec { + match deadline { + None => { + let _ = self.finished.recv(); + } + Some(deadline) => { + let _ = self + .finished + .recv_timeout(deadline.saturating_duration_since(Instant::now())); + } } + self.collected + .lock() + .unwrap_or_else(PoisonError::into_inner) + .clone() } - collected } /// Read the child's whole answer from `stdout`. @@ -380,9 +407,23 @@ fn read_response(child: &mut Child, mut stdout: impl Read) -> Result> { /// pipe ends the whole process before `write_all` can return, so each forward /// runs inside [`with_sigpipe_ignored`] (see [`forward_best_effort`]). A read error ends the /// collection and is itself recorded as the last line. -fn collect_stderr(stderr: impl Read, mut forward: Option) -> Vec { +#[cfg(test)] +fn collect_stderr(stderr: impl Read, forward: Option) -> Vec { + let collected = Mutex::new(Vec::new()); + collect_stderr_into(stderr, forward, &collected); + collected + .into_inner() + .unwrap_or_else(PoisonError::into_inner) +} + +/// [`collect_stderr`], appending to `collected` as each line arrives so a caller +/// that stops waiting still has what was read. +fn collect_stderr_into( + stderr: impl Read, + mut forward: Option, + collected: &Mutex>, +) { let mut reader = io::BufReader::new(stderr); - let mut collected = Vec::new(); let mut line = Vec::new(); loop { line.clear(); @@ -392,19 +433,25 @@ fn collect_stderr(stderr: impl Read, mut forward: Option) -> Vec if let Some(sink) = forward.as_mut() { forward_best_effort(sink, &line); } - collected.extend_from_slice(&line); + append(collected, &line); } Err(error) => { let message = format!("failed to read engine progress: {error}\n"); if let Some(sink) = forward.as_mut() { forward_best_effort(sink, message.as_bytes()); } - collected.extend_from_slice(message.as_bytes()); + append(collected, message.as_bytes()); break; } } } +} + +fn append(collected: &Mutex>, bytes: &[u8]) { collected + .lock() + .unwrap_or_else(PoisonError::into_inner) + .extend_from_slice(bytes); } /// Write `bytes` to the forwarding sink and drop the result: a sink that has @@ -615,50 +662,61 @@ mod tests { ); } - /// A source that does not answer until released, then never stops answering, - /// and says when it is dropped. + /// A source that does not answer until released, then answers once and ends. struct Gated { release: mpsc::Receiver<()>, - dropped: mpsc::Sender<()>, released: bool, } impl Read for Gated { fn read(&mut self, buffer: &mut [u8]) -> io::Result { - if !self.released { - self.release.recv().unwrap(); - self.released = true; + if self.released { + return Ok(0); } - buffer[..4].copy_from_slice(b"late"); - Ok(4) - } - } - impl Drop for Gated { - fn drop(&mut self) { - let _ = self.dropped.send(()); + self.release.recv().unwrap(); + self.released = true; + buffer[..5].copy_from_slice(b"late\n"); + Ok(5) } } - /// Once the bounded read has given up, the reading thread that is still - /// blocked is not left spinning: when its source finally answers it finds - /// nobody listening and ends (it drops the source, which the test sees). The - /// source never reaches its end, so a thread that kept reading after the - /// failed send would never end. + /// A collector whose stderr is still open at the deadline reports what had + /// arrived and is not waited for; the reading thread is left alone and the + /// rest is there once the stderr does end. #[test] - fn the_reader_left_behind_by_a_bounded_read_ends_when_nobody_is_listening() { + fn a_collector_stops_waiting_at_its_deadline_and_keeps_collecting() { let (release, released) = mpsc::channel(); - let (dropped, dropped_signal) = mpsc::channel(); - let source = Gated { - release: released, - dropped, - released: false, - }; - let collected = read_available(source, Duration::from_millis(50)); - assert!(collected.is_empty()); + let collector = StderrCollector::start( + Gated { + release: released, + released: false, + }, + None::, + ); + let started = Instant::now(); + let early = collector.finish(Some(Instant::now() + Duration::from_millis(50))); + assert!(early.is_empty()); + assert!(started.elapsed() < Duration::from_secs(5)); release.send(()).unwrap(); - dropped_signal - .recv_timeout(Duration::from_secs(10)) - .expect("the abandoned reader thread did not end"); + assert_eq!(collector.finish(None), b"late\n"); + } + + /// An expired deadline returns at once with what is there, however much is + /// queued behind it. + #[test] + fn an_expired_deadline_returns_what_is_collected_without_waiting() { + let (release, released) = mpsc::channel(); + let collector = StderrCollector::start( + Gated { + release: released, + released: false, + }, + None::, + ); + let started = Instant::now(); + assert!(collector.finish(Some(Instant::now())).is_empty()); + assert!(started.elapsed() < Duration::from_secs(1)); + drop(release); } /// A stderr whose reads fail. @@ -700,55 +758,6 @@ mod tests { ); } - /// Output already queued when the deadline has passed is not collected: - /// `recv_timeout(Duration::ZERO)` would still return it, one message after - /// another, for as long as the queue stays non-empty. - #[test] - fn collecting_stops_at_the_deadline_even_with_output_queued() { - let (sender, receiver) = mpsc::channel::>(); - for _ in 0..100_000 { - sender.send(vec![b'x'; 16]).unwrap(); - } - let started = Instant::now(); - let collected = collect_until(&receiver, Instant::now()); - let after_deadline = collected.len(); - assert_eq!(after_deadline, 0, "collected bytes after the deadline"); - assert!(started.elapsed() < Duration::from_secs(1)); - - // Before the deadline, queued output is collected, and a closed channel - // ends the collection early. - drop(sender); - let collected = collect_until(&receiver, Instant::now() + Duration::from_secs(30)); - assert_eq!(collected.len(), 100_000 * 16); - } - - /// A producer that keeps writing (a descendant holding the pipe) cannot hold - /// the collection past its deadline. A secondary guard: it is timing-based - /// and also passes with the loop that let queued output bypass the deadline, - /// because that producer sleeps between messages. The fix is covered by - /// `collecting_stops_at_the_deadline_even_with_output_queued` above, which is - /// deterministic. - #[test] - fn collecting_returns_at_the_deadline_while_output_keeps_arriving() { - let (sender, receiver) = mpsc::channel::>(); - let producer = thread::spawn(move || { - let until = Instant::now() + Duration::from_secs(3); - while Instant::now() < until { - if sender.send(vec![b'y'; 1024]).is_err() { - break; - } - thread::sleep(Duration::from_millis(1)); - } - }); - let started = Instant::now(); - let collected = collect_until(&receiver, Instant::now() + Duration::from_millis(100)); - let elapsed = started.elapsed(); - drop(receiver); - producer.join().unwrap(); - assert!(elapsed < Duration::from_secs(2), "took {elapsed:?}"); - assert!(!collected.is_empty(), "what arrived in time is kept"); - } - #[test] fn forwarded_stderr_is_written_out_and_still_collected() { let mut shown = Vec::new(); @@ -979,6 +988,37 @@ mod tests { .unwrap_or_else(|_| panic!("did not finish within {limit:?}")) } + /// An engine that fills its stderr before it has drained a request larger + /// than a pipe buffer does not deadlock the client: the stderr is read from + /// the start, so the engine's write completes while the request is still + /// being written. (With the collector started after the write, the engine + /// blocks on a full stderr pipe and the client on a full stdin pipe.) + #[test] + fn an_engine_that_writes_a_lot_of_stderr_before_reading_its_request_does_not_deadlock() { + let engine = FakeEngine::new( + "client-stderr-first", + false, + &format!( + "head -c 300000 /dev/zero | tr '\\0' x >&2\ncat > /dev/null\nprintf '%s' '{STOPPED}'" + ), + ); + let envelope = + engine_request_envelope(EngineMethod::Stop, &json!({"blob": "y".repeat(1 << 20)})) + .unwrap(); + let program = engine.path().to_path_buf(); + let answer = finishes_within(Duration::from_secs(20), move || { + run_engine_stdio( + &program, + ["stdio"], + &EngineStdioOptions::default(), + &envelope, + ) + }) + .expect("the engine answered"); + assert!(answer.ok); + drop(engine); + } + /// A request that cannot be written because the engine is gone reports /// what the engine said on stderr, not only that the write failed: the /// engine's own words are the explanation. The request is larger than a @@ -1021,8 +1061,14 @@ mod tests { assert!(Path::new(&proc_entry).exists(), "the child is running"); let error = finishes_within(Duration::from_secs(10), move || { + let mut child = child; + let stderr = child + .stderr + .take() + .map(|pipe| StderrCollector::start(pipe, None::)); abandon_unwritten_request( child, + stderr, &io::Error::from(io::ErrorKind::BrokenPipe), Duration::from_secs(1), ) @@ -1061,9 +1107,14 @@ mod tests { child.wait().unwrap(); let started = Instant::now(); + let stderr = child + .stderr + .take() + .map(|pipe| StderrCollector::start(pipe, None::)); let error = finishes_within(Duration::from_secs(4), move || { abandon_unwritten_request( child, + stderr, &io::Error::from(io::ErrorKind::BrokenPipe), Duration::from_millis(300), ) diff --git a/docs/testing.md b/docs/testing.md index 38aa720de3..25ada8387f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -301,13 +301,20 @@ runs the whole client against a fake that exits without reading; `the_stderr_of_a_failed_write_is_not_waited_for_past_the_grace` one whose grandchild keeps the pipe open. The last two run the step on a real child with an injected write error, because whether a real write fails before the child has -closed its stdin is a race. The answer of an engine that exits without draining a request larger than a pipe buffer is not read on this path (every engine here reads stdin to the end before answering, so that needs one that breaks the protocol); the module comment in `stdio_client.rs` states the limit. The bound compares the deadline with the clock -before every receive and receives nothing once it has passed, because a receive -with an expired timeout still returns output that is already queued, which would -let a backlog or a descendant that keeps writing hold the error path past it; -`collecting_stops_at_the_deadline_even_with_output_queued` and -`collecting_returns_at_the_deadline_while_output_keeps_arriving` run that -collection directly, on every platform. +closed its stdin is a race. The answer of an engine that exits without draining a request larger than a pipe buffer is not read on this path (every engine here reads stdin to the end before answering, so that needs one that breaks the protocol); the module comment in `stdio_client.rs` states the limit. The stderr of a failed +write is collected by a thread that starts before the request is written and is +waited for with one deadline (`StderrCollector::finish`), returning what had +arrived when it passes; `a_collector_stops_waiting_at_its_deadline_and_keeps_collecting` +and `an_expired_deadline_returns_what_is_collected_without_waiting` run that on +every platform. + +Starting the collector before the write also means an engine that fills its +stderr pipe before it has drained a request larger than a pipe buffer cannot +deadlock the client (the engine blocked on a full stderr, the client on a full +stdin, nobody reading either). +`an_engine_that_writes_a_lot_of_stderr_before_reading_its_request_does_not_deadlock` +runs that against a fake that writes 300 kB to stderr first and is red when the +collector is started after the write. One wait is deliberately not bounded and has no test: after the child exits, the answer is read to the end of its stdout and its stderr collector is joined, so an From a1b005a5ca18151e279a60d61804f77a362aeb79 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 19:51:37 +0000 Subject: [PATCH 31/38] refactor(rocm): call AppPaths::env_vars directly and fix the engine protocol lib.rs entry app_path_env_vars in engine_runtime_select.rs had become a one-line delegate to AppPaths::env_vars. Its three call sites (the engine client, the chat tool bridge and the helpers beside it) now call paths.env_vars(), the delegate is gone, and so is its test, which repeated the one already beside AppPaths::env_vars in rocm-core. The module comments and the architecture entry that named the delegate are updated. The architecture entry for the engine protocol crate's lib.rs still opened with "the whole crate", which stopped being true when the stdio client and the fake engine got entries of their own; it now says it is the protocol contract. Signed-off-by: Jussi Elo --- apps/rocm/src/chat/mcp_bridge.rs | 3 +-- apps/rocm/src/engine_ipc.rs | 9 ++++----- apps/rocm/src/engine_runtime_select.rs | 27 ++++++-------------------- docs/architecture.md | 4 ++-- 4 files changed, 13 insertions(+), 30 deletions(-) diff --git a/apps/rocm/src/chat/mcp_bridge.rs b/apps/rocm/src/chat/mcp_bridge.rs index 85af28b5aa..ac1239a7c0 100644 --- a/apps/rocm/src/chat/mcp_bridge.rs +++ b/apps/rocm/src/chat/mcp_bridge.rs @@ -22,7 +22,6 @@ use super::summaries::managed_service_running_state; use crate::automations::{render_automations_text, render_daemon_text}; -use crate::engine_runtime_select::app_path_env_vars; use crate::freeform_planner::{format_structured_tool_call, plan_freeform_request_with_context}; use crate::managed_service_status::{load_managed_services, managed_service_is_live}; use crate::rocm_argv::{chat_cli_arg_value, chat_cli_has_flag}; @@ -459,7 +458,7 @@ fn run_rocm_capture_for_paths( args: &[&str], timeout: Duration, ) -> Result { - rocm_core::run_rocm_capture_with_env(args, &app_path_env_vars(paths), timeout) + rocm_core::run_rocm_capture_with_env(args, &paths.env_vars(), timeout) } fn run_rocm_command_for_paths( diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index 98b2dfd2b1..6891e6f24b 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -45,9 +45,8 @@ //! `ScopedEnvVar` have no callers outside this module and stay private. The //! one helper this module calls back into at the crate root, `device_policy_name`, //! stays in `main.rs` and is reached via `use crate::device_policy_name;`. -//! `parse_device_policy` is in `gpu_policy.rs`; `app_path_env_vars` and -//! `engine_recipe_json_arg` are in `engine_runtime_select.rs`, and -//! `optional_arg` is in `rocm_engine_protocol`. +//! `parse_device_policy` is in `gpu_policy.rs`, `engine_recipe_json_arg` is in +//! `engine_runtime_select.rs`, and `optional_arg` is in `rocm_engine_protocol`. use std::ffi::OsString; use std::io::{self, Read}; @@ -65,7 +64,7 @@ use serde::Serialize; use serde::de::DeserializeOwned; use crate::device_policy_name; -use crate::engine_runtime_select::{app_path_env_vars, engine_recipe_json_arg}; +use crate::engine_runtime_select::engine_recipe_json_arg; use crate::gpu_policy::parse_device_policy; static BUILTIN_ENGINE_ENV_LOCK: OnceLock> = OnceLock::new(); @@ -223,7 +222,7 @@ fn with_scoped_builtin_engine_env( let _guard = lock.lock().expect("builtin engine env lock poisoned"); let mut vars = Vec::new(); if let Some(paths) = paths { - for (key, value) in app_path_env_vars(paths) { + for (key, value) in paths.env_vars() { vars.push(ScopedEnvVar::set_path(key, value)); } } diff --git a/apps/rocm/src/engine_runtime_select.rs b/apps/rocm/src/engine_runtime_select.rs index 4f4a1816e5..cd80046712 100644 --- a/apps/rocm/src/engine_runtime_select.rs +++ b/apps/rocm/src/engine_runtime_select.rs @@ -11,8 +11,8 @@ //! `parse_engine_recipe_json_arg`, //! `daemon_run_argv`, `managed_service_launcher_path`), the `ROCM_CLI_*` //! environment variables every managed child inherits -//! (`app_path_env_vars`/`app_path_env_var_values`/`app_path_env_var_refs`/ -//! `apply_app_path_env`), and polling a freshly launched service until it can +//! (`app_path_env_var_values`/`app_path_env_var_refs`/ +//! `apply_app_path_env`, over [`AppPaths::env_vars`]), and polling a freshly launched service until it can //! actually answer inference requests (`wait_for_service_http_ready`, //! `wait_for_service_http_ready_with_progress`, //! `service_http_readiness_paths`, `http_get_local_service`, @@ -64,9 +64,7 @@ //! `engine_recipe_json_arg` is `pub(crate)`, read from `engine_ipc.rs`'s //! `engine_request`; `parse_engine_recipe_json_arg` is `pub(crate)`, read //! from `main.rs`'s `dispatch()` and `service_control.rs`'s `restart_child_argv`. -//! `app_path_env_vars` (which is [`AppPaths::env_vars`]) and `apply_app_path_env` -//! are `pub(crate)`, read from `engine_ipc.rs`, `app_path_env_vars` is also -//! read from `chat/mcp_bridge.rs`, and `apply_app_path_env` is also read from +//! `apply_app_path_env` is `pub(crate)`, read from `engine_ipc.rs` and //! `engines_cmd.rs`, //! `service_launch.rs`, and `service_control.rs`'s //! `restart_internal_managed_service_with`; `app_path_env_var_values`/ @@ -267,16 +265,13 @@ pub(crate) fn parse_engine_recipe_json_arg( .transpose() } -pub(crate) fn app_path_env_vars(paths: &AppPaths) -> [(&'static str, &Path); 3] { - paths.env_vars() -} - #[cfg_attr(not(windows), allow(dead_code))] pub(crate) fn app_path_env_var_values( paths: &AppPaths, env_root: Option<&Path>, ) -> Vec<(&'static str, PathBuf)> { - let mut vars = app_path_env_vars(paths) + let mut vars = paths + .env_vars() .into_iter() .map(|(key, value)| (key, value.to_path_buf())) .collect::>(); @@ -314,7 +309,7 @@ pub(crate) fn managed_service_launcher_path() -> Result { } pub(crate) fn apply_app_path_env(command: &mut ProcessCommand, paths: &AppPaths) { - for (key, value) in app_path_env_vars(paths) { + for (key, value) in paths.env_vars() { command.env(key, value); } } @@ -911,16 +906,6 @@ mod tests { Ok(()) } - #[test] - fn app_path_env_vars_include_config_data_and_cache() { - let paths = test_app_paths(); - let vars = app_path_env_vars(&paths); - - assert_eq!(vars[0], ("ROCM_CLI_CONFIG_DIR", paths.config_dir.as_path())); - assert_eq!(vars[1], ("ROCM_CLI_DATA_DIR", paths.data_dir.as_path())); - assert_eq!(vars[2], ("ROCM_CLI_CACHE_DIR", paths.cache_dir.as_path())); - } - #[test] fn app_path_env_var_values_include_engine_env_root_when_needed() { let paths = test_app_paths(); diff --git a/docs/architecture.md b/docs/architecture.md index 11abf9da89..e608f2b3ae 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -60,7 +60,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`endpoint_keys.rs`](../apps/rocm/src/endpoint_keys.rs) — mechanical relocation, no owned types, reached by qualified path. - [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) — mechanical relocation out of [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), no owned types: which runtime an engine installs into and where its environment lives. `engine_manages_own_runtime` says whether an engine brings its own runtime; `env_root_for_engine_install`, `env_root_for_runtime` and `env_root_for_self_managed_engine` resolve the engine environment root; `runtime_manifest_for_selector` resolves a selector to one installed runtime; `runtime_key_for_python` names the runtime whose install root contains an interpreter (the private `runtime_key_owning_python` is the decision behind it). It is the neutral layer that [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) and [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) (with its `torch_alignment.rs` child) both import from, so neither imports the other for these helpers. It imports `recover_setup_runtime_registration` from [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs), `select_runtime_manifest` and `validate_runtime_manifest_for_activation` from [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), and the runtime-manifest model from [`therock.rs`](../apps/rocm/src/therock.rs); it does not go through the re-exports in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs), which would lead back to it via `runtime_services.rs`. Its functions are `pub(crate)` for those two modules plus [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs). Their tests moved with them. - [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-213): resolving an engine's binary path (`resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, `find_engine_plugin_binary`), the request entry points, which either answer from the in-process built-in handler or run the resolved engine binary with `stdio` (`engine_request`, `engine_request_with_env_root`, and `engine_stdio_child_request`, which hands the child to the stdio client `rocm` shares with `rocmd`, from [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs), and chooses what `rocm` asks of it: the app paths and engine environments root for the child, progress for an install, and the child's stderr forwarded to the terminal), the `ScopedEnvVar`/`with_scoped_builtin_engine_env` env-scoping helpers and the `BUILTIN_ENGINE_ENV_LOCK` mutex they serialize on, `builtin_engine_request`, `run_builtin_engine_stdio`), and the built-in HTTP-serving path (`builtin_engine_available`, `run_builtin_engine_serve_http`, `builtin_engine_serve_http_args`). One item that sits physically inside this span in `main.rs` stayed there instead of moving: `impl From for WatcherMode` is unrelated clap-arg conversion that merely happened to be interleaved between `find_engine_plugin_binary` and `engine_request` (it has since moved to [`cli.rs`](../apps/rocm/src/cli.rs), next to the enum it converts). `resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, `engine_request`, `engine_request_with_env_root`, `builtin_engine_available`, and `builtin_engine_serve_http_args` are `pub(crate)` and read from other modules (for example [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) and [`service_launch.rs`](../apps/rocm/src/service_launch.rs)), not from `main.rs`. `run_builtin_engine_stdio` and `run_builtin_engine_serve_http` are `pub(crate)` and reached directly from `main.rs`'s `dispatch()` (the `Command::EngineStdio` and `__engine-serve-http` arms); `find_engine_plugin_binary`, `with_scoped_builtin_engine_env`, `builtin_engine_request`, and `ScopedEnvVar` have no callers outside this module and stay private. -- [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-215): deciding which runtime/env a `serve`/`engines install` invocation resolves to (`EngineSelection`, `resolve_engine_selection`, `validate_engine_selection_runtime`, `single_ready_runtime_key`, `resolve_runtime_selector_to_exact_key`), small argv-building helpers used when relaunching a managed engine (`engine_recipe_json_arg`, `parse_engine_recipe_json_arg`, `daemon_run_argv`, `managed_service_launcher_path`), the `ROCM_CLI_*` environment variables every managed child inherits (`app_path_env_vars`, which is [`AppPaths::env_vars`](../crates/rocm-core/src/paths.rs), `app_path_env_var_values`/`app_path_env_var_refs`/`apply_app_path_env`), and polling a freshly launched service until it can answer inference requests (`wait_for_service_http_ready`, `wait_for_service_http_ready_with_progress`, `service_http_readiness_paths`, `http_get_local_service`, `http_post_local_service_json`, `service_http_readiness_response_ready`, `lemonade_health_ready_for_model`, `model_list_ready_for_model`, `service_model_gpu_ready`, `service_model_reports_rocm_backend`, `service_model_names_match`). `single_ready_runtime_key` does not decide which runtimes count as ready: it asks [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs)'s `select_single_ready_runtime`, which `rocm comfyui install` shares. `const fn status_for_readiness` sits physically interleaved in this span in `main.rs` but stayed there instead of moving — it has callers outside this cluster ([`service_launch.rs`](../apps/rocm/src/service_launch.rs) and [`service_control.rs`](../apps/rocm/src/service_control.rs)'s `record_restart_readiness`, which `restart_internal_managed_service_with` calls), so it stays a shared crate-root function, reached only from this module's own tests, via `use crate::status_for_readiness;`. Which items are `pub(crate)` and who calls them is documented in the module's own doc comment rather than repeated here. The cluster's regression tests live in the module's own `mod tests`. What remains in `main.rs` immediately after this cluster is `treat_as_natural_language`, dispatch-adjacent natural-language-invocation routing left over from an earlier phase (ROCMAI-87), explicitly out of scope and untouched by this split. +- [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-215): deciding which runtime/env a `serve`/`engines install` invocation resolves to (`EngineSelection`, `resolve_engine_selection`, `validate_engine_selection_runtime`, `single_ready_runtime_key`, `resolve_runtime_selector_to_exact_key`), small argv-building helpers used when relaunching a managed engine (`engine_recipe_json_arg`, `parse_engine_recipe_json_arg`, `daemon_run_argv`, `managed_service_launcher_path`), the `ROCM_CLI_*` environment variables every managed child inherits (`app_path_env_var_values`/`app_path_env_var_refs`/`apply_app_path_env`, over [`AppPaths::env_vars`](../crates/rocm-core/src/paths.rs)), and polling a freshly launched service until it can answer inference requests (`wait_for_service_http_ready`, `wait_for_service_http_ready_with_progress`, `service_http_readiness_paths`, `http_get_local_service`, `http_post_local_service_json`, `service_http_readiness_response_ready`, `lemonade_health_ready_for_model`, `model_list_ready_for_model`, `service_model_gpu_ready`, `service_model_reports_rocm_backend`, `service_model_names_match`). `single_ready_runtime_key` does not decide which runtimes count as ready: it asks [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs)'s `select_single_ready_runtime`, which `rocm comfyui install` shares. `const fn status_for_readiness` sits physically interleaved in this span in `main.rs` but stayed there instead of moving — it has callers outside this cluster ([`service_launch.rs`](../apps/rocm/src/service_launch.rs) and [`service_control.rs`](../apps/rocm/src/service_control.rs)'s `record_restart_readiness`, which `restart_internal_managed_service_with` calls), so it stays a shared crate-root function, reached only from this module's own tests, via `use crate::status_for_readiness;`. Which items are `pub(crate)` and who calls them is documented in the module's own doc comment rather than repeated here. The cluster's regression tests live in the module's own `mod tests`. What remains in `main.rs` immediately after this cluster is `treat_as_natural_language`, dispatch-adjacent natural-language-invocation routing left over from an earlier phase (ROCMAI-87), explicitly out of scope and untouched by this split. - [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) — mechanical relocation, owns private types (Phase 6a): `engines()` (the `rocm engines` command), `ensure_self_managed_engine_ready` (the readiness check [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) runs for engines that bring their own runtime) and `env_root_for_service` (read from [`service_launch.rs`](../apps/rocm/src/service_launch.rs) and [`service_control.rs`](../apps/rocm/src/service_control.rs)). Which runtime an engine installs into and where its environment lives is answered in [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs), which this module imports from. This module also calls into [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) for `settle_engine_install` and the host system-package installs; that module does not import this one. - [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs) — mechanical relocation, dispatch-adjacent, owns private types: `rocm examine`'s command handler (`examine()`), which prints the human report through [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs)'s `examine_human_report`, or with `--json` the document built from the private `ExamineJson`/`ExamineJsonSummary` types (the `Examination` verbatim plus a summary of what the CLI knows but the probe cannot). `examine()` is `pub(crate)` and re-imported in [`main.rs`](../apps/rocm/src/main.rs) for `dispatch()`. No test in the crate calls it or builds the document directly. - [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs) — mechanical relocation: the launch-summary/examine text surfaces (`render_launch_summary`, `render_examine_text`, `render_examine_text_with_paths`, `examine_human_report`, `render_examine_plain_header`, `plain_status_label`) and the engine-inventory listing shared by `rocm engines` and `rocm examine` (`write_default_engine_legend`, `render_engine_inventory_text_with_paths`, `append_examine_runtime_state`, `append_examine_engine_inventory`, `append_engine_detect_summary`, `friendly_engine_runtime_kind`, `friendly_engine_detect_notes`, `friendly_engine_detect_note_fallback`, `engine_runtime_status_label`). The `DEFAULT_ENGINE_MARKER` constant is shared by both engine-inventory renderers so the two copies cannot drift apart. The `rocm model` catalog renderer is not here; it lives in [`model_catalog_text.rs`](../apps/rocm/src/model_catalog_text.rs), and nothing in this file calls into it. [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs)'s `examine()`, which the `Command::Examine` arm calls, imports and calls `examine_human_report`; `main.rs`'s own `launch_default` calls `render_launch_summary`, re-imported there. `render_examine_text`, `render_examine_text_with_paths` and `render_engine_inventory_text_with_paths` are `pub(crate)`, read from [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs)'s read-only command runner; `render_engine_inventory_text_with_paths` is also read from [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs)'s `render_engine_inventory_text`; `append_examine_runtime_state` is also `pub(crate)`, read only from [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)'s own test module, so `main.rs`'s re-import of it is itself `#[cfg(test)]` rather than carrying an otherwise-unused import into a non-test build. Every other item in this list has no caller outside this module and stays private. This module also reaches items that live at the crate root because they have other callers: `engine_inventory()` (also read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs) and [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs)), `PlatformRuleOut` and `engine_ruled_out_by_platform` (defined in [`model_fit.rs`](../apps/rocm/src/model_fit.rs) and re-imported in `main.rs`) — all reached via `use crate::{...};`. The regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` and `write_test_pip_runtime`/`runtime_manifest_path` through `crate::test_support::`/`crate::runtimes_cmd::`. Its tests do not call into `model_catalog_text.rs`: the test of the note the verbose `rocm model` listing prints for a rule-out engine lives beside that renderer. @@ -390,7 +390,7 @@ A contract surface: verify all impacted engines after any protocol change here ( - [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs) — the client side of the engine stdio protocol, the one implementation `rocm` and `rocmd` both run an engine child through: the request line, the child's environment (`EngineStdioOptions`: app paths, engine environments root, endpoint key file, progress request), stderr handling (kept for the error, optionally forwarded as it arrives), the SIGPIPE guard around the write and around each forward of the child's stderr, killing and reaping the child when its request cannot be written (the error then carries what it wrote to stderr) or its answer cannot be read, what a failing exit status or an unusable answer is reported as, and `decode_engine_response`. It sets no timeout, and does not bound the wait for the child's pipes after it exits (a grandchild holding one makes the request wait); its module comment states that limit. Each caller still chooses which program to run: `rocm` the resolved engine binary with `stdio` (or the built-in handler, with no child), `rocmd` the `rocm` executable with `__engine-stdio `. It adds no wire format; the envelope types are the ones in [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs). - [`fake_engine.rs`](../crates/rocm-engine-protocol/src/fake_engine.rs) — test-only, behind the `test-support` feature the `rocm` and `rocmd` dev-dependencies turn on: a shell-script stand-in for an engine child that records its argv, environment and stdin and answers (or crashes) as a test asks, so both callers' tests run against the same fake. -- [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs) — the whole crate: the protocol and recipe-contract version constants, the request/response envelopes and the per-method request and response types for the `EngineMethod` calls (`detect`, `install`, `capabilities`, `resolve_model`, `launch`, `healthcheck`, `endpoint`, `stop`, `logs`), the `DevicePolicy` and `GpuSelection` types with the GPU-visibility helpers engines apply to a child process, engine-plugin binary naming and discovery, and the endpoint API key file carrier (`ENDPOINT_API_KEY_FILE_ENV`, `resolve_endpoint_api_key`, `is_public_bind_host`) that `rocm` and `rocmd` share, and `optional_arg`, the `flag value` argument helper both binaries build child command lines with. +- [`lib.rs`](../crates/rocm-engine-protocol/src/lib.rs) — the protocol contract: the protocol and recipe-contract version constants, the request/response envelopes and the per-method request and response types for the `EngineMethod` calls (`detect`, `install`, `capabilities`, `resolve_model`, `launch`, `healthcheck`, `endpoint`, `stop`, `logs`), the `DevicePolicy` and `GpuSelection` types with the GPU-visibility helpers engines apply to a child process, engine-plugin binary naming and discovery, and the endpoint API key file carrier (`ENDPOINT_API_KEY_FILE_ENV`, `resolve_endpoint_api_key`, `is_public_bind_host`) that `rocm` and `rocmd` share, and `optional_arg`, the `flag value` argument helper both binaries build child command lines with. ### `crates/rocm-deps` From 161300377303ce5e71f08cdb4b964f294ef4e76f Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 20:40:47 +0000 Subject: [PATCH 32/38] fix(engine-protocol): publish stderr as it arrives and stop retaining it once a request has failed Two problems in the stderr collector the failed-write path relies on. It read with read_until, so a diagnostic with no trailing newline stayed in the reader's private line buffer until the pipe ended. When a surviving descendant held the pipe open, the grace period expired with nothing published and the error lost text that had arrived in time. The collector now reads in fixed-size chunks and publishes each as it arrives, and forwards them as they arrive too. The grace-period test now writes its diagnostic with printf and no newline. And after a failed-write timeout or a failed stdout read the reader thread outlived the collector and kept appending whatever a surviving descendant wrote, so repeated failures could grow the long-lived daemon. Dropping the collector now marks the capture abandoned under the buffer's lock and releases what it held; the thread keeps draining the pipe (closing it would hand a still-writing descendant a SIGPIPE it did not get before) but keeps and forwards nothing, reading in chunks of a fixed size. Both tests were shown red by removing the behaviour: line-buffering the published output fails the grace-period test, and removing the drop handling fails the abandonment test, which keeps a handle on the shared buffer and checks it is empty after later output arrives and that nothing is forwarded. docs/testing.md describes them. Signed-off-by: Jussi Elo --- .../rocm-engine-protocol/src/stdio_client.rs | 191 ++++++++++++++---- docs/testing.md | 11 +- 2 files changed, 157 insertions(+), 45 deletions(-) diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 4093a1cb10..e789df2f86 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -70,7 +70,7 @@ //! deadlock the write. use std::ffi::OsStr; -use std::io::{self, BufRead, Read, Write}; +use std::io::{self, Read, Write}; use std::path::Path; use std::process::{Child, Command, ExitStatus, Stdio}; use std::sync::mpsc; @@ -332,31 +332,45 @@ fn abandon_unwritten_request( /// A child's stderr being collected on a thread of its own, from the moment it /// is started, whatever the main thread is doing meanwhile. +/// +/// The thread outlives the collector when a request fails and a descendant of +/// the engine still holds the pipe: nobody will read what it collects, but it +/// must keep draining the pipe (closing it would hand that descendant a SIGPIPE +/// it did not get before). So dropping the collector marks the capture +/// abandoned, which releases what was retained and makes the thread discard +/// whatever arrives later, in chunks of a fixed size, so a long-lived daemon does +/// not grow with every failed request. struct StderrCollector { - /// Everything read so far, appended line by line as it arrives. - collected: Arc>>, + shared: Arc>, /// Receives one message when the stderr has been read to its end. finished: mpsc::Receiver<()>, } +/// What the collecting thread has read, and whether anyone still wants it. +#[derive(Default)] +struct Captured { + bytes: Vec, + /// Set when the collector is dropped. Read and written under the same lock as + /// `bytes`, so "emptied and marked" is one step as far as the reader is + /// concerned and the buffer cannot grow after the drop returns. + abandoned: bool, +} + impl StderrCollector { - /// Start reading `stderr`, forwarding each line to `forward` when there is + /// Start reading `stderr`, forwarding what arrives to `forward` when there is /// one (see [`collect_stderr_into`]). fn start( stderr: impl Read + Send + 'static, forward: Option, ) -> Self { - let collected = Arc::new(Mutex::new(Vec::new())); + let shared = Arc::new(Mutex::new(Captured::default())); let (done, finished) = mpsc::channel(); - let sink = Arc::clone(&collected); + let sink = Arc::clone(&shared); thread::spawn(move || { collect_stderr_into(stderr, forward, &sink); let _ = done.send(()); }); - Self { - collected, - finished, - } + Self { shared, finished } } /// What was collected, once the stderr has ended, or what has arrived by @@ -374,13 +388,22 @@ impl StderrCollector { .recv_timeout(deadline.saturating_duration_since(Instant::now())); } } - self.collected - .lock() - .unwrap_or_else(PoisonError::into_inner) - .clone() + lock(&self.shared).bytes.clone() } } +impl Drop for StderrCollector { + fn drop(&mut self) { + let mut captured = lock(&self.shared); + captured.abandoned = true; + captured.bytes = Vec::new(); + } +} + +fn lock(shared: &Mutex) -> std::sync::MutexGuard<'_, Captured> { + shared.lock().unwrap_or_else(PoisonError::into_inner) +} + /// Read the child's whole answer from `stdout`. /// /// A failed read would leave the child running with nobody waiting on it, and @@ -409,49 +432,52 @@ fn read_response(child: &mut Child, mut stdout: impl Read) -> Result> { /// collection and is itself recorded as the last line. #[cfg(test)] fn collect_stderr(stderr: impl Read, forward: Option) -> Vec { - let collected = Mutex::new(Vec::new()); - collect_stderr_into(stderr, forward, &collected); - collected + let shared = Mutex::new(Captured::default()); + collect_stderr_into(stderr, forward, &shared); + shared .into_inner() .unwrap_or_else(PoisonError::into_inner) + .bytes } -/// [`collect_stderr`], appending to `collected` as each line arrives so a caller -/// that stops waiting still has what was read. +/// [`collect_stderr`], publishing into `shared` as each chunk arrives, not when a +/// line is complete, so a caller that stops waiting has everything that had been +/// written (a diagnostic with no trailing newline included) and not only the +/// complete lines. Once the capture is abandoned the thread keeps reading, so the +/// pipe stays drained, but keeps and forwards nothing. fn collect_stderr_into( - stderr: impl Read, + mut stderr: impl Read, mut forward: Option, - collected: &Mutex>, + shared: &Mutex, ) { - let mut reader = io::BufReader::new(stderr); - let mut line = Vec::new(); + let mut chunk = [0_u8; 4096]; loop { - line.clear(); - match reader.read_until(b'\n', &mut line) { + let bytes: Vec = match stderr.read(&mut chunk) { Ok(0) => break, - Ok(_) => { - if let Some(sink) = forward.as_mut() { - forward_best_effort(sink, &line); - } - append(collected, &line); - } + Ok(read) => chunk[..read].to_vec(), + Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, Err(error) => { - let message = format!("failed to read engine progress: {error}\n"); - if let Some(sink) = forward.as_mut() { - forward_best_effort(sink, message.as_bytes()); - } - append(collected, message.as_bytes()); + let message = format!("failed to read engine progress: {error}\n").into_bytes(); + publish(shared, forward.as_mut(), &message); break; } - } + }; + publish(shared, forward.as_mut(), &bytes); } } -fn append(collected: &Mutex>, bytes: &[u8]) { - collected - .lock() - .unwrap_or_else(PoisonError::into_inner) - .extend_from_slice(bytes); +/// Keep `bytes` and forward them, unless the capture has been abandoned. +fn publish(shared: &Mutex, forward: Option<&mut impl Write>, bytes: &[u8]) { + { + let mut captured = lock(shared); + if captured.abandoned { + return; + } + captured.bytes.extend_from_slice(bytes); + } + if let Some(sink) = forward { + forward_best_effort(sink, bytes); + } } /// Write `bytes` to the forwarding sink and drop the result: a sink that has @@ -646,6 +672,81 @@ mod tests { ); } + /// A source fed by the test, one chunk per message, that says when it is + /// dropped (the collecting thread has ended). + struct Fed { + chunks: mpsc::Receiver>, + dropped: mpsc::Sender<()>, + } + impl Read for Fed { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + match self.chunks.recv() { + Ok(chunk) => { + buffer[..chunk.len()].copy_from_slice(&chunk); + Ok(chunk.len()) + } + Err(_) => Ok(0), + } + } + } + impl Drop for Fed { + fn drop(&mut self) { + let _ = self.dropped.send(()); + } + } + + /// After a request has failed and its collector is gone, a descendant that + /// still writes to the pipe must not make the daemon grow: what was retained + /// is released at once, and what arrives later is drained and discarded (and + /// not forwarded), in chunks of a fixed size. The test keeps a handle on the + /// shared buffer to see that. + #[test] + fn an_abandoned_collector_releases_what_it_held_and_discards_what_arrives_later() { + let (feed, chunks) = mpsc::channel(); + let (dropped, ended) = mpsc::channel(); + let forwarded = Arc::new(Mutex::new(Vec::new())); + let sink = { + let forwarded = Arc::clone(&forwarded); + FnSink(move |bytes: &[u8]| { + forwarded.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + }) + }; + let collector = StderrCollector::start(Fed { chunks, dropped }, Some(sink)); + let shared = Arc::clone(&collector.shared); + + feed.send(b"before the failure\n".to_vec()).unwrap(); + // Wait for the thread to have published it, not for the stderr to end. + let deadline = Instant::now() + Duration::from_secs(10); + while lock(&shared).bytes.is_empty() { + assert!(Instant::now() < deadline, "the chunk was never published"); + thread::sleep(Duration::from_millis(2)); + } + assert_eq!(lock(&shared).bytes, b"before the failure\n"); + + drop(collector); + assert!(lock(&shared).bytes.is_empty(), "what was held is released"); + assert!(lock(&shared).abandoned); + + for _ in 0..50 { + feed.send(vec![b'x'; 4096]).unwrap(); + } + drop(feed); + ended + .recv_timeout(Duration::from_secs(10)) + .expect("the thread kept draining until the pipe closed"); + let retained = lock(&shared).bytes.len(); + assert_eq!( + retained, 0, + "bytes retained after the capture was abandoned" + ); + assert_eq!( + forwarded.lock().unwrap().as_slice(), + b"before the failure\n", + "nothing is forwarded after the request has ended" + ); + } + /// A status that is neither an exit nor a signal (a stopped process, on /// Unix) has neither a code nor a signal to name, and is reported as that. #[cfg(unix)] @@ -1092,11 +1193,13 @@ mod tests { /// A grandchild that holds the child's stderr open past its exit cannot /// make the error path wait for it: what had been written is reported - /// and the rest is not waited for. + /// and the rest is not waited for. The diagnostic has no trailing newline, + /// as an engine's last words often do not, and is still reported: the + /// collector publishes what has arrived, not only complete lines. #[test] fn the_stderr_of_a_failed_write_is_not_waited_for_past_the_grace() { let mut child = Command::new("sh") - .args(["-c", "echo said >&2; sleep 5 & exit 3"]) + .args(["-c", "printf said >&2; sleep 5 & exit 3"]) .stdin(Stdio::piped()) .stdout(Stdio::null()) .stderr(Stdio::piped()) diff --git a/docs/testing.md b/docs/testing.md index 7d85e48908..1c6b39f6eb 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -370,7 +370,16 @@ write is collected by a thread that starts before the request is written and is waited for with one deadline (`StderrCollector::finish`), returning what had arrived when it passes; `a_collector_stops_waiting_at_its_deadline_and_keeps_collecting` and `an_expired_deadline_returns_what_is_collected_without_waiting` run that on -every platform. +every platform. It publishes each chunk as it arrives, not each complete line, so +a diagnostic with no trailing newline that arrived before the deadline is +reported (`the_stderr_of_a_failed_write_is_not_waited_for_past_the_grace` uses +`printf` without a newline for that reason). When the request has failed and the +collector is dropped, the capture is marked abandoned: what it held is released, +and the thread, which must keep draining the pipe so a surviving descendant is not +sent a SIGPIPE, discards and no longer forwards what arrives later, in chunks of +a fixed size, so a long-lived daemon does not grow with every failed request +(`an_abandoned_collector_releases_what_it_held_and_discards_what_arrives_later`, +red without the drop handling). Starting the collector before the write also means an engine that fills its stderr pipe before it has drained a request larger than a pipe buffer cannot From bb914a70a36cbb0b3d68862861eb776189577f1d Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Tue, 10 Nov 2026 10:57:14 +0000 Subject: [PATCH 33/38] fix(engine-protocol): report stderr when the answer cannot be read, kill on a missing pipe, note a failing exit after an answer A failed read of the child's stdout dropped the stderr already collected, unlike every other failure path; the error now carries it ("...: pipe went away; stderr: ..."), through the same bounded grace as a failed write. A child whose stdin or stdout pipe is missing is killed and reaped instead of being left running (unreachable while the command pipes all three streams, so the check is a small function a test drives with a missing pipe). A child that answers and then exits with a failing status is still read as having answered, but its status is no longer dropped without a trace: the daemon asks the client (note_failing_exit_after_answer) to leave a one-line note on its stderr, which is its log. The CLI does not ask, since its stderr is a terminal. The decision is unchanged. StderrCollector::finish now takes the collected bytes instead of cloning them. The three-line loop that set the app-path variables on a command, repeated in the client and in rocm's apply_app_path_env, is one AppPaths::apply_env in rocm-core, and rocm's copy and its imports are gone. Each new test was shown red by removing the behaviour it covers; docs/testing.md describes them. Signed-off-by: Jussi Elo --- apps/rocm/src/engine_ipc.rs | 2 + apps/rocm/src/engine_runtime_select.rs | 16 +- apps/rocm/src/engines_cmd.rs | 4 +- apps/rocm/src/service_control.rs | 3 +- .../src/service_launch/helper_autostart.rs | 3 +- apps/rocm/src/service_launch/spawn.rs | 3 +- apps/rocmd/src/engine_ipc.rs | 3 + crates/rocm-core/src/paths.rs | 28 +++ .../rocm-engine-protocol/src/stdio_client.rs | 175 ++++++++++++++++-- docs/architecture.md | 2 +- docs/testing.md | 11 ++ 11 files changed, 215 insertions(+), 35 deletions(-) diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index 6891e6f24b..d36318c7ae 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -179,6 +179,7 @@ const fn stdio_options<'a>( endpoint_key_file: None, request_progress: stream_progress, forward_stderr: true, + note_failing_exit_after_answer: false, } } @@ -442,6 +443,7 @@ mod tests { assert!(options.request_progress); // The terminal user watches an install through the child's stderr. assert!(options.forward_stderr); + assert!(!options.note_failing_exit_after_answer); // No request `rocm` makes is about a running service's endpoint. assert_eq!(options.endpoint_key_file, None); diff --git a/apps/rocm/src/engine_runtime_select.rs b/apps/rocm/src/engine_runtime_select.rs index cd80046712..aa65af269d 100644 --- a/apps/rocm/src/engine_runtime_select.rs +++ b/apps/rocm/src/engine_runtime_select.rs @@ -11,8 +11,8 @@ //! `parse_engine_recipe_json_arg`, //! `daemon_run_argv`, `managed_service_launcher_path`), the `ROCM_CLI_*` //! environment variables every managed child inherits -//! (`app_path_env_var_values`/`app_path_env_var_refs`/ -//! `apply_app_path_env`, over [`AppPaths::env_vars`]), and polling a freshly launched service until it can +//! (`app_path_env_var_values`/`app_path_env_var_refs`, over +//! [`AppPaths::env_vars`]; a command gets them from `AppPaths::apply_env`), and polling a freshly launched service until it can //! actually answer inference requests (`wait_for_service_http_ready`, //! `wait_for_service_http_ready_with_progress`, //! `service_http_readiness_paths`, `http_get_local_service`, @@ -64,10 +64,7 @@ //! `engine_recipe_json_arg` is `pub(crate)`, read from `engine_ipc.rs`'s //! `engine_request`; `parse_engine_recipe_json_arg` is `pub(crate)`, read //! from `main.rs`'s `dispatch()` and `service_control.rs`'s `restart_child_argv`. -//! `apply_app_path_env` is `pub(crate)`, read from `engine_ipc.rs` and -//! `engines_cmd.rs`, -//! `service_launch.rs`, and `service_control.rs`'s -//! `restart_internal_managed_service_with`; `app_path_env_var_values`/ +//! `app_path_env_var_values`/ //! `app_path_env_var_refs` are `pub(crate)`, read from `service_launch.rs` //! and `service_control.rs`'s `restart_internal_managed_service_with`. //! `daemon_run_argv` is `pub(crate)`, read from `main.rs`'s `dispatch()`; @@ -93,7 +90,6 @@ use std::ffi::OsString; use std::path::{Path, PathBuf}; -use std::process::Command as ProcessCommand; use std::thread; use std::time::Duration; @@ -308,12 +304,6 @@ pub(crate) fn managed_service_launcher_path() -> Result { Ok(current_exe) } -pub(crate) fn apply_app_path_env(command: &mut ProcessCommand, paths: &AppPaths) { - for (key, value) in paths.env_vars() { - command.env(key, value); - } -} - pub(crate) fn wait_for_service_http_ready( engine: &str, host: &str, diff --git a/apps/rocm/src/engines_cmd.rs b/apps/rocm/src/engines_cmd.rs index eaee862b45..dab1629cba 100644 --- a/apps/rocm/src/engines_cmd.rs +++ b/apps/rocm/src/engines_cmd.rs @@ -41,7 +41,7 @@ use crate::engine_install_env::{ }; use crate::engine_ipc::{engine_request, engine_request_with_env_root}; use crate::engine_runtime_select::{ - apply_app_path_env, resolve_engine_selection, resolve_runtime_selector_to_exact_key, + resolve_engine_selection, resolve_runtime_selector_to_exact_key, validate_engine_selection_runtime, }; use crate::runtime_install_finalize::{ @@ -469,7 +469,7 @@ fn engine_shell( .env("ROCM_CLI_ENV_ID", &resolved.env_id) .env("ROCM_CLI_RUNTIME_ID", &resolved.runtime_id) .env("ROCM_CLI_PYTHON", &resolved.python_executable); - apply_app_path_env(&mut command, &paths); + paths.apply_env(&mut command); let prompt = format!("(rocm:{engine}) "); let mut prompt_marked = false; diff --git a/apps/rocm/src/service_control.rs b/apps/rocm/src/service_control.rs index 2f4db4617e..bdb3d0038b 100644 --- a/apps/rocm/src/service_control.rs +++ b/apps/rocm/src/service_control.rs @@ -57,7 +57,6 @@ use rocm_engine_protocol::{EngineMethod, StopRequest, StopResponse}; use crate::endpoint_keys; use crate::engine_ipc::{builtin_engine_serve_http_args, engine_request}; #[cfg(not(windows))] -use crate::engine_runtime_select::apply_app_path_env; #[cfg(windows)] use crate::engine_runtime_select::{app_path_env_var_refs, app_path_env_var_values}; use crate::engine_runtime_select::{ @@ -715,7 +714,7 @@ fn restart_internal_managed_service_inner( command.stdin(Stdio::null()); attach_background_stdio(&mut command, Some(&record.log_path))?; detach_background_command(&mut command); - apply_app_path_env(&mut command, paths); + paths.apply_env(&mut command); if let Some(engine_envs_root) = engine_envs_root.as_deref() { command.env("ROCM_CLI_ENGINE_ENVS_ROOT", engine_envs_root); } diff --git a/apps/rocm/src/service_launch/helper_autostart.rs b/apps/rocm/src/service_launch/helper_autostart.rs index 7a54cb5134..19872240bc 100644 --- a/apps/rocm/src/service_launch/helper_autostart.rs +++ b/apps/rocm/src/service_launch/helper_autostart.rs @@ -11,7 +11,6 @@ use super::spawn::{ }; use crate::background_helper::background_helper_already_running; #[cfg(not(windows))] -use crate::engine_runtime_select::apply_app_path_env; use crate::engine_runtime_select::managed_service_launcher_path; #[cfg(windows)] use crate::engine_runtime_select::{app_path_env_var_refs, app_path_env_var_values}; @@ -151,7 +150,7 @@ fn spawn_background_helper(paths: &AppPaths) -> Result> { command.stdin(Stdio::null()); attach_background_stdio(&mut command, None)?; detach_background_command(&mut command); - apply_app_path_env(&mut command, paths); + paths.apply_env(&mut command); command .spawn() .map(|child| child.id()) diff --git a/apps/rocm/src/service_launch/spawn.rs b/apps/rocm/src/service_launch/spawn.rs index 0bc3d02a34..3eaafc6c65 100644 --- a/apps/rocm/src/service_launch/spawn.rs +++ b/apps/rocm/src/service_launch/spawn.rs @@ -14,7 +14,6 @@ use super::launch_transaction::{ use crate::endpoint_keys; use crate::engine_ipc::builtin_engine_serve_http_args; #[cfg(not(windows))] -use crate::engine_runtime_select::apply_app_path_env; use crate::engine_runtime_select::managed_service_launcher_path; #[cfg(windows)] use crate::engine_runtime_select::{app_path_env_var_refs, app_path_env_var_values}; @@ -510,7 +509,7 @@ fn launch_managed_engine_child_locked( command.stdin(Stdio::null()); attach_background_stdio(&mut command, Some(&record.log_path))?; detach_background_command(&mut command); - apply_app_path_env(&mut command, paths); + paths.apply_env(&mut command); if let Some(engine_envs_root) = engine_envs_root.as_deref() { command.env("ROCM_CLI_ENGINE_ENVS_ROOT", engine_envs_root); } diff --git a/apps/rocmd/src/engine_ipc.rs b/apps/rocmd/src/engine_ipc.rs index 0c28e551a5..f4749c3eb2 100644 --- a/apps/rocmd/src/engine_ipc.rs +++ b/apps/rocmd/src/engine_ipc.rs @@ -213,6 +213,7 @@ const fn stdio_options<'a>( endpoint_key_file, request_progress: false, forward_stderr: false, + note_failing_exit_after_answer: true, } } @@ -400,6 +401,8 @@ mod tests { // a daemon's terminal. assert!(!options.request_progress); assert!(!options.forward_stderr); + // A daemon's stderr is its log, so a failing exit after an answer leaves a note there. + assert!(options.note_failing_exit_after_answer); assert_eq!(options.env_root, None); assert_eq!(stdio_options(&paths, None).endpoint_key_file, None); diff --git a/crates/rocm-core/src/paths.rs b/crates/rocm-core/src/paths.rs index 80810bbe71..fa5e152fcc 100644 --- a/crates/rocm-core/src/paths.rs +++ b/crates/rocm-core/src/paths.rs @@ -60,6 +60,14 @@ impl AppPaths { ] } + /// Set [`Self::env_vars`] on `command`, so the child it starts runs on these + /// roots. + pub fn apply_env(&self, command: &mut std::process::Command) { + for (key, value) in self.env_vars() { + command.env(key, value); + } + } + pub fn discover() -> Result { let data_dir_override = env_path_override(DATA_DIR_ENV); let cache_dir_override = env_path_override(CACHE_DIR_ENV); @@ -386,6 +394,26 @@ mod tests { /// `env_vars` and `discover` share one constant per name, so they cannot /// disagree; the names themselves are what a child process and a user's /// shell see, so they are pinned here. + #[test] + fn apply_env_sets_exactly_the_three_roots_on_a_command() { + let (_root, paths) = temp_app_paths("apply-env"); + let mut command = std::process::Command::new("true"); + paths.apply_env(&mut command); + let mut set: Vec<_> = command + .get_envs() + .map(|(key, value)| (key.to_owned(), value.map(std::ffi::OsStr::to_owned))) + .collect(); + let expected: Vec<_> = paths + .env_vars() + .iter() + .map(|(key, value)| ((*key).into(), Some(value.as_os_str().to_owned()))) + .collect(); + let mut expected = expected; + set.sort(); + expected.sort(); + assert_eq!(set, expected); + } + #[test] fn env_vars_pair_each_root_with_its_public_variable_name() { let (_root, paths) = temp_app_paths("env-vars"); diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index e789df2f86..802bb438b4 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -114,6 +114,11 @@ pub struct EngineStdioOptions<'a> { /// terminal user watches an install. Either way the whole of it is kept and /// shown in the error when the request fails. pub forward_stderr: bool, + /// Say, on this process's stderr, when the child answered and then exited + /// with a failing status. The answer is used either way; this only leaves a + /// trace of the status, which the caller would otherwise never see. For a + /// daemon, whose stderr is its log; a terminal user has no use for it. + pub note_failing_exit_after_answer: bool, } /// Build the request envelope for `method` from `request`. @@ -143,9 +148,7 @@ where .stdout(Stdio::piped()) .stderr(Stdio::piped()); if let Some(paths) = options.app_paths { - for (key, value) in paths.env_vars() { - command.env(key, value); - } + paths.apply_env(&mut command); } if let Some(env_root) = options.env_root { command.env(ENGINE_ENVS_ROOT_ENV, env_root); @@ -208,10 +211,8 @@ where .spawn() .with_context(|| format!("failed to spawn engine stdio process {}", program.display()))?; - let mut stdin = child - .stdin - .take() - .context("engine stdio child did not expose stdin")?; + let stdin = child.stdin.take(); + let mut stdin = require_pipe(&mut child, stdin, "stdin")?; // Read the child's stderr from the start, on its own thread: an engine that // fills the stderr pipe before it has drained a request larger than a pipe // buffer would otherwise block on that write while this process blocks on @@ -239,11 +240,18 @@ where // End of the request: the engine reads to EOF before it answers. drop(stdin); - let stdout_pipe = child - .stdout - .take() - .context("engine stdio child did not expose stdout")?; - let stdout = read_response(&mut child, stdout_pipe)?; + let stdout_pipe = child.stdout.take(); + let stdout_pipe = require_pipe(&mut child, stdout_pipe, "stdout")?; + let stdout = match read_response(&mut child, stdout_pipe) { + Ok(stdout) => stdout, + Err(error) => { + return Err(read_failure_with_stderr( + error, + stderr_collector, + EXITED_STDERR_GRACE, + )); + } + }; let status = child .wait() .context("failed waiting for engine stdio response")?; @@ -252,6 +260,12 @@ where .unwrap_or_default(); let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); + if !status.success() && !stdout.is_empty() && options.note_failing_exit_after_answer { + eprintln!( + "{} after printing its answer; the answer was used", + exit_note(status) + ); + } if !status.success() && stdout.is_empty() { let exited = exit_note(status); if stderr.is_empty() { @@ -294,6 +308,36 @@ fn unparseable_answer_message(status: ExitStatus, stderr: &str) -> String { message } +/// `pipe` of `child`, or an error that also kills and reaps the child: a child +/// whose request cannot be sent must not be left running, as on every other +/// failure path. Unreachable while the command pipes all three streams, which is +/// why it is a function of its own, testable with a pipe that is missing. +fn require_pipe(child: &mut Child, pipe: Option, name: &str) -> Result { + pipe.ok_or_else(|| { + let _ = child.kill(); + let _ = child.wait(); + anyhow::anyhow!("engine stdio child did not expose {name}") + }) +} + +/// The error for a stdout that could not be read (the child is already killed +/// and reaped), with what its stderr had said, as every other failure carries +/// it: a crashing or signalled engine is where the explanation matters most. +fn read_failure_with_stderr( + error: anyhow::Error, + stderr: Option, + grace: Duration, +) -> anyhow::Error { + let stderr = stderr + .map(|collector| collector.finish(Some(Instant::now() + grace))) + .unwrap_or_default(); + let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); + if stderr.is_empty() { + return error; + } + anyhow::anyhow!("{error:#}; stderr: {stderr}") +} + /// How long the stderr of a child that was killed, or had exited, is waited for /// when its request could not be written. const EXITED_STDERR_GRACE: Duration = Duration::from_secs(1); @@ -388,7 +432,7 @@ impl StderrCollector { .recv_timeout(deadline.saturating_duration_since(Instant::now())); } } - lock(&self.shared).bytes.clone() + std::mem::take(&mut lock(&self.shared).bytes) } } @@ -562,6 +606,7 @@ mod tests { endpoint_key_file: Some(Path::new("/keys/svc.endpoint-key")), request_progress: true, forward_stderr: true, + note_failing_exit_after_answer: false, }, ); assert_eq!(args_of(&command), ["__engine-stdio", "vllm"]); @@ -938,6 +983,7 @@ mod tests { endpoint_key_file: Some(Path::new("/keys/k")), request_progress: true, forward_stderr: false, + note_failing_exit_after_answer: false, }, ) .unwrap(); @@ -987,6 +1033,109 @@ mod tests { ); } + /// The status of a child that answered and then failed is not part of the + /// answer, so a caller that asks for it (the daemon) gets a note on this + /// process's stderr, and one that does not (the CLI) gets nothing. Run in + /// a re-executed test binary so the parent can read that stderr; the + /// decision itself is unchanged: the answer is used. + #[test] + fn a_failing_exit_after_an_answer_is_noted_only_when_asked_for() { + if let Some(arm) = std::env::var_os("ROCM_TEST_CLIENT_NOTE_ARM") { + let engine = FakeEngine::answering("client-note", STOPPED, 7); + let options = EngineStdioOptions { + note_failing_exit_after_answer: arm == "on", + ..EngineStdioOptions::default() + }; + let answer = run(&engine, &options).unwrap(); + assert_eq!(answer, json!({"stopped": true})); + return; + } + for (arm, expected) in [("on", true), ("off", false)] { + let output = Command::new(std::env::current_exe().unwrap()) + .args([ + "stdio_client::tests::child::a_failing_exit_after_an_answer_is_noted_only_when_asked_for", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_CLIENT_NOTE_ARM", arm) + .output() + .unwrap(); + assert!(output.status.success(), "{arm}: {:?}", output.status); + let stderr = String::from_utf8_lossy(&output.stderr); + let note = "engine stdio process exited with code 7 after printing its answer; the answer was used"; + assert_eq!(stderr.contains(note), expected, "{arm}:\n{stderr}"); + } + } + + /// A child whose stdin or stdout pipe is missing is killed and reaped, as + /// on every other failure path, not left running. Linux-only for the + /// `/proc` entry a zombie would keep. + #[cfg(target_os = "linux")] + #[test] + fn a_missing_pipe_kills_and_reaps_the_child() { + let mut child = Command::new("sleep") + .arg("60") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let proc_entry = format!("/proc/{}", child.id()); + assert!(Path::new(&proc_entry).exists()); + + let error = + require_pipe(&mut child, None::, "stdin").unwrap_err(); + + assert_eq!(error.to_string(), "engine stdio child did not expose stdin"); + assert!(!Path::new(&proc_entry).exists(), "killed and reaped"); + } + + /// A pipe that is there is handed back and the child left alone. + #[test] + fn a_present_pipe_is_returned_and_the_child_left_running() { + let mut child = Command::new("sleep") + .arg("60") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + assert_eq!(require_pipe(&mut child, Some(5), "stdout").unwrap(), 5); + assert!(child.try_wait().unwrap().is_none()); + child.kill().unwrap(); + child.wait().unwrap(); + } + + /// A failed read of the answer reports what the engine had said on + /// stderr, with the read error, like every other failure; the message and + /// its content are asserted together. + #[test] + fn a_failed_read_of_the_answer_reports_the_stderr_too() { + let collector = StderrCollector::start( + io::Cursor::new(b"engine crashed: out of memory\n".to_vec()), + None::, + ); + // The thread ends at once on a cursor; wait for its text. + let error = read_failure_with_stderr( + anyhow::Error::new(io::Error::other("pipe went away")) + .context("failed to read engine stdio response"), + Some(collector), + Duration::from_secs(5), + ); + assert_eq!( + error.to_string(), + "failed to read engine stdio response: pipe went away; stderr: engine crashed: out of memory" + ); + + // With nothing on stderr the error is the read error, unchanged. + let quiet = read_failure_with_stderr( + anyhow::anyhow!("failed to read engine stdio response"), + None, + Duration::from_secs(1), + ); + assert_eq!(quiet.to_string(), "failed to read engine stdio response"); + } + #[test] fn an_unparseable_answer_carries_the_childs_stderr() { let engine = FakeEngine::new( diff --git a/docs/architecture.md b/docs/architecture.md index 7582e56213..12f1d0b47f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -60,7 +60,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`endpoint_keys.rs`](../apps/rocm/src/endpoint_keys.rs) — mechanical relocation, no owned types, reached by qualified path. - [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs) — mechanical relocation out of [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), no owned types: which runtime an engine installs into and where its environment lives. `engine_manages_own_runtime` says whether an engine brings its own runtime; `env_root_for_engine_install`, `env_root_for_runtime` and `env_root_for_self_managed_engine` resolve the engine environment root; `runtime_manifest_for_selector` resolves a selector to one installed runtime; `runtime_key_for_python` names the runtime whose install root contains an interpreter (the private `runtime_key_owning_python` is the decision behind it). It is the neutral layer that [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) and [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) (with its `torch_alignment.rs` child) both import from, so neither imports the other for these helpers. It imports `recover_setup_runtime_registration` from [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs), `select_runtime_manifest` and `validate_runtime_manifest_for_activation` from [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), and the runtime-manifest model from [`therock.rs`](../apps/rocm/src/therock.rs); it does not go through the re-exports in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs), which would lead back to it via `runtime_services.rs`. Its functions are `pub(crate)` for those two modules plus [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs). Their tests moved with them. - [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-213): resolving an engine's binary path (`resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, `find_engine_plugin_binary`), the request entry points, which either answer from the in-process built-in handler or run the resolved engine binary with `stdio` (`engine_request`, `engine_request_with_env_root`, and `engine_stdio_child_request`, which hands the child to the stdio client `rocm` shares with `rocmd`, from [`stdio_client.rs`](../crates/rocm-engine-protocol/src/stdio_client.rs), and chooses what `rocm` asks of it: the app paths and engine environments root for the child, progress for an install, and the child's stderr forwarded to the terminal), the `ScopedEnvVar`/`with_scoped_builtin_engine_env` env-scoping helpers and the `BUILTIN_ENGINE_ENV_LOCK` mutex they serialize on, `builtin_engine_request`, `run_builtin_engine_stdio`), and the built-in HTTP-serving path (`builtin_engine_available`, `run_builtin_engine_serve_http`, `builtin_engine_serve_http_args`). One item that sits physically inside this span in `main.rs` stayed there instead of moving: `impl From for WatcherMode` is unrelated clap-arg conversion that merely happened to be interleaved between `find_engine_plugin_binary` and `engine_request` (it has since moved to [`cli.rs`](../apps/rocm/src/cli.rs), next to the enum it converts). `resolve_engine_binary_path`, `resolve_engine_binary_path_with_paths`, `engine_request`, `engine_request_with_env_root`, `builtin_engine_available`, and `builtin_engine_serve_http_args` are `pub(crate)` and read from other modules (for example [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) and [`service_launch.rs`](../apps/rocm/src/service_launch.rs)), not from `main.rs`. `run_builtin_engine_stdio` and `run_builtin_engine_serve_http` are `pub(crate)` and reached directly from `main.rs`'s `dispatch()` (the `Command::EngineStdio` and `__engine-serve-http` arms); `find_engine_plugin_binary`, `with_scoped_builtin_engine_env`, `builtin_engine_request`, and `ScopedEnvVar` have no callers outside this module and stay private. -- [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-215): deciding which runtime/env a `serve`/`engines install` invocation resolves to (`EngineSelection`, `resolve_engine_selection`, `validate_engine_selection_runtime`, `single_ready_runtime_key`, `resolve_runtime_selector_to_exact_key`), small argv-building helpers used when relaunching a managed engine (`engine_recipe_json_arg`, `parse_engine_recipe_json_arg`, `daemon_run_argv`, `managed_service_launcher_path`), the `ROCM_CLI_*` environment variables every managed child inherits (`app_path_env_var_values`/`app_path_env_var_refs`/`apply_app_path_env`, over [`AppPaths::env_vars`](../crates/rocm-core/src/paths.rs)), and polling a freshly launched service until it can answer inference requests (`wait_for_service_http_ready`, `wait_for_service_http_ready_with_progress`, `service_http_readiness_paths`, `http_get_local_service`, `http_post_local_service_json`, `service_http_readiness_response_ready`, `lemonade_health_ready_for_model`, `model_list_ready_for_model`, `service_model_gpu_ready`, `service_model_reports_rocm_backend`, `service_model_names_match`). `single_ready_runtime_key` does not decide which runtimes count as ready: it asks [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs)'s `select_single_ready_runtime`, which `rocm comfyui install` shares. `const fn status_for_readiness` sits physically interleaved in this span in `main.rs` but stayed there instead of moving — it has callers outside this cluster ([`service_launch.rs`](../apps/rocm/src/service_launch.rs) and [`service_control.rs`](../apps/rocm/src/service_control.rs)'s `record_restart_readiness`, which `restart_internal_managed_service_with` calls), so it stays a shared crate-root function, reached only from this module's own tests, via `use crate::status_for_readiness;`. Which items are `pub(crate)` and who calls them is documented in the module's own doc comment rather than repeated here. The cluster's regression tests live in the module's own `mod tests`. What remains in `main.rs` immediately after this cluster is `treat_as_natural_language`, dispatch-adjacent natural-language-invocation routing left over from an earlier phase (ROCMAI-87), explicitly out of scope and untouched by this split. +- [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-215): deciding which runtime/env a `serve`/`engines install` invocation resolves to (`EngineSelection`, `resolve_engine_selection`, `validate_engine_selection_runtime`, `single_ready_runtime_key`, `resolve_runtime_selector_to_exact_key`), small argv-building helpers used when relaunching a managed engine (`engine_recipe_json_arg`, `parse_engine_recipe_json_arg`, `daemon_run_argv`, `managed_service_launcher_path`), the `ROCM_CLI_*` environment variables every managed child inherits (`app_path_env_var_values`/`app_path_env_var_refs`, over [`AppPaths::env_vars`](../crates/rocm-core/src/paths.rs)), and polling a freshly launched service until it can answer inference requests (`wait_for_service_http_ready`, `wait_for_service_http_ready_with_progress`, `service_http_readiness_paths`, `http_get_local_service`, `http_post_local_service_json`, `service_http_readiness_response_ready`, `lemonade_health_ready_for_model`, `model_list_ready_for_model`, `service_model_gpu_ready`, `service_model_reports_rocm_backend`, `service_model_names_match`). `single_ready_runtime_key` does not decide which runtimes count as ready: it asks [`runtime_adopt.rs`](../apps/rocm/src/runtime_adopt.rs)'s `select_single_ready_runtime`, which `rocm comfyui install` shares. `const fn status_for_readiness` sits physically interleaved in this span in `main.rs` but stayed there instead of moving — it has callers outside this cluster ([`service_launch.rs`](../apps/rocm/src/service_launch.rs) and [`service_control.rs`](../apps/rocm/src/service_control.rs)'s `record_restart_readiness`, which `restart_internal_managed_service_with` calls), so it stays a shared crate-root function, reached only from this module's own tests, via `use crate::status_for_readiness;`. Which items are `pub(crate)` and who calls them is documented in the module's own doc comment rather than repeated here. The cluster's regression tests live in the module's own `mod tests`. What remains in `main.rs` immediately after this cluster is `treat_as_natural_language`, dispatch-adjacent natural-language-invocation routing left over from an earlier phase (ROCMAI-87), explicitly out of scope and untouched by this split. - [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) — mechanical relocation, owns private types (Phase 6a): `engines()` (the `rocm engines` command), `ensure_self_managed_engine_ready` (the readiness check [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) runs for engines that bring their own runtime) and `env_root_for_service` (read from [`service_launch.rs`](../apps/rocm/src/service_launch.rs) and [`service_control.rs`](../apps/rocm/src/service_control.rs)). Which runtime an engine installs into and where its environment lives is answered in [`engine_install_env.rs`](../apps/rocm/src/engine_install_env.rs), which this module imports from. This module also calls into [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) for `settle_engine_install` and the host system-package installs; that module does not import this one. - [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs) — mechanical relocation, dispatch-adjacent, owns private types: `rocm examine`'s command handler (`examine()`), which prints the human report through [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs)'s `examine_human_report`, or with `--json` the document built from the private `ExamineJson`/`ExamineJsonSummary` types (the `Examination` verbatim plus a summary of what the CLI knows but the probe cannot). `examine()` is `pub(crate)` and re-imported in [`main.rs`](../apps/rocm/src/main.rs) for `dispatch()`. No test in the crate calls it or builds the document directly. - [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs) — mechanical relocation: the launch-summary/examine text surfaces (`render_launch_summary`, `render_examine_text`, `render_examine_text_with_paths`, `examine_human_report`, `render_examine_plain_header`, `plain_status_label`) and the engine-inventory listing shared by `rocm engines` and `rocm examine` (`write_default_engine_legend`, `render_engine_inventory_text_with_paths`, `append_examine_runtime_state`, `append_examine_engine_inventory`, `append_engine_detect_summary`, `friendly_engine_runtime_kind`, `friendly_engine_detect_notes`, `friendly_engine_detect_note_fallback`, `engine_runtime_status_label`). The `DEFAULT_ENGINE_MARKER` constant is shared by both engine-inventory renderers so the two copies cannot drift apart. The `rocm model` catalog renderer is not here; it lives in [`model_catalog_text.rs`](../apps/rocm/src/model_catalog_text.rs), and nothing in this file calls into it. [`examine_cmd.rs`](../apps/rocm/src/examine_cmd.rs)'s `examine()`, which the `Command::Examine` arm calls, imports and calls `examine_human_report`; `main.rs`'s own `launch_default` calls `render_launch_summary`, re-imported there. `render_examine_text`, `render_examine_text_with_paths` and `render_engine_inventory_text_with_paths` are `pub(crate)`, read from [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs)'s read-only command runner; `render_engine_inventory_text_with_paths` is also read from [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs)'s `render_engine_inventory_text`; `append_examine_runtime_state` is also `pub(crate)`, read only from [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)'s own test module, so `main.rs`'s re-import of it is itself `#[cfg(test)]` rather than carrying an otherwise-unused import into a non-test build. Every other item in this list has no caller outside this module and stays private. This module also reaches items that live at the crate root because they have other callers: `engine_inventory()` (also read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs) and [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs)), `PlatformRuleOut` and `engine_ruled_out_by_platform` (defined in [`model_fit.rs`](../apps/rocm/src/model_fit.rs) and re-imported in `main.rs`) — all reached via `use crate::{...};`. The regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` and `write_test_pip_runtime`/`runtime_manifest_path` through `crate::test_support::`/`crate::runtimes_cmd::`. Its tests do not call into `model_catalog_text.rs`: the test of the note the verbose `rocm model` listing prints for a rule-out engine lives beside that renderer. diff --git a/docs/testing.md b/docs/testing.md index 088c28c7ec..efe173d880 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -345,6 +345,17 @@ binary whose own stderr the parent reads line is there with the option and absent without it). Replacing the one `forward_stderr.then(io::stderr)` with `None` turns the second red. +A child that answers and then exits with a failing status is read as having +answered (see the doc comment of `run_engine_stdio`), and its status is not part +of the answer. The daemon asks the client to leave a note on its stderr, which is +its log (`note_failing_exit_after_answer`); the CLI does not, since its stderr is a +terminal. `a_failing_exit_after_an_answer_is_noted_only_when_asked_for` reads the +stderr of a re-executed test binary for both settings. A failed read of the +answer carries the child's stderr like every other failure +(`a_failed_read_of_the_answer_reports_the_stderr_too`), and a child whose stdin or +stdout pipe is missing is killed and reaped, not left running +(`a_missing_pipe_kills_and_reaps_the_child`, Linux only). + A child whose answer cannot be read is killed and reaped before the error is returned, as it is when the request cannot be written, so the long-lived daemon does not collect a running engine or a zombie per failure. From 5ad422fa9c4307933eed28e5b48d9489bf80a5da Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Tue, 10 Nov 2026 11:31:19 +0000 Subject: [PATCH 34/38] fix(rocm): keep the Windows-only engine_runtime_select imports when dropping apply_app_path_env Removing the cfg(not(windows)) import of apply_app_path_env left its attribute attached to the next use in service_control.rs, service_launch/spawn.rs and service_launch/helper_autostart.rs, which made managed_service_launcher_path unresolved on Windows (and app_path_env_var_values / app_path_env_var_refs in service_control.rs). Only a Windows-target build shows it. The attribute now goes with the line it belonged to, and the whole workspace passes clippy with -D warnings for x86_64-pc-windows-msvc. Signed-off-by: Jussi Elo --- apps/rocm/src/service_control.rs | 1 - apps/rocm/src/service_launch/helper_autostart.rs | 1 - apps/rocm/src/service_launch/spawn.rs | 1 - 3 files changed, 3 deletions(-) diff --git a/apps/rocm/src/service_control.rs b/apps/rocm/src/service_control.rs index bdb3d0038b..d2b9128d2b 100644 --- a/apps/rocm/src/service_control.rs +++ b/apps/rocm/src/service_control.rs @@ -56,7 +56,6 @@ use rocm_engine_protocol::{EngineMethod, StopRequest, StopResponse}; use crate::endpoint_keys; use crate::engine_ipc::{builtin_engine_serve_http_args, engine_request}; -#[cfg(not(windows))] #[cfg(windows)] use crate::engine_runtime_select::{app_path_env_var_refs, app_path_env_var_values}; use crate::engine_runtime_select::{ diff --git a/apps/rocm/src/service_launch/helper_autostart.rs b/apps/rocm/src/service_launch/helper_autostart.rs index 19872240bc..430d152097 100644 --- a/apps/rocm/src/service_launch/helper_autostart.rs +++ b/apps/rocm/src/service_launch/helper_autostart.rs @@ -10,7 +10,6 @@ use super::spawn::{ attach_background_stdio, detach_background_command, managed_service_process_command, }; use crate::background_helper::background_helper_already_running; -#[cfg(not(windows))] use crate::engine_runtime_select::managed_service_launcher_path; #[cfg(windows)] use crate::engine_runtime_select::{app_path_env_var_refs, app_path_env_var_values}; diff --git a/apps/rocm/src/service_launch/spawn.rs b/apps/rocm/src/service_launch/spawn.rs index 3eaafc6c65..9b3252efc0 100644 --- a/apps/rocm/src/service_launch/spawn.rs +++ b/apps/rocm/src/service_launch/spawn.rs @@ -13,7 +13,6 @@ use super::launch_transaction::{ }; use crate::endpoint_keys; use crate::engine_ipc::builtin_engine_serve_http_args; -#[cfg(not(windows))] use crate::engine_runtime_select::managed_service_launcher_path; #[cfg(windows)] use crate::engine_runtime_select::{app_path_env_var_refs, app_path_env_var_values}; From 94cde4b2e72c50b3331f08fd49f59dc968d91b1b Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Tue, 10 Nov 2026 13:04:54 +0000 Subject: [PATCH 35/38] fix(engine-protocol): write the failing-exit note through the SIGPIPE guard and only for a parsed answer; see split-line stdio clients The note added for a child that answers and then fails was written with eprintln!, which is neither SIGPIPE-guarded nor non-panicking: with stderr a closed pipe it would end a daemon hosted in rocm (SIGPIPE at its default action) or panic one that ignores the signal, instead of returning the answer. It now goes through the same guarded, error-ignoring write as the stderr forwarding. It also said "the answer was used" as soon as stdout was non-empty, which is false when the output is unreadable and the child failed. The note is now made after the envelope has parsed; an unparseable answer is reported by the existing error, with the status. One test asserts the outcome and the stderr together for both cases, and another runs with stderr a pipe without a reader under SIGPIPE's default action and requires the answer back and the process alive. Each was shown red by removing the behaviour (a plain eprint, and noting without the parse check). The one-client contract scan looked for .stdin( and piped() on the same line, so a Stdio built in one place and handed to .stdin(..) in another (a shape two helpers in the tree already use) went unseen. It now looks for the ingredients anywhere in a file, with a fixture for the split shape that fails under the per-line check; the module comment and docs/testing.md state the coarser trade-off and the limits that remain. Signed-off-by: Jussi Elo --- .../rocm-engine-protocol/src/stdio_client.rs | 118 ++++++++++++++++-- docs/testing.md | 15 ++- xtask/src/engine_stdio_client_contract.rs | 29 ++++- 3 files changed, 150 insertions(+), 12 deletions(-) diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 802bb438b4..a149191abe 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -260,12 +260,6 @@ where .unwrap_or_default(); let stderr = String::from_utf8_lossy(&stderr).trim().to_owned(); - if !status.success() && !stdout.is_empty() && options.note_failing_exit_after_answer { - eprintln!( - "{} after printing its answer; the answer was used", - exit_note(status) - ); - } if !status.success() && stdout.is_empty() { let exited = exit_note(status); if stderr.is_empty() { @@ -273,7 +267,26 @@ where } bail!("{exited}: {stderr}"); } - serde_json::from_slice(&stdout).with_context(|| unparseable_answer_message(status, &stderr)) + let answer = serde_json::from_slice(&stdout) + .with_context(|| unparseable_answer_message(status, &stderr)); + // The note says the answer was used, so it is only made for an answer that + // parsed; an unparseable one is reported by the error above, with the status. + if answer.is_ok() && !status.success() && options.note_failing_exit_after_answer { + note_failing_exit_after_answer(&mut io::stderr(), status); + } + answer +} + +/// Say on `sink` (this process's stderr, in production) that the child answered +/// and then failed. Written like the stderr forwarding, through the SIGPIPE guard +/// and ignoring a failed write: a closed stderr must neither end a daemon hosted +/// in `rocm` (SIGPIPE at its default action) nor panic it, as `eprintln!` would. +fn note_failing_exit_after_answer(sink: &mut impl Write, status: ExitStatus) { + let note = format!( + "{} after printing its answer; the answer was used\n", + exit_note(status) + ); + forward_best_effort(sink, note.as_bytes()); } /// How a failing exit is worded wherever the status is reported. Built from the @@ -1067,6 +1080,97 @@ mod tests { } } + /// The note is only made for an answer that parsed: a child that prints + /// something unreadable and fails is reported by the error (with its + /// status), and must not also log that "the answer was used". Both cases + /// assert the outcome and the stderr together, in a re-executed binary. + #[test] + fn the_note_is_made_only_for_an_answer_that_parsed() { + if let Some(arm) = std::env::var_os("ROCM_TEST_CLIENT_NOTE_PARSE_ARM") { + let (body, status) = if arm == "parsed" { + (format!("printf '%s' '{STOPPED}'"), 7) + } else { + ("printf 'not json'".to_owned(), 7) + }; + let engine = + FakeEngine::new("client-note-parse", true, &format!("{body}\nexit {status}")); + let options = EngineStdioOptions { + note_failing_exit_after_answer: true, + ..EngineStdioOptions::default() + }; + let outcome = run(&engine, &options); + if arm == "parsed" { + assert_eq!(outcome.unwrap(), json!({"stopped": true})); + } else { + assert_eq!( + outcome.unwrap_err().to_string(), + "failed to parse engine response envelope \ + (engine stdio process exited with code 7)" + ); + } + return; + } + for (arm, expected) in [("parsed", true), ("unparseable", false)] { + let output = Command::new(std::env::current_exe().unwrap()) + .args([ + "stdio_client::tests::child::the_note_is_made_only_for_an_answer_that_parsed", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_CLIENT_NOTE_PARSE_ARM", arm) + .output() + .unwrap(); + assert!(output.status.success(), "{arm}: {:?}", output.status); + let stderr = String::from_utf8_lossy(&output.stderr); + assert_eq!( + stderr.contains("the answer was used"), + expected, + "{arm}:\n{stderr}" + ); + } + } + + /// With stderr a closed pipe and SIGPIPE at its default action (how `rocm` + /// runs, and the daemon with it), the note must not end the process or + /// panic: the answer is still returned. Re-executes this test in a process + /// whose stderr is a pipe with no reader. + #[test] + #[allow(unsafe_code)] // libc::signal, to install the default action in the child + fn the_note_to_a_closed_stderr_neither_kills_nor_panics_the_process() { + if std::env::var_os("ROCM_TEST_CLIENT_NOTE_CLOSED_ARM").is_some() { + // SAFETY: installing a signal disposition is always safe. + unsafe { + libc::signal(libc::SIGPIPE, libc::SIG_DFL); + } + let engine = FakeEngine::answering("client-note-closed", STOPPED, 7); + let options = EngineStdioOptions { + note_failing_exit_after_answer: true, + ..EngineStdioOptions::default() + }; + let answer = run(&engine, &options).expect("the answer is still returned"); + assert_eq!(answer, json!({"stopped": true})); + return; + } + let (reader, writer) = io::pipe().unwrap(); + drop(reader); + let status = Command::new(std::env::current_exe().unwrap()) + .args([ + "stdio_client::tests::child::the_note_to_a_closed_stderr_neither_kills_nor_panics_the_process", + "--exact", + "--nocapture", + ]) + .env("ROCM_TEST_CLIENT_NOTE_CLOSED_ARM", "1") + .stderr(Stdio::from(writer)) + .status() + .unwrap(); + assert_eq!( + status.signal(), + None, + "SIGPIPE ended the process: {status:?}" + ); + assert!(status.success(), "{status:?}"); + } + /// A child whose stdin or stdout pipe is missing is killed and reaped, as /// on every other failure path, not left running. Linux-only for the /// `/proc` entry a zombie would keep. diff --git a/docs/testing.md b/docs/testing.md index efe173d880..39b31e6eff 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -292,7 +292,11 @@ second copy: is a static text scan over every `.rs` file in the tree (comments stripped) that fails when a file other than the shared client both pipes a child's stdin and names an engine request or response envelope, which is what an engine stdio -client is made of. A caller that reaches an engine through `run_engine_stdio` +client is made of. Both are looked for anywhere in the file, not on one line, so +a copy that builds its `Stdio` in one place and passes it to `.stdin(..)` in +another is seen too (a fixture covers it); the price is that a file which pipes +some child's stdin for another reason and also names an envelope type is +reported, and the answer to that is a reviewed allowlist entry. A caller that reaches an engine through `run_engine_stdio` has neither. Pasting the old rocmd copy back into `apps/rocmd/src/engine_ipc.rs` fails it by name (and, separately, five of rocmd's pinning tests: the app-paths, crash, unparseable-answer, nonzero-with-answer and SIGPIPE ones). Like @@ -350,7 +354,14 @@ answered (see the doc comment of `run_engine_stdio`), and its status is not part of the answer. The daemon asks the client to leave a note on its stderr, which is its log (`note_failing_exit_after_answer`); the CLI does not, since its stderr is a terminal. `a_failing_exit_after_an_answer_is_noted_only_when_asked_for` reads the -stderr of a re-executed test binary for both settings. A failed read of the +stderr of a re-executed test binary for both settings. The note is made only +for an answer that parsed (an unreadable answer with a failing exit is reported +by the error, with the status, and is not also logged as used), and it is written +through the same SIGPIPE-guarded, error-ignoring write as the stderr forwarding, +so a closed stderr neither ends a daemon hosted in `rocm` nor panics it +(`the_note_is_made_only_for_an_answer_that_parsed` and +`the_note_to_a_closed_stderr_neither_kills_nor_panics_the_process`, the latter +with stderr a pipe without a reader under SIGPIPE's default action). A failed read of the answer carries the child's stderr like every other failure (`a_failed_read_of_the_answer_reports_the_stderr_too`), and a child whose stdin or stdout pipe is missing is killed and reaped, not left running diff --git a/xtask/src/engine_stdio_client_contract.rs b/xtask/src/engine_stdio_client_contract.rs index 81f2709a12..f311abe55c 100644 --- a/xtask/src/engine_stdio_client_contract.rs +++ b/xtask/src/engine_stdio_client_contract.rs @@ -21,6 +21,12 @@ //! client has to either use the shared one or add itself to [`ALLOWED`] in a //! reviewed diff that says why, as for [`crate::crate_edges::ALLOWLIST`]. //! +//! The two ingredients are looked for anywhere in a file, not on one line, so a +//! copy that builds its `Stdio` in one place and passes it to `.stdin(..)` in +//! another is still seen. That makes the scan coarse on purpose: a file that +//! pipes some child's stdin for any reason and also names an envelope type is +//! reported, and the answer to a false positive is a reviewed [`ALLOWED`] entry. +//! //! What this does **not** claim: it is a source-text scan, not a call-graph //! analysis. A copy that builds its stdin another way (a pipe made by hand), or //! reaches the envelope types under another name, is outside what it can see, @@ -117,9 +123,11 @@ mod tests { /// ignoring comments: the two ingredients of an engine stdio client. fn is_a_client(text: &str) -> bool { let code: Vec<&str> = text.lines().map(strip_comment).collect(); - let pipes_stdin = code - .iter() - .any(|line| line.contains(".stdin(") && line.contains("piped()")); + // Anywhere in the file, not on one line: a `Stdio` built on one line and + // handed to `.stdin(..)` on another (a helper that takes it as a + // parameter) is the same ingredient. + let pipes_stdin = code.iter().any(|line| line.contains(".stdin(")) + && code.iter().any(|line| line.contains("piped()")); let names_envelope = code .iter() .any(|line| ENVELOPE_TOKENS.iter().any(|token| line.contains(token))); @@ -211,6 +219,21 @@ fn engine_request(engine_binary: &Path, envelope: &EngineRequestEnvelope) -> Res assert!(violations[0].contains("run_engine_stdio")); } + #[test] + fn a_copy_that_builds_its_stdio_on_one_line_and_uses_it_on_another_is_recognised() { + let split = " +fn spawn_engine(engine: &Path, stdin: Stdio) -> Result { + Ok(Command::new(engine).stdin(stdin).stdout(Stdio::null()).spawn()?) +} +fn ask(engine: &Path, envelope: &EngineRequestEnvelope) -> Result<()> { + let mut child = spawn_engine(engine, Stdio::piped())?; + serde_json::to_writer(child.stdin.as_mut().unwrap(), envelope)?; + Ok(()) +} +"; + assert!(is_a_client(split)); + } + #[test] fn a_caller_of_the_shared_client_is_not_a_client() { let caller = " From 6d83971a2b2c4c03bab4201aacd16dacde6d2f2f Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Tue, 10 Nov 2026 14:38:01 +0000 Subject: [PATCH 36/38] fix(engine-protocol): forward only complete UTF-8 so a split character is not lost on a Windows console The stderr collector publishes each chunk it reads as it arrives and forwarded the same chunk. A chunk can end in the middle of a multi-byte character, and Rust's stdio on a Windows console refuses a write that is not valid UTF-8; the best-effort forwarding discards that error, so valid progress or diagnostic output was silently lost. A small pure type, Utf8Carry, now sits between the collector and the sink: it forwards the complete UTF-8 of what has arrived, carries an incomplete trailing sequence to the next chunk, replaces bytes that can never be text with U+FFFD, and flushes what is still carried at the end of the stream lossily. The kept copy used for errors is still the raw bytes, published at once, so an unterminated diagnostic is still reported when the grace period expires. Tests run on every platform: every two-point split of a string with two-, three- and four-byte characters is forwarded whole and valid at each boundary; invalid bytes and a cut-off tail are replaced and flushed; and the collector, fed one byte at a time, forwards the characters whole and keeps the raw bytes. Forwarding raw chunks turns all three red, and dropping the final flush turns the tail test red. Signed-off-by: Jussi Elo --- .../rocm-engine-protocol/src/stdio_client.rs | 155 +++++++++++++++++- docs/testing.md | 14 ++ 2 files changed, 164 insertions(+), 5 deletions(-) diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index a149191abe..6ce21d326e 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -508,6 +508,7 @@ fn collect_stderr_into( shared: &Mutex, ) { let mut chunk = [0_u8; 4096]; + let mut carry = Utf8Carry::default(); loop { let bytes: Vec = match stderr.read(&mut chunk) { Ok(0) => break, @@ -515,16 +516,29 @@ fn collect_stderr_into( Err(error) if error.kind() == io::ErrorKind::Interrupted => continue, Err(error) => { let message = format!("failed to read engine progress: {error}\n").into_bytes(); - publish(shared, forward.as_mut(), &message); + publish(shared, forward.as_mut(), &mut carry, &message); break; } }; - publish(shared, forward.as_mut(), &bytes); + publish(shared, forward.as_mut(), &mut carry, &bytes); + } + // What the pipe ended in the middle of a character of is forwarded lossily. + if let Some(sink) = forward.as_mut() + && !lock(shared).abandoned + { + forward_best_effort(sink, &carry.finish()); } } -/// Keep `bytes` and forward them, unless the capture has been abandoned. -fn publish(shared: &Mutex, forward: Option<&mut impl Write>, bytes: &[u8]) { +/// Keep `bytes` and forward the complete UTF-8 of them, unless the capture has +/// been abandoned. The kept copy is the raw bytes; only what is forwarded is +/// held back at an incomplete trailing character. +fn publish( + shared: &Mutex, + forward: Option<&mut impl Write>, + carry: &mut Utf8Carry, + bytes: &[u8], +) { { let mut captured = lock(shared); if captured.abandoned { @@ -533,7 +547,61 @@ fn publish(shared: &Mutex, forward: Option<&mut impl Write>, bytes: &[ captured.bytes.extend_from_slice(bytes); } if let Some(sink) = forward { - forward_best_effort(sink, bytes); + let complete = carry.push(bytes); + if !complete.is_empty() { + forward_best_effort(sink, &complete); + } + } +} + +/// Turns arbitrary chunks of a byte stream into text that is valid UTF-8 at every +/// boundary, for a sink that requires it. Rust's stdio on a Windows console +/// refuses a write that is not valid UTF-8, so a chunk that ends in the middle of +/// a multi-byte character would fail, and the forwarding discards that error, +/// losing valid output. The incomplete trailing sequence is carried to the next +/// chunk instead; bytes that can never be valid are replaced with U+FFFD, and +/// what is still carried when the stream ends is flushed the same way. +#[derive(Default)] +struct Utf8Carry { + pending: Vec, +} + +impl Utf8Carry { + /// The part of `pending + bytes` that is complete UTF-8 (with invalid bytes + /// replaced), keeping an incomplete trailing sequence for the next call. + fn push(&mut self, bytes: &[u8]) -> Vec { + self.pending.extend_from_slice(bytes); + let mut out = Vec::with_capacity(self.pending.len()); + let mut rest = std::mem::take(&mut self.pending); + loop { + match std::str::from_utf8(&rest) { + Ok(valid) => { + out.extend_from_slice(valid.as_bytes()); + return out; + } + Err(error) => { + let valid_up_to = error.valid_up_to(); + out.extend_from_slice(&rest[..valid_up_to]); + match error.error_len() { + // The input ends inside a character: wait for the rest. + None => { + self.pending = rest[valid_up_to..].to_vec(); + return out; + } + Some(len) => { + out.extend_from_slice("\u{FFFD}".as_bytes()); + rest.drain(..valid_up_to + len); + } + } + } + } + } + } + + /// What is carried at the end of the stream, replaced lossily. + fn finish(&mut self) -> Vec { + let pending = std::mem::take(&mut self.pending); + String::from_utf8_lossy(&pending).into_owned().into_bytes() } } @@ -805,6 +873,83 @@ mod tests { ); } + /// The multi-byte text the tests split: two, three and four byte characters. + const MULTIBYTE: &str = "caf\u{e9} \u{2192} \u{1f4e6} done\n"; + + /// A stream split at every possible boundary is forwarded whole: every chunk + /// handed to the sink is valid UTF-8 on its own, and together they are the + /// input. Pure, so it runs on the Windows job too. + #[test] + fn text_split_across_chunks_is_forwarded_whole_and_valid_at_every_boundary() { + let bytes = MULTIBYTE.as_bytes(); + for first in 1..bytes.len() { + for second in first..bytes.len() { + let mut carry = Utf8Carry::default(); + let mut forwarded = Vec::new(); + for part in [&bytes[..first], &bytes[first..second], &bytes[second..]] { + let out = carry.push(part); + assert!( + std::str::from_utf8(&out).is_ok(), + "split {first}/{second}: a chunk was not valid UTF-8: {out:?}" + ); + forwarded.extend(out); + } + forwarded.extend(carry.finish()); + assert_eq!(forwarded, bytes, "split {first}/{second}: bytes were lost"); + } + } + } + + /// Bytes that can never be text are replaced, and a character cut off by the + /// end of the stream is flushed lossily, never dropped. + #[test] + fn bytes_that_are_not_text_are_replaced_and_a_cut_off_tail_is_flushed() { + let mut carry = Utf8Carry::default(); + assert_eq!( + carry.push(b"ok \xff\xfe done"), + "ok \u{fffd}\u{fffd} done".as_bytes() + ); + // A lead byte of a three-byte character with nothing after it. + assert_eq!(carry.push(b"tail \xe2\x86"), b"tail "); + assert_eq!(carry.finish(), "\u{fffd}".as_bytes()); + assert!(carry.finish().is_empty()); + } + + /// The collector forwards a character split across reads in one piece, and + /// keeps the raw bytes for the error. The source hands out one byte at a time. + #[test] + fn the_collector_forwards_a_split_character_whole_and_keeps_the_raw_bytes() { + struct OneByte(std::vec::IntoIter); + impl Read for OneByte { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + match self.0.next() { + Some(byte) => { + buffer[0] = byte; + Ok(1) + } + None => Ok(0), + } + } + } + let chunks = std::cell::RefCell::new(Vec::>::new()); + let sink = FnSink(|bytes: &[u8]| { + chunks.borrow_mut().push(bytes.to_vec()); + Ok(bytes.len()) + }); + let collected = collect_stderr( + OneByte(MULTIBYTE.as_bytes().to_vec().into_iter()), + Some(sink), + ); + assert_eq!(collected, MULTIBYTE.as_bytes()); + let chunks = chunks.into_inner(); + assert!( + chunks + .iter() + .all(|chunk| std::str::from_utf8(chunk).is_ok()) + ); + assert_eq!(chunks.concat(), MULTIBYTE.as_bytes()); + } + /// A status that is neither an exit nor a signal (a stopped process, on /// Unix) has neither a code nor a signal to name, and is reported as that. #[cfg(unix)] diff --git a/docs/testing.md b/docs/testing.md index 39b31e6eff..d019fdeba8 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -367,6 +367,20 @@ answer carries the child's stderr like every other failure stdout pipe is missing is killed and reaped, not left running (`a_missing_pipe_kills_and_reaps_the_child`, Linux only). +What is forwarded to the terminal is complete UTF-8: stderr is read in chunks +that can end in the middle of a multi-byte character, and Rust's stdio on a +Windows console refuses a write that is not valid UTF-8, an error the best-effort +forwarding discards, so a split character would silently lose output. The +collector carries an incomplete trailing sequence to the next chunk (invalid +bytes become U+FFFD, and a character cut off by the end of the stream is flushed +lossily) while still publishing the raw bytes at once for the error. The logic is +the pure `Utf8Carry`, tested on every platform by +`text_split_across_chunks_is_forwarded_whole_and_valid_at_every_boundary` (every +two-point split of a string with two-, three- and four-byte characters), +`bytes_that_are_not_text_are_replaced_and_a_cut_off_tail_is_flushed` and +`the_collector_forwards_a_split_character_whole_and_keeps_the_raw_bytes` (a source +that hands out one byte at a time). + A child whose answer cannot be read is killed and reaped before the error is returned, as it is when the request cannot be written, so the long-lived daemon does not collect a running engine or a zombie per failure. From 6b9a6f9de68c552e423ce0bf254883ff8c33507e Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Tue, 10 Nov 2026 14:42:31 +0000 Subject: [PATCH 37/38] fix(engine-protocol): keep the replacement character out of a production string literal The output contract fails a non-ASCII literal in production code, and the escape for U+FFFD in the UTF-8 carry counted as one. The replacement is now its three UTF-8 bytes as a named constant. Behaviour is unchanged and the carry tests pin it. Signed-off-by: Jussi Elo --- crates/rocm-engine-protocol/src/stdio_client.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 6ce21d326e..5a8989e5a7 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -554,6 +554,10 @@ fn publish( } } +/// U+FFFD in UTF-8, as bytes: a literal here would be a non-ASCII string in +/// production code, which the output contract forbids. +const REPLACEMENT_CHARACTER: &[u8] = &[0xEF, 0xBF, 0xBD]; + /// Turns arbitrary chunks of a byte stream into text that is valid UTF-8 at every /// boundary, for a sink that requires it. Rust's stdio on a Windows console /// refuses a write that is not valid UTF-8, so a chunk that ends in the middle of @@ -589,7 +593,7 @@ impl Utf8Carry { return out; } Some(len) => { - out.extend_from_slice("\u{FFFD}".as_bytes()); + out.extend_from_slice(REPLACEMENT_CHARACTER); rest.drain(..valid_up_to + len); } } From 60cb59e818ca91dc6f3386f2b968ea4267d91ee8 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Tue, 10 Nov 2026 16:06:40 +0000 Subject: [PATCH 38/38] fix(rocm-core): drop the duplicated test attribute left by the main merge in paths.rs Signed-off-by: Jussi Elo --- crates/rocm-core/src/paths.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/crates/rocm-core/src/paths.rs b/crates/rocm-core/src/paths.rs index c1daf3e2bc..ee4a6f1a07 100644 --- a/crates/rocm-core/src/paths.rs +++ b/crates/rocm-core/src/paths.rs @@ -646,7 +646,6 @@ mod tests { ); } - #[test] /// For every `config.json` that `load` accepts, discovery reads the /// recorded folder as `load` does: the saved `therock_venv`, trimmed, and /// nothing when it is absent or blank.