From d9019c1d8e6655d557b6af82df888bba6cb4eb68 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 2 Nov 2026 10:45:27 +0000 Subject: [PATCH 01/13] refactor(rocmd): move the sandbox-tool vocabulary into sandbox_tool.rs Move SandboxToolArg (with its Linux-only writes_data/writes_cache), SandboxToolPolicy and resolve_huggingface_token out of cli.rs into a new leaf module. sandbox.rs, service.rs, watchers.rs and the lib.rs tests now import it from the new module, and cli.rs imports it for its Command fields, so those modules no longer depend on cli.rs; no re-export is left behind in cli.rs. Part of the rocmd module-cycle follow-up work. Other than the moved lines, the module declaration and the import-path edits, the only change is the docs/architecture.md entry for the new file (and dropping the moved types from the cli.rs entry). No tests move: none target these types directly. Signed-off-by: Jussi Elo --- apps/rocmd/src/cli.rs | 84 +-------------------------------- apps/rocmd/src/lib.rs | 5 +- apps/rocmd/src/sandbox.rs | 2 +- apps/rocmd/src/sandbox_tool.rs | 86 ++++++++++++++++++++++++++++++++++ apps/rocmd/src/service.rs | 2 +- apps/rocmd/src/watchers.rs | 12 ++--- docs/architecture.md | 3 +- 7 files changed, 101 insertions(+), 93 deletions(-) create mode 100644 apps/rocmd/src/sandbox_tool.rs diff --git a/apps/rocmd/src/cli.rs b/apps/rocmd/src/cli.rs index 10fa91d81c..0c8941876b 100644 --- a/apps/rocmd/src/cli.rs +++ b/apps/rocmd/src/cli.rs @@ -2,8 +2,9 @@ // // SPDX-License-Identifier: MIT +use crate::sandbox_tool::{SandboxToolArg, SandboxToolPolicy}; use anyhow::{Context, Result, bail}; -use clap::{Parser, Subcommand, ValueEnum}; +use clap::{Parser, Subcommand}; use rocm_core::{AppPaths, DEFAULT_LOCAL_HOST}; use serde_json::Value; use serde_json::json; @@ -118,72 +119,6 @@ pub(crate) enum Command { }, } -#[derive(Debug, Clone, Copy, Eq, PartialEq, ValueEnum)] -#[value(rename_all = "snake_case")] -pub(crate) enum SandboxToolArg { - CheckUpdates, - DriverPlan, - ExamineSnapshot, - ListServers, - RestartServer, - StopServer, - PrefetchArtifact, - NotifyUser, -} - -impl SandboxToolArg { - pub(crate) const fn as_cli_value(self) -> &'static str { - match self { - Self::CheckUpdates => "check_updates", - Self::DriverPlan => "driver_plan", - Self::ExamineSnapshot => "examine_snapshot", - Self::ListServers => "list_servers", - Self::RestartServer => "restart_server", - Self::StopServer => "stop_server", - Self::PrefetchArtifact => "prefetch_artifact", - Self::NotifyUser => "notify_user", - } - } - - #[cfg(target_os = "linux")] - pub(crate) const fn writes_data(self) -> bool { - matches!( - self, - Self::RestartServer | Self::StopServer | Self::NotifyUser - ) - } - - #[cfg(target_os = "linux")] - pub(crate) const fn writes_cache(self) -> bool { - matches!(self, Self::CheckUpdates | Self::PrefetchArtifact) - } -} - -#[derive(Debug, Clone, Default)] -pub(crate) struct SandboxToolPolicy { - pub(crate) allow_artifact_download: bool, - pub(crate) artifact_max_bytes: Option, - pub(crate) allow_huggingface_download: bool, - pub(crate) huggingface_token: Option, -} - -impl SandboxToolPolicy { - pub(crate) fn from_cli( - allow_artifact_download: bool, - artifact_max_bytes: Option, - allow_huggingface_download: bool, - ) -> Self { - Self { - allow_artifact_download, - artifact_max_bytes, - allow_huggingface_download, - huggingface_token: allow_huggingface_download - .then(resolve_huggingface_token) - .flatten(), - } - } -} - fn print_bridge_snapshot(paths: &AppPaths, pretty: bool) -> Result<()> { let snapshot = crate::common::build_bridge_snapshot(paths)?; @@ -203,21 +138,6 @@ fn print_bridge_snapshot(paths: &AppPaths, pretty: bool) -> Result<()> { Ok(()) } -fn resolve_huggingface_token() -> Option { - [ - "ROCM_CLI_HUGGINGFACE_TOKEN", - "HF_TOKEN", - "HUGGING_FACE_HUB_TOKEN", - ] - .iter() - .find_map(|name| { - std::env::var(name) - .ok() - .map(|value| value.trim().to_owned()) - .filter(|value| !value.is_empty()) - }) -} - #[tokio::main] pub async fn run_bin_cli() -> Result<()> { let cli = Cli::parse(); diff --git a/apps/rocmd/src/lib.rs b/apps/rocmd/src/lib.rs index 3333461903..917303bc05 100644 --- a/apps/rocmd/src/lib.rs +++ b/apps/rocmd/src/lib.rs @@ -9,6 +9,7 @@ mod common; mod mcp; mod persistence; mod sandbox; +mod sandbox_tool; mod service; #[cfg(test)] mod test_support; @@ -61,11 +62,11 @@ mod tests { let value = sandbox::run_sandbox_tool( &paths, - cli::SandboxToolArg::StopServer, + sandbox_tool::SandboxToolArg::StopServer, Some("svc-current".to_owned()), None, None, - cli::SandboxToolPolicy::default(), + sandbox_tool::SandboxToolPolicy::default(), )?; let reloaded = watchers::load_service_record(&paths, "svc-current")?; fs::remove_dir_all(root).ok(); diff --git a/apps/rocmd/src/sandbox.rs b/apps/rocmd/src/sandbox.rs index d7b0e0dc93..951c18abf8 100644 --- a/apps/rocmd/src/sandbox.rs +++ b/apps/rocmd/src/sandbox.rs @@ -3,9 +3,9 @@ // SPDX-License-Identifier: MIT use crate::ARTIFACT_PREFETCH_TIMEOUT; -use crate::cli::{SandboxToolArg, SandboxToolPolicy}; use crate::common; use crate::persistence::load_managed_services; +use crate::sandbox_tool::{SandboxToolArg, SandboxToolPolicy}; use crate::service::{sandbox_stop_server_value, stop_managed_service}; use crate::watchers::restart_managed_service; use anyhow::{Context, Result, bail}; diff --git a/apps/rocmd/src/sandbox_tool.rs b/apps/rocmd/src/sandbox_tool.rs new file mode 100644 index 0000000000..107b66f155 --- /dev/null +++ b/apps/rocmd/src/sandbox_tool.rs @@ -0,0 +1,86 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +use clap::ValueEnum; + +#[derive(Debug, Clone, Copy, Eq, PartialEq, ValueEnum)] +#[value(rename_all = "snake_case")] +pub(crate) enum SandboxToolArg { + CheckUpdates, + DriverPlan, + ExamineSnapshot, + ListServers, + RestartServer, + StopServer, + PrefetchArtifact, + NotifyUser, +} + +impl SandboxToolArg { + pub(crate) const fn as_cli_value(self) -> &'static str { + match self { + Self::CheckUpdates => "check_updates", + Self::DriverPlan => "driver_plan", + Self::ExamineSnapshot => "examine_snapshot", + Self::ListServers => "list_servers", + Self::RestartServer => "restart_server", + Self::StopServer => "stop_server", + Self::PrefetchArtifact => "prefetch_artifact", + Self::NotifyUser => "notify_user", + } + } + + #[cfg(target_os = "linux")] + pub(crate) const fn writes_data(self) -> bool { + matches!( + self, + Self::RestartServer | Self::StopServer | Self::NotifyUser + ) + } + + #[cfg(target_os = "linux")] + pub(crate) const fn writes_cache(self) -> bool { + matches!(self, Self::CheckUpdates | Self::PrefetchArtifact) + } +} + +#[derive(Debug, Clone, Default)] +pub(crate) struct SandboxToolPolicy { + pub(crate) allow_artifact_download: bool, + pub(crate) artifact_max_bytes: Option, + pub(crate) allow_huggingface_download: bool, + pub(crate) huggingface_token: Option, +} + +impl SandboxToolPolicy { + pub(crate) fn from_cli( + allow_artifact_download: bool, + artifact_max_bytes: Option, + allow_huggingface_download: bool, + ) -> Self { + Self { + allow_artifact_download, + artifact_max_bytes, + allow_huggingface_download, + huggingface_token: allow_huggingface_download + .then(resolve_huggingface_token) + .flatten(), + } + } +} + +fn resolve_huggingface_token() -> Option { + [ + "ROCM_CLI_HUGGINGFACE_TOKEN", + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", + ] + .iter() + .find_map(|name| { + std::env::var(name) + .ok() + .map(|value| value.trim().to_owned()) + .filter(|value| !value.is_empty()) + }) +} diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index 79793d1f8a..2eb26a4484 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -224,7 +224,7 @@ pub(crate) fn sandbox_stop_server_value(stopped: Value) -> Value { "stop_unconfirmed" }; json!({ - "tool": crate::cli::SandboxToolArg::StopServer.as_cli_value(), + "tool": crate::sandbox_tool::SandboxToolArg::StopServer.as_cli_value(), "status": status, "mutating": true, "result": stopped, diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index 5ec222daa4..8155e10e78 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -237,11 +237,11 @@ fn handle_therock_update_event( handle_therock_update_event_with_runner(paths, mode, state, event, |paths| { crate::sandbox::run_sandbox_tool( paths, - crate::cli::SandboxToolArg::CheckUpdates, + crate::sandbox_tool::SandboxToolArg::CheckUpdates, None, None, None, - crate::cli::SandboxToolPolicy::default(), + crate::sandbox_tool::SandboxToolPolicy::default(), ) }) } @@ -362,7 +362,7 @@ fn restricted_check_updates_result(value: &Value) -> Result Result Date: Mon, 2 Nov 2026 10:48:51 +0000 Subject: [PATCH 02/13] refactor(rocmd): move stop-server result shaping into sandbox Move sandbox_stop_server_value out of service.rs and ARTIFACT_PREFETCH_TIMEOUT out of lib.rs into sandbox.rs, together with the two lib.rs tests that exercise the stop_server tool (sandbox_tool_stop_server_updates_manifest_and_skips_current_pid and sandbox_tool_stop_server_reports_an_unconfirmed_stop). run_sandbox_tool was the only production caller of the function, and it was the only thing that made service.rs depend on the sandbox-tool types; stop_report_confirmed stays in service.rs because mcp.rs also reads it. Part of the rocmd module-cycle follow-up work. Besides the moved lines and import wiring, this commit changes: sandbox_stop_server_value and ARTIFACT_PREFETCH_TIMEOUT become private (their only users are now in sandbox.rs); the two moved tests drop the cli::/sandbox::/watchers:: path prefixes now that they sit beside the code; lib.rs loses its emptied test module; docs/architecture.md gains stop_server in the sandbox.rs entry and drops the constant and test module from the lib.rs entry. Signed-off-by: Jussi Elo --- apps/rocmd/src/lib.rs | 101 --------------------------------- apps/rocmd/src/sandbox.rs | 114 +++++++++++++++++++++++++++++++++++++- apps/rocmd/src/service.rs | 17 ------ docs/architecture.md | 4 +- 4 files changed, 113 insertions(+), 123 deletions(-) diff --git a/apps/rocmd/src/lib.rs b/apps/rocmd/src/lib.rs index 917303bc05..0e3008dda6 100644 --- a/apps/rocmd/src/lib.rs +++ b/apps/rocmd/src/lib.rs @@ -21,104 +21,3 @@ pub use cli::{parse_args, run_bin_cli, run_from_args}; use std::time::Duration; const WATCHER_TICK_INTERVAL: Duration = Duration::from_secs(5); -const ARTIFACT_PREFETCH_TIMEOUT: Duration = Duration::from_mins(10); - -#[cfg(test)] -mod tests { - use super::*; - use crate::service::sandbox_stop_server_value; - use crate::test_support::{ - assert_unconfirmed_stop_kept_the_service, seed_keyed_service, stop_with_outcome, - temp_app_paths, - }; - use crate::watchers::load_service_record; - use anyhow::Result; - use rocm_core::ManagedServiceRecord; - use serde_json::Value; - use std::fs; - - #[test] - fn sandbox_tool_stop_server_updates_manifest_and_skips_current_pid() -> Result<()> { - let (root, paths) = temp_app_paths("sandbox-stop-current-pid"); - paths.ensure()?; - let current_pid = std::process::id(); - let mut record = ManagedServiceRecord::new( - &paths, - "svc-current", - "vllm", - "qwen", - "Qwen/Qwen3.5", - "127.0.0.1", - 11435, - "managed", - current_pid, - None, - None, - None, - ); - record.engine_pid = Some(current_pid); - record.status = "ready".to_owned(); - record.write()?; - - let value = sandbox::run_sandbox_tool( - &paths, - sandbox_tool::SandboxToolArg::StopServer, - Some("svc-current".to_owned()), - None, - None, - sandbox_tool::SandboxToolPolicy::default(), - )?; - let reloaded = watchers::load_service_record(&paths, "svc-current")?; - fs::remove_dir_all(root).ok(); - - assert_eq!(value.get("status").and_then(Value::as_str), Some("stopped")); - assert_eq!(value.get("mutating").and_then(Value::as_bool), Some(true)); - assert_eq!(reloaded.status, "stopped"); - assert!( - value - .get("result") - .and_then(|result| result.get("skipped_pids")) - .and_then(Value::as_array) - .is_some_and(|pids| pids - .iter() - .any(|pid| pid.as_u64() == Some(u64::from(current_pid)))) - ); - Ok(()) - } - - /// The sandbox tool used to report `status: stopped` unconditionally. - #[test] - fn sandbox_tool_stop_server_reports_an_unconfirmed_stop() -> Result<()> { - let (root, paths) = temp_app_paths("sandbox-stop-unconfirmed"); - paths.ensure()?; - let service_id = "svc-sandbox-stop-unconfirmed"; - let key_path = seed_keyed_service(&paths, service_id, 11448)?; - - // The tool is `stop_managed_service` feeding `sandbox_stop_server_value`; - // `sandbox_tool_stop_server_updates_manifest_and_skips_current_pid` - // drives that whole tool for real. - let result = stop_with_outcome( - &paths, - service_id, - rocm_core::TerminationOutcome::Unverified, - ) - .map(sandbox_stop_server_value); - let key_kept = key_path.exists(); - let reloaded = load_service_record(&paths, service_id); - fs::remove_dir_all(root).ok(); - - let value = result?; - assert_eq!( - value.get("status").and_then(Value::as_str), - Some("stop_unconfirmed"), - "{value}" - ); - assert_eq!( - value.pointer("/result/stopped").and_then(Value::as_bool), - Some(false), - "{value}" - ); - assert_unconfirmed_stop_kept_the_service(&reloaded?, key_kept); - Ok(()) - } -} diff --git a/apps/rocmd/src/sandbox.rs b/apps/rocmd/src/sandbox.rs index 951c18abf8..229fe9c0af 100644 --- a/apps/rocmd/src/sandbox.rs +++ b/apps/rocmd/src/sandbox.rs @@ -2,11 +2,10 @@ // // SPDX-License-Identifier: MIT -use crate::ARTIFACT_PREFETCH_TIMEOUT; use crate::common; use crate::persistence::load_managed_services; use crate::sandbox_tool::{SandboxToolArg, SandboxToolPolicy}; -use crate::service::{sandbox_stop_server_value, stop_managed_service}; +use crate::service::{stop_managed_service, stop_report_confirmed}; use crate::watchers::restart_managed_service; use anyhow::{Context, Result, bail}; use rocm_core::{ @@ -30,6 +29,8 @@ use std::process::{Command as ProcessCommand, Stdio}; use std::thread; use std::time::Duration; +const ARTIFACT_PREFETCH_TIMEOUT: Duration = Duration::from_mins(10); + pub(crate) fn run_sandbox_runner( paths: &AppPaths, tool: SandboxToolArg, @@ -256,6 +257,23 @@ pub(crate) fn run_sandbox_tool( } } +/// The `stop_server` sandbox tool's answer for a [`stop_managed_service`] +/// report. The status is taken from the same verdict that decided what the +/// stop wrote to the manifest, so the two cannot disagree. +fn sandbox_stop_server_value(stopped: Value) -> Value { + let status = if stop_report_confirmed(&stopped) { + "stopped" + } else { + "stop_unconfirmed" + }; + json!({ + "tool": SandboxToolArg::StopServer.as_cli_value(), + "status": status, + "mutating": true, + "result": stopped, + }) +} + pub(crate) fn record_notification_audit( paths: &AppPaths, actor: &str, @@ -973,7 +991,12 @@ fn command_available(name: &str) -> bool { #[cfg(test)] mod tests { use super::*; - use crate::test_support::temp_app_paths; + use crate::test_support::{ + assert_unconfirmed_stop_kept_the_service, seed_keyed_service, stop_with_outcome, + temp_app_paths, + }; + use crate::watchers::load_service_record; + use rocm_core::ManagedServiceRecord; use std::io::Read; fn serve_one_http_response(bytes: Vec) -> Result { @@ -2248,4 +2271,89 @@ mod tests { "https://huggingface.co@evil.example/Qwen/Test-1B" )); } + + #[test] + fn sandbox_tool_stop_server_updates_manifest_and_skips_current_pid() -> Result<()> { + let (root, paths) = temp_app_paths("sandbox-stop-current-pid"); + paths.ensure()?; + let current_pid = std::process::id(); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-current", + "vllm", + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + 11435, + "managed", + current_pid, + None, + None, + None, + ); + record.engine_pid = Some(current_pid); + record.status = "ready".to_owned(); + record.write()?; + + let value = run_sandbox_tool( + &paths, + SandboxToolArg::StopServer, + Some("svc-current".to_owned()), + None, + None, + SandboxToolPolicy::default(), + )?; + let reloaded = load_service_record(&paths, "svc-current")?; + fs::remove_dir_all(root).ok(); + + assert_eq!(value.get("status").and_then(Value::as_str), Some("stopped")); + assert_eq!(value.get("mutating").and_then(Value::as_bool), Some(true)); + assert_eq!(reloaded.status, "stopped"); + assert!( + value + .get("result") + .and_then(|result| result.get("skipped_pids")) + .and_then(Value::as_array) + .is_some_and(|pids| pids + .iter() + .any(|pid| pid.as_u64() == Some(u64::from(current_pid)))) + ); + Ok(()) + } + + /// The sandbox tool used to report `status: stopped` unconditionally. + #[test] + fn sandbox_tool_stop_server_reports_an_unconfirmed_stop() -> Result<()> { + let (root, paths) = temp_app_paths("sandbox-stop-unconfirmed"); + paths.ensure()?; + let service_id = "svc-sandbox-stop-unconfirmed"; + let key_path = seed_keyed_service(&paths, service_id, 11448)?; + + // The tool is `stop_managed_service` feeding `sandbox_stop_server_value`; + // `sandbox_tool_stop_server_updates_manifest_and_skips_current_pid` + // drives that whole tool for real. + let result = stop_with_outcome( + &paths, + service_id, + rocm_core::TerminationOutcome::Unverified, + ) + .map(sandbox_stop_server_value); + let key_kept = key_path.exists(); + let reloaded = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let value = result?; + assert_eq!( + value.get("status").and_then(Value::as_str), + Some("stop_unconfirmed"), + "{value}" + ); + assert_eq!( + value.pointer("/result/stopped").and_then(Value::as_bool), + Some(false), + "{value}" + ); + assert_unconfirmed_stop_kept_the_service(&reloaded?, key_kept); + Ok(()) + } } diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index 2eb26a4484..a1fa240e32 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -214,23 +214,6 @@ pub(crate) fn stop_report_confirmed(report: &Value) -> bool { report.get("stopped").and_then(Value::as_bool) == Some(true) } -/// The `stop_server` sandbox tool's answer for a [`stop_managed_service`] -/// report. The status is taken from the same verdict that decided what the -/// stop wrote to the manifest, so the two cannot disagree. -pub(crate) fn sandbox_stop_server_value(stopped: Value) -> Value { - let status = if stop_report_confirmed(&stopped) { - "stopped" - } else { - "stop_unconfirmed" - }; - json!({ - "tool": crate::sandbox_tool::SandboxToolArg::StopServer.as_cli_value(), - "status": status, - "mutating": true, - "result": stopped, - }) -} - /// How long a stop waits for a recorded process tree to exit on its own before /// escalating to a forced kill. Matches `rocm services stop`, which terminates /// the same records. diff --git a/docs/architecture.md b/docs/architecture.md index 54cdaecd81..aee0ef6021 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -87,13 +87,13 @@ Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): - [`common.rs`](../apps/rocmd/src/common.rs) — helpers the other modules call into: GPU/amd-smi snapshotting, the bridge-snapshot diagnostic, the `rocm`-subprocess capture wrappers the sandbox and MCP clusters both call (they only choose the child's `ROCM_CLI_*_DIR` environment; `CommandCapture` and the spawn itself live in [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), shared with `rocm`'s chat tool bridge, and the bounded-execution half in [`process.rs`](../crates/rocm-core/src/process.rs)), and small arg/healthcheck/endpoint-key utilities. Not every helper here has more than one caller (`wait_for_port`, for example, is called only from [`watchers.rs`](../apps/rocmd/src/watchers.rs)). - [`mcp.rs`](../apps/rocmd/src/mcp.rs) — the MCP stdio server, tool schema table, tool dispatch, and the argv-building and result-shaping helpers behind the MCP tools (the `rocm` subprocess itself is spawned through [`common.rs`](../apps/rocmd/src/common.rs)). - [`persistence.rs`](../apps/rocmd/src/persistence.rs) — `record_event`/`load_managed_services`, the automation-event/audit-log and managed-service-registry I/O shared across the daemon's sandbox, MCP, service-lifecycle, and watcher code. -- [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) — bubblewrap/native sandbox execution, atomic-write helpers, artifact prefetch policy gating, and the sandbox-tool result shaping for `check_updates`/`driver_plan`. +- [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) — bubblewrap/native sandbox execution, atomic-write helpers, artifact prefetch policy gating, and the sandbox-tool result shaping for `check_updates`/`driver_plan`/`stop_server`. - [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs) — the sandbox-tool vocabulary: the `SandboxToolArg` value enum naming each sandbox tool, and `SandboxToolPolicy`, the artifact-download policy built from the `--allow-*` flags (including the Hugging Face token read from the environment when authenticated downloads are allowed). It uses no other module of the crate, so modules that need the vocabulary do not import `cli.rs`. - [`service.rs`](../apps/rocmd/src/service.rs) — managed-service PID lifecycle/stop, `run_daemon`'s foreground loop, `supervise_service`'s spawn-and-recover path, and serve-log startup-phase polling. - [`watchers.rs`](../apps/rocmd/src/watchers.rs) — event collection/dispatch for all built-in watchers (TheRock update, GPU metrics/thermal-pressure, cache-warm, driver-upgrade, server-recover), plus managed-service recovery classification and automation-proposal queuing. - [`webhook.rs`](../apps/rocmd/src/webhook.rs) — the local webhook source: its axum routes, request validation, and watcher-kind allow-list. -[`lib.rs`](../apps/rocmd/src/lib.rs) itself is now top-level glue: module declarations, the three externally-consumed entry points re-exported from [`cli.rs`](../apps/rocmd/src/cli.rs) (`run_bin_cli`/`run_from_args`, plus `parse_args`, which `apps/rocm`'s test-only [`advised_commands.rs`](../apps/rocm/src/advised_commands.rs) uses to check advised `rocmd` commands), the `WATCHER_TICK_INTERVAL`/`ARTIFACT_PREFETCH_TIMEOUT` constants, and a test module. +[`lib.rs`](../apps/rocmd/src/lib.rs) itself is now top-level glue: module declarations, the three externally-consumed entry points re-exported from [`cli.rs`](../apps/rocmd/src/cli.rs) (`run_bin_cli`/`run_from_args`, plus `parse_args`, which `apps/rocm`'s test-only [`advised_commands.rs`](../apps/rocm/src/advised_commands.rs) uses to check advised `rocmd` commands), and the `WATCHER_TICK_INTERVAL` constant. ### `crates/rocm-core` — core library From 630023305c7aa27fb36c6b368d18d761f5d9ca3d Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 2 Nov 2026 10:50:23 +0000 Subject: [PATCH 03/13] refactor(rocmd): move load_service_record into persistence Move load_service_record, the single-manifest read, out of watchers.rs into persistence.rs next to load_managed_services. update_supervised_record in service.rs calls it, and watchers.rs calls update_supervised_record, so leaving it in watchers.rs would make the supervisor code that is about to leave service.rs and watchers.rs depend on each other. Production code in service.rs and watchers.rs, and the test imports in service.rs, mcp.rs, sandbox.rs and watchers.rs, now point at persistence.rs; persistence.rs still uses no other module of the crate. Part of the rocmd module-cycle follow-up work. Besides the moved lines and import wiring, the only other change is the docs/architecture.md entry for persistence.rs, which now names the function. No tests move: none target this function directly. Signed-off-by: Jussi Elo --- apps/rocmd/src/mcp.rs | 2 +- apps/rocmd/src/persistence.rs | 27 ++++++++++++++++++++++++++- apps/rocmd/src/sandbox.rs | 2 +- apps/rocmd/src/service.rs | 6 +++--- apps/rocmd/src/watchers.rs | 26 +------------------------- docs/architecture.md | 2 +- 6 files changed, 33 insertions(+), 32 deletions(-) diff --git a/apps/rocmd/src/mcp.rs b/apps/rocmd/src/mcp.rs index bffe3fae47..4c11340b62 100644 --- a/apps/rocmd/src/mcp.rs +++ b/apps/rocmd/src/mcp.rs @@ -1142,11 +1142,11 @@ fn system_prefix_requires_ack(prefix: &std::path::Path) -> bool { #[cfg(test)] mod tests { use super::*; + use crate::persistence::load_service_record; use crate::test_support::{ assert_unconfirmed_stop_kept_the_service, seed_keyed_service, stop_with_outcome, temp_app_paths, workspace_test_artifact_dir, }; - use crate::watchers::load_service_record; use std::fs; use std::path::PathBuf; diff --git a/apps/rocmd/src/persistence.rs b/apps/rocmd/src/persistence.rs index 847c92b7aa..5d1d3f1214 100644 --- a/apps/rocmd/src/persistence.rs +++ b/apps/rocmd/src/persistence.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -use anyhow::{Context, Result}; +use anyhow::{Context, Result, bail}; use rocm_core::{ AppPaths, AuditEventRecord, AutomationEventRecord, AutomationRuntimeState, ManagedServiceRecord, append_audit_event, append_automation_event, unix_time_millis, @@ -78,6 +78,31 @@ pub(crate) fn load_managed_services(paths: &AppPaths) -> Result Result { + rocm_core::ServiceId::new(service_id) + .with_context(|| format!("invalid managed service id `{service_id}`"))?; + let manifest_path = paths.service_manifest_path(service_id); + let bytes = fs::read(&manifest_path).with_context(|| { + format!( + "managed service `{service_id}` not found at {}", + manifest_path.display() + ) + })?; + let record = serde_json::from_slice::(&bytes) + .with_context(|| format!("failed to parse {}", manifest_path.display()))?; + if record.service_id != service_id { + bail!( + "managed service manifest {} contains service_id `{}`, expected `{service_id}`", + manifest_path.display(), + record.service_id + ); + } + Ok(record) +} + #[cfg(test)] mod tests { use super::*; diff --git a/apps/rocmd/src/sandbox.rs b/apps/rocmd/src/sandbox.rs index 229fe9c0af..0c64f8ce8d 100644 --- a/apps/rocmd/src/sandbox.rs +++ b/apps/rocmd/src/sandbox.rs @@ -991,11 +991,11 @@ fn command_available(name: &str) -> bool { #[cfg(test)] mod tests { use super::*; + use crate::persistence::load_service_record; use crate::test_support::{ assert_unconfirmed_stop_kept_the_service, seed_keyed_service, stop_with_outcome, temp_app_paths, }; - use crate::watchers::load_service_record; use rocm_core::ManagedServiceRecord; use std::io::Read; diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index a1fa240e32..12abef473a 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -837,7 +837,7 @@ pub(crate) fn update_supervised_record( if !paths.service_manifest_path(&supervisor.service_id).exists() { return Ok(()); } - let mut current = crate::watchers::load_service_record(paths, &supervisor.service_id)?; + let mut current = crate::persistence::load_service_record(paths, &supervisor.service_id)?; if current.supervisor_pid != supervisor.supervisor_pid || current.supervisor_start_ticks != supervisor.supervisor_start_ticks { @@ -1045,11 +1045,11 @@ fn wait_for_service_ready( #[cfg(test)] mod tests { use super::*; + use crate::persistence::load_service_record; use crate::test_support::identity_probe_record; use crate::test_support::{ UNCONFIRMED_STOP_PID, seed_keyed_service, temp_app_paths, unique_test_root, }; - use crate::watchers::load_service_record; use crate::watchers::{find_recoverable_service, stop_requested}; /// Drive `supervise_service` far enough to reach the key guard, and return @@ -1500,7 +1500,7 @@ mod tests { assert!(!key_path.exists()); let result = stop_managed_service(&paths, service_id); - let reloaded = crate::watchers::load_service_record(&paths, service_id); + let reloaded = crate::persistence::load_service_record(&paths, service_id); fs::remove_dir_all(root).ok(); let value = result?; diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index 8155e10e78..54e985ea14 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: MIT +use crate::persistence::load_service_record; use anyhow::{Context, Result, bail}; use rocm_core::{ AppPaths, AutomationProposalRecord, AutomationRuntimeState, AutomationTriggerEvent, @@ -1459,31 +1460,6 @@ fn endpoint_service_recovery_reason(record: &ManagedServiceRecord) -> Option Result { - rocm_core::ServiceId::new(service_id) - .with_context(|| format!("invalid managed service id `{service_id}`"))?; - let manifest_path = paths.service_manifest_path(service_id); - let bytes = fs::read(&manifest_path).with_context(|| { - format!( - "managed service `{service_id}` not found at {}", - manifest_path.display() - ) - })?; - let record = serde_json::from_slice::(&bytes) - .with_context(|| format!("failed to parse {}", manifest_path.display()))?; - if record.service_id != service_id { - bail!( - "managed service manifest {} contains service_id `{}`, expected `{service_id}`", - manifest_path.display(), - record.service_id - ); - } - Ok(record) -} - fn manifest_service_recovery_reason( record: &ManagedServiceRecord, now_unix_ms: u128, diff --git a/docs/architecture.md b/docs/architecture.md index aee0ef6021..d00028d482 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -86,7 +86,7 @@ Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): - [`cli.rs`](../apps/rocmd/src/cli.rs) — the `Cli`/`Command` clap definitions, the bridge-snapshot diagnostic printer, and the top-level dispatch in `run_cli`/`run_bin_cli`/`run_from_args`. - [`common.rs`](../apps/rocmd/src/common.rs) — helpers the other modules call into: GPU/amd-smi snapshotting, the bridge-snapshot diagnostic, the `rocm`-subprocess capture wrappers the sandbox and MCP clusters both call (they only choose the child's `ROCM_CLI_*_DIR` environment; `CommandCapture` and the spawn itself live in [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), shared with `rocm`'s chat tool bridge, and the bounded-execution half in [`process.rs`](../crates/rocm-core/src/process.rs)), and small arg/healthcheck/endpoint-key utilities. Not every helper here has more than one caller (`wait_for_port`, for example, is called only from [`watchers.rs`](../apps/rocmd/src/watchers.rs)). - [`mcp.rs`](../apps/rocmd/src/mcp.rs) — the MCP stdio server, tool schema table, tool dispatch, and the argv-building and result-shaping helpers behind the MCP tools (the `rocm` subprocess itself is spawned through [`common.rs`](../apps/rocmd/src/common.rs)). -- [`persistence.rs`](../apps/rocmd/src/persistence.rs) — `record_event`/`load_managed_services`, the automation-event/audit-log and managed-service-registry I/O shared across the daemon's sandbox, MCP, service-lifecycle, and watcher code. +- [`persistence.rs`](../apps/rocmd/src/persistence.rs) — `record_event`/`load_managed_services`/`load_service_record`, the automation-event/audit-log and managed-service-registry I/O shared across the daemon's sandbox, MCP, service-lifecycle, and watcher code. - [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) — bubblewrap/native sandbox execution, atomic-write helpers, artifact prefetch policy gating, and the sandbox-tool result shaping for `check_updates`/`driver_plan`/`stop_server`. - [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs) — the sandbox-tool vocabulary: the `SandboxToolArg` value enum naming each sandbox tool, and `SandboxToolPolicy`, the artifact-download policy built from the `--allow-*` flags (including the Hugging Face token read from the environment when authenticated downloads are allowed). It uses no other module of the crate, so modules that need the vocabulary do not import `cli.rs`. - [`service.rs`](../apps/rocmd/src/service.rs) — managed-service PID lifecycle/stop, `run_daemon`'s foreground loop, `supervise_service`'s spawn-and-recover path, and serve-log startup-phase polling. From 99213639afb6a5b57eefc8602fbcb6ff4c899114 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 2 Nov 2026 10:55:10 +0000 Subject: [PATCH 04/13] refactor(rocmd): move supervisor and readiness code into supervise.rs Move the code behind `rocmd supervise` out of service.rs into a new supervise.rs: parse_gpu_indices_arg, supervise_service, update_supervised_record, record_supervised_exit, engine_serve_http_args, last_cr_segment, classify_startup_phase, read_new_log_phase, engine_healthcheck_ready and wait_for_service_ready, with the tests that exercise only those symbols (the three key-guard tests and their helpers, the startup-phase and log-tail tests, the two engine_serve_http_args tests, the supervisor identity-token, first-write and exit-write tests). cli.rs and watchers.rs call the moved items at their new path. service.rs keeps the stop code and the daemon loop. Part of the rocmd module-cycle follow-up work. Besides the moved lines, module and import wiring, this commit changes: the Windows-only taskkill block that stays in service.rs keeps its process imports, now gated with cfg(windows) because nothing else in that file uses them on Linux, and the service.rs test module takes its own cfg(unix) copy for the tests that spawn helper processes; two comments in rocm-dash-daemon registry.rs that named a lib.rs path for supervise_service now name supervise.rs; docs/architecture.md gets an entry for the new file and a reworded service.rs entry. The doc comments on supervise_service and update_supervised_record that refer to a stop's re-read of the record were checked against the stop code left in service.rs and need no change. Signed-off-by: Jussi Elo --- apps/rocmd/src/cli.rs | 2 +- apps/rocmd/src/lib.rs | 1 + apps/rocmd/src/service.rs | 927 +---------------------- apps/rocmd/src/supervise.rs | 930 ++++++++++++++++++++++++ apps/rocmd/src/watchers.rs | 4 +- crates/rocm-dash-daemon/src/registry.rs | 4 +- docs/architecture.md | 3 +- 7 files changed, 945 insertions(+), 926 deletions(-) create mode 100644 apps/rocmd/src/supervise.rs diff --git a/apps/rocmd/src/cli.rs b/apps/rocmd/src/cli.rs index 0c8941876b..c332c9e044 100644 --- a/apps/rocmd/src/cli.rs +++ b/apps/rocmd/src/cli.rs @@ -182,7 +182,7 @@ async fn run_cli(cli: Cli) -> Result<()> { device_policy, gpu, engine_recipe_json, - } => crate::service::supervise_service( + } => crate::supervise::supervise_service( &paths, service_id, engine, diff --git a/apps/rocmd/src/lib.rs b/apps/rocmd/src/lib.rs index 0e3008dda6..7db6b7297a 100644 --- a/apps/rocmd/src/lib.rs +++ b/apps/rocmd/src/lib.rs @@ -11,6 +11,7 @@ mod persistence; mod sandbox; mod sandbox_tool; mod service; +mod supervise; #[cfg(test)] mod test_support; mod watchers; diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index 12abef473a..6459320431 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -5,16 +5,14 @@ use anyhow::{Context, Result, bail}; use rocm_core::{ AppPaths, AutomationRuntimeState, ManagedServiceRecord, RocmCliConfig, WatcherRuntimeSnapshot, - builtin_watchers, daemon_binary_path, unix_time_millis, + builtin_watchers, unix_time_millis, }; use rocm_engine_protocol::{EngineMethod, StopRequest, StopResponse}; use serde_json::Value; use serde_json::json; -use std::fs; use std::io::{self, Write}; -use std::path::Path; +#[cfg(windows)] use std::process::{Command as ProcessCommand, Stdio}; -use std::thread; use std::time::Duration; use tokio::time::{self, MissedTickBehavior}; @@ -622,282 +620,6 @@ pub(crate) fn print_status(paths: &AppPaths) -> Result<()> { Ok(()) } -fn parse_gpu_indices_arg(value: Option<&str>) -> Result> { - let Some(raw) = value else { - return Ok(Vec::new()); - }; - match rocm_engine_protocol::GpuSelection::parse_cli_value(raw).map_err(anyhow::Error::msg)? { - rocm_engine_protocol::GpuSelection::Auto => Ok(Vec::new()), - rocm_engine_protocol::GpuSelection::Index(index) => Ok(vec![index]), - } -} - -#[allow(clippy::too_many_arguments)] -pub(crate) fn supervise_service( - paths: &AppPaths, - service_id: String, - engine: String, - model_ref: String, - canonical_model_id: String, - runtime_id: Option, - env_id: Option, - host: String, - port: u16, - device_policy: String, - gpu: Option, - engine_recipe_json: Option, -) -> Result<()> { - paths.ensure()?; - fs::create_dir_all(paths.engine_logs_dir(&engine))?; - fs::create_dir_all(paths.engine_state_dir(&engine))?; - fs::create_dir_all(paths.services_dir())?; - - let gpu_indices = parse_gpu_indices_arg(gpu.as_deref())?; - let _ = daemon_binary_path(); - - let mut record = ManagedServiceRecord::new( - paths, - service_id, - engine.clone(), - model_ref, - canonical_model_id.clone(), - host, - port, - "managed", - std::process::id(), - runtime_id.clone(), - env_id.clone(), - Some(device_policy.clone()), - ); - // `new` recorded this PID; this pairs it with its start-time token. - record.record_supervisor_identity(std::process::id()); - record.gpu_indices = gpu_indices; - record.engine_recipe_json = engine_recipe_json.clone(); - // Carried over from whatever is on disk. `ManagedServiceRecord::new` starts - // this false, so rebuilding a record here without restoring it would not - // just skip the check now — it would write the weakened record back and - // disarm every later `rocm services restart` as well. - // - // Propagated, not defaulted. This read arms the guard below, so it is not - // best-effort the way an identical-looking call feeding a printed warning - // would be. `load_managed_services` already *skips* unparseable records, so - // an `Err` here is a real I/O failure — and a missing directory is `Ok` - // anyway. Swallowing it would say "no service ever required a key", the - // key-file fallback is false precisely when a service has been stopped, and - // the weakened record would then be written back at the bottom of this - // function. That is the outcome the comment above says must not happen. - let existing = crate::persistence::load_managed_services(paths) - .context( - "could not read the service registry to check whether this service requires an \ - endpoint API key; refusing to recover it rather than assume it does not", - )? - .into_iter() - .find(|existing| existing.service_id == record.service_id); - let previously_required = existing - .as_ref() - .is_some_and(|existing| existing.requires_api_key); - // Only what the registry recorded. The `|| key-file-is-present` clause that - // used to be here re-derived the flag the same way `spawn_managed_engine_child` - // did, and was wrong for the same reason: a public bind always has a key file - // whether or not auth was ever demanded, so recovery re-armed this on services - // that never asked for it and refused them with the wrong remediation. - record.requires_api_key = previously_required; - // A standing stop request is carried over too. The restart that launched - // this supervisor clears the marker before it does so, so one found here was - // set by a stop issued since — and `ManagedServiceRecord::new` starting it - // at `None` would quietly withdraw that stop and hand the service straight - // back to the daemon's recovery. - record.stop_requested_unix_ms = existing.and_then(|existing| existing.stop_requested_unix_ms); - // Refuse a keyless public respawn before the manifest write, so a refused - // attempt leaves the recorded restart_count and timestamps intact instead of - // clobbering them with a record no live process will ever back. The spawn - // site below re-checks against the key actually threaded onto the command. - crate::common::ensure_public_service_has_endpoint_key( - &record.host, - rocm_engine_protocol::endpoint_key_file_if_present(paths, &record.service_id) - .and_then(|path| rocm_engine_protocol::endpoint_api_key_file_if_valid(&path)) - .is_some(), - record.requires_api_key, - )?; - record.write()?; - - let log_file = fs::File::create(&record.log_path) - .with_context(|| format!("failed to create {}", record.log_path.display()))?; - let log_file_err = log_file - .try_clone() - .context("failed to clone service log file handle")?; - - let rocm_binary = - std::env::current_exe().context("failed to resolve current rocm executable path")?; - let mut command = ProcessCommand::new(rocm_binary); - command - .args(engine_serve_http_args( - &engine, - &record.service_id, - &canonical_model_id, - &record.host, - record.port, - &device_policy, - &record.gpu_indices, - runtime_id.as_deref(), - env_id.as_deref(), - engine_recipe_json.as_deref(), - &record.engine_state_path, - )) - .stdin(Stdio::null()) - .stdout(Stdio::from(log_file)) - .stderr(Stdio::from(log_file_err)); - // Re-thread the endpoint key file (public bind only) onto the engine child, - // same as the initial `rocm serve` spawn. This path also runs on daemon - // recovery (`restart_managed_service` re-execs `rocmd supervise`), so - // without this a previously-authenticated public service would come back - // up anonymous after a crash/recover cycle. - // If the key is gone the child would listen on the recorded public host with - // no auth, so fail closed instead — an unreachable service is recoverable, - // an anonymous public one is not. - let endpoint_key_applied = - crate::common::apply_endpoint_key_env(&mut command, paths, &record.service_id); - crate::common::ensure_public_service_has_endpoint_key( - &record.host, - endpoint_key_applied, - record.requires_api_key, - )?; - let mut child = command - .spawn() - .with_context(|| format!("failed to spawn engine supervisor child for {engine}"))?; - - // Every write from here on goes through `update_supervised_record`: it - // applies only this supervisor's own fields to the record as it is on disk - // now, so a stop marker, or anything else another writer persisted meanwhile, - // survives it. - // - // Captured while the child is known alive, so a later stop verifies this - // exact process instead of whatever has since inherited its PID. - let engine_pid = child.id(); - update_supervised_record(paths, &record, |current| { - current.record_engine_identity(engine_pid); - current.status = "running".to_owned(); - })?; - - let became_ready = wait_for_service_ready( - paths, - &record.engine, - &record.service_id, - &record.log_path, - Duration::from_mins(3), - |phase| { - let _ = update_supervised_record(paths, &record, |current| { - current.startup_phase = Some(phase.to_owned()); - }); - }, - ); - if became_ready { - update_supervised_record(paths, &record, |current| { - current.status = "ready".to_owned(); - // The phase only describes the coming-up window; clear it once ready. - current.startup_phase = None; - })?; - } - - let exit_status = child.wait().context("failed waiting for engine child")?; - record_supervised_exit(paths, &record, exit_status.success())?; - - if exit_status.success() { - Ok(()) - } else { - std::process::exit(exit_status.code().unwrap_or(1)); - } -} - -/// Apply one of [`supervise_service`]'s own changes to its service's record as -/// it is on disk now, rather than writing back the copy the supervisor built at -/// startup. -/// -/// `write` replaces the whole file, and the supervisor's copy is stale the -/// moment anyone else writes: a stop records `stop_requested_unix_ms` before it -/// signals anything, and `rocm`'s liveness refresh and the daemon's restart -/// write here too. Writing the startup snapshot back would erase that stop -/// marker — and a record reading `failed` with no marker is exactly what the -/// daemon's recovery restarts, undoing the stop. So `update` touches only the -/// fields the supervisor owns, and never the marker. -/// -/// Nothing is written when the record no longer belongs to this supervisor: -/// -/// - it is gone — `rocm services remove` deleted it, and writing would -/// resurrect a service the operator removed; -/// - it names a different supervisor process — a confirmed stop clears the -/// recorded PIDs, and a restart records its own. Either way the record now -/// describes something this supervisor is not, and its view of the service -/// (`running`, `failed`, …) would overwrite a truer one. -pub(crate) fn update_supervised_record( - paths: &AppPaths, - supervisor: &ManagedServiceRecord, - update: impl FnOnce(&mut ManagedServiceRecord), -) -> Result<()> { - if !paths.service_manifest_path(&supervisor.service_id).exists() { - return Ok(()); - } - let mut current = crate::persistence::load_service_record(paths, &supervisor.service_id)?; - if current.supervisor_pid != supervisor.supervisor_pid - || current.supervisor_start_ticks != supervisor.supervisor_start_ticks - { - return Ok(()); - } - update(&mut current); - current.write() -} - -/// [`supervise_service`]'s last write: the engine child has exited, so record -/// how. Separate so a test can drive exactly this write. -fn record_supervised_exit( - paths: &AppPaths, - supervisor: &ManagedServiceRecord, - success: bool, -) -> Result<()> { - update_supervised_record(paths, supervisor, |current| { - current.status = if success { "stopped" } else { "failed" }.to_owned(); - }) -} - -#[allow(clippy::too_many_arguments)] -fn engine_serve_http_args( - engine: &str, - service_id: &str, - canonical_model_id: &str, - host: &str, - port: u16, - device_policy: &str, - gpu_indices: &[u32], - runtime_id: Option<&str>, - env_id: Option<&str>, - engine_recipe_json: Option<&str>, - state_path: &Path, -) -> Vec { - let mut args = vec![ - "__engine-serve-http".to_owned(), - engine.to_owned(), - service_id.to_owned(), - canonical_model_id.to_owned(), - "--host".to_owned(), - host.to_owned(), - "--port".to_owned(), - port.to_string(), - "--device-policy".to_owned(), - device_policy.to_owned(), - ]; - if let Some(csv) = rocm_engine_protocol::gpu_indices_to_csv(gpu_indices) { - args.extend(["--gpu".to_owned(), csv]); - } - args.extend(crate::common::optional_arg("--runtime-id", runtime_id)); - args.extend(crate::common::optional_arg("--env-id", env_id)); - args.extend(crate::common::optional_arg( - "--engine-recipe-json", - engine_recipe_json, - )); - args.extend(["--state-path".to_owned(), state_path.display().to_string()]); - args -} - fn build_runtime_state( config: &RocmCliConfig, automations_enabled: bool, @@ -940,480 +662,15 @@ async fn shutdown_signal() { let _ = tokio::signal::ctrl_c().await; } -/// Keep only the final visible segment of a `\r`-redrawn progress line. -/// -/// Progress tools (pip, tqdm, Hugging Face) redraw a line in place with a bare -/// carriage return and no newline, so the segment after the last `\r` is its -/// final visible state. Lines without `\r` pass through unchanged. (Same -/// collapse rule the dashboard job console applies to streamed job output.) -fn last_cr_segment(line: &str) -> &str { - line.rsplit('\r').next().unwrap_or(line) -} - -/// Classify a single serve-log line into a coarse startup phase token -/// (`downloading`/`loading`/`warmup`), or `None` when the line carries no phase -/// signal. Case-insensitive substring match over the common vLLM / llama.cpp / -/// Hugging Face startup vocabulary. Checked warmup → loading → downloading so -/// the latest lifecycle stage a line mentions wins. -fn classify_startup_phase(line: &str) -> Option<&'static str> { - let lower = line.to_ascii_lowercase(); - if lower.contains("capturing cuda graph") - || lower.contains("capturing the model") - || lower.contains("warming up") - || lower.contains("warmup") - { - Some("warmup") - } else if lower.contains("loading weights") - || lower.contains("loading model") - || lower.contains("load_tensors") - || lower.contains("llama_model_loader") - || lower.contains("model loading took") - { - Some("loading") - } else if lower.contains("downloading") || lower.contains("fetching") { - Some("downloading") - } else { - None - } -} - -/// Read log bytes appended since `*pos`, advance `*pos`, and return the most -/// recent recognizable startup phase in that new output (later lines win, so a -/// download → load → warmup progression advances naturally). -/// -/// Best-effort: any I/O error (file not created yet, transient read) yields -/// `None`. A shrunk file (rotation/truncation) resets the cursor to the top. -fn read_new_log_phase(log_path: &Path, pos: &mut u64) -> Option<&'static str> { - use std::io::{Read, Seek, SeekFrom}; - let mut file = fs::File::open(log_path).ok()?; - let len = file.metadata().ok()?.len(); - if len < *pos { - *pos = 0; - } - if len == *pos { - return None; - } - file.seek(SeekFrom::Start(*pos)).ok()?; - let mut bytes = Vec::new(); - let read = file.read_to_end(&mut bytes).ok()?; - *pos += read as u64; - let text = String::from_utf8_lossy(&bytes); - let mut phase = None; - for line in text.lines() { - if let Some(found) = classify_startup_phase(last_cr_segment(line)) { - phase = Some(found); - } - } - phase -} - -fn engine_healthcheck_ready(paths: &AppPaths, engine: &str, service_id: &str) -> Result { - Ok(crate::common::healthcheck_response_ready( - &crate::common::engine_healthcheck_response(paths, engine, service_id)?, - )) -} - -/// Poll a freshly-spawned service until its healthcheck reports ready (or the -/// timeout elapses), tailing its log file meanwhile and reporting each coarse -/// startup phase transition via `on_phase`. -fn wait_for_service_ready( - paths: &AppPaths, - engine: &str, - service_id: &str, - log_path: &Path, - timeout: Duration, - mut on_phase: impl FnMut(&str), -) -> bool { - let start = std::time::Instant::now(); - let mut log_pos: u64 = 0; - let mut last_phase: Option<&'static str> = None; - while start.elapsed() < timeout { - if let Some(phase) = read_new_log_phase(log_path, &mut log_pos) - && last_phase != Some(phase) - { - last_phase = Some(phase); - on_phase(phase); - } - if engine_healthcheck_ready(paths, engine, service_id).unwrap_or(false) { - return true; - } - thread::sleep(Duration::from_millis(200)); - } - false -} - #[cfg(test)] mod tests { use super::*; use crate::persistence::load_service_record; - use crate::test_support::identity_probe_record; - use crate::test_support::{ - UNCONFIRMED_STOP_PID, seed_keyed_service, temp_app_paths, unique_test_root, - }; - use crate::watchers::{find_recoverable_service, stop_requested}; - - /// Drive `supervise_service` far enough to reach the key guard, and return - /// what it did. - /// - /// The guard sits before the manifest write and well before any spawn, so a - /// refusal returns without starting a process — which is what makes the real - /// call site testable at all. The arguments below are the shape a recovery - /// re-exec passes: a loopback bind, no GPU, no recipe. - /// - /// This exists because testing `ensure_public_service_has_endpoint_key` - /// directly with literal arguments cannot catch the defect that actually - /// happened twice in this crate's history — the guard being *wired up* with - /// the wrong value at its call site. - /// - /// Bounded, and the bound is the assertion. A guard that fails to refuse - /// does not return an error — it falls through to the engine spawn and - /// supervises a child that never exits, so an unbounded call would hang the - /// suite instead of failing it. Both callers below are regression tests for - /// a fail-*open*, which is exactly the shape that turns into a hang. - fn supervise_at_the_guard(paths: &AppPaths, service_id: &str) -> Result<()> { - let paths = paths.clone(); - let service_id = service_id.to_owned(); - let (sender, receiver) = std::sync::mpsc::channel(); - std::thread::spawn(move || { - let outcome = supervise_service( - &paths, - service_id, - "llamacpp".to_owned(), - "a-model".to_owned(), - "a-model".to_owned(), - None, - None, - "127.0.0.1".to_owned(), - 11434, - "gpu_required".to_owned(), - None, - None, - ); - let _ = sender.send(outcome.map_err(|error| format!("{error:#}"))); - }); - receiver - .recv_timeout(std::time::Duration::from_secs(30)) - .unwrap_or_else(|_| { - panic!( - "supervise_service did not return within 30s: the key guard let the call \ - through and it reached the engine spawn, which is the fail-open this test \ - exists to catch" - ) - }) - .map_err(anyhow::Error::msg) - } - - /// Write a service record into the registry the way a live service would - /// have left it behind. - fn seed_registry(paths: &AppPaths, service_id: &str, requires_api_key: bool) { - fs::create_dir_all(paths.services_dir()).unwrap(); - let mut record = ManagedServiceRecord::new( - paths, - service_id.to_owned(), - "llamacpp".to_owned(), - "a-model".to_owned(), - "a-model".to_owned(), - "127.0.0.1".to_owned(), - 11434, - "managed", - std::process::id(), - None, - None, - Some("gpu_required".to_owned()), - ); - record.requires_api_key = requires_api_key; - record.write().unwrap(); - } - - /// Drive `supervise_service` and report whether the key guard let it past. - /// - /// Decided on what the call *returns*, not on any file. The obvious - /// observable — the manifest appearing — is useless here, because - /// `seed_registry` has already written one, so polling for it passes - /// whatever the guard does. That mistake was made first and caught by - /// mutating the code the test claims to protect. - /// - /// A call the guard admits does not return: it carries on to the engine - /// spawn. So the guard's refusal is the only thing that comes back quickly, - /// and it is identified by its message rather than by the mere fact of an - /// error — a later, unrelated failure must not read as a refusal. - fn guard_admits(paths: &AppPaths, service_id: &str) -> bool { - let owned_paths = paths.clone(); - let owned_id = service_id.to_owned(); - let (sender, receiver) = std::sync::mpsc::channel(); - std::thread::spawn(move || { - let outcome = supervise_service( - &owned_paths, - owned_id, - "llamacpp".to_owned(), - "a-model".to_owned(), - "a-model".to_owned(), - None, - None, - "127.0.0.1".to_owned(), - 11434, - "gpu_required".to_owned(), - None, - None, - ); - let _ = sender.send(outcome.map_err(|error| format!("{error:#}"))); - }); - - match receiver.recv_timeout(std::time::Duration::from_secs(10)) { - // The key guard refused, by its own words. - Ok(Err(rendered)) if rendered.contains("without authentication") => false, - // Anything else means it got past the guard: it either finished, or - // failed later for a reason that is not this guard, or is still - // running because it reached the spawn. - _ => true, - } - } - - #[test] - fn a_service_that_never_required_a_key_is_not_refused_for_lacking_one() { - // The other direction of the guard, and the one no test covered. - // Hardcoding `record.requires_api_key = true` at the restore site passes - // every other test in this crate, because they all seed a service that - // *does* require a key. This is the case that catches it. - // - // Two records are seeded, not one: with a single record the - // `existing.service_id == record.service_id` half of the lookup does - // nothing, so dropping that comparison would go unnoticed and one - // service's requirement would leak onto another's. - let (root, paths) = temp_app_paths("supervise-no-key-needed"); - seed_registry(&paths, "svc-needs-key", true); - seed_registry(&paths, "svc-plain", false); - - assert!( - guard_admits(&paths, "svc-plain"), - "a loopback service that never asked for a key must not be refused for lacking one" - ); - - let _ = fs::remove_dir_all(&root); - } - - #[test] - fn supervising_a_service_that_required_a_key_refuses_when_the_key_is_gone() { - // The real call site, not the guard in isolation. `supervise_service` - // rebuilds the record with `ManagedServiceRecord::new`, which starts - // `requires_api_key` false, and restores it from the registry. Passing - // the wrong value here — a literal, or the freshly-built field before it - // is restored — is exactly the miswiring that shipped twice in this - // crate and that a literal-argument unit test cannot see. - let (root, paths) = temp_app_paths("supervise-requires-key"); - seed_registry(&paths, "svc-needs-key", true); - - let error = supervise_at_the_guard(&paths, "svc-needs-key") - .expect_err("a service that required a key must not be recovered without one"); - assert!( - format!("{error:#}").contains("without authentication"), - "{error:#}" - ); - - // And the refusal must not have weakened what is on disk. The guard runs - // before `record.write()` precisely so a refused attempt leaves the - // recorded requirement armed for the next attempt. - let stored = crate::persistence::load_managed_services(&paths).unwrap(); - let stored = stored - .iter() - .find(|candidate| candidate.service_id == "svc-needs-key") - .expect("the seeded record must survive a refused recovery"); - assert!( - stored.requires_api_key, - "a refused recovery must not disarm the requirement" - ); - - let _ = fs::remove_dir_all(&root); - } - - #[test] - fn a_registry_that_cannot_be_read_refuses_recovery_rather_than_assuming_no_key() { - // `load_managed_services` already skips records it cannot parse, so an - // `Err` from it is a real I/O failure — and a missing directory is `Ok`. - // Defaulting it away therefore says "no service ever required a key", - // which is fail-open on an auth gate and, worse, gets written back. - // - // The failure is provoked portably: a directory named like a record - // makes the `fs::read` inside the loop fail rather than the read_dir. - let (root, paths) = temp_app_paths("supervise-unreadable-registry"); - fs::create_dir_all(paths.services_dir().join("not-a-record.json")).unwrap(); - - let error = supervise_at_the_guard(&paths, "svc-unknown") - .expect_err("an unreadable registry must refuse, not assume no key was required"); - let rendered = format!("{error:#}"); - assert!( - rendered.contains("could not read the service registry"), - "{rendered}" - ); - - // Nothing was written: a registry we could not read is not a registry we - // may add a weakened record to. - assert!( - !paths.service_manifest_path("svc-unknown").exists(), - "a refused recovery must not persist a record" - ); - - let _ = fs::remove_dir_all(&root); - } - - #[test] - fn last_cr_segment_keeps_final_progress_redraw() { - // A tqdm/HF-style in-place redraw collapses to its last segment. - assert_eq!( - last_cr_segment("Downloading: 10%\rDownloading: 55%\rDownloading: 100%"), - "Downloading: 100%" - ); - // A plain line is unchanged. - assert_eq!( - last_cr_segment("Loading model weights"), - "Loading model weights" - ); - } - - #[test] - fn classify_startup_phase_maps_engine_vocabulary() { - assert_eq!( - classify_startup_phase("Downloading shards: 100%"), - Some("downloading") - ); - assert_eq!( - classify_startup_phase("Fetching 12 files"), - Some("downloading") - ); - assert_eq!( - classify_startup_phase("INFO: Loading model weights took 4.2s"), - Some("loading") - ); - assert_eq!( - classify_startup_phase("llama_model_loader: loaded meta data"), - Some("loading") - ); - assert_eq!( - classify_startup_phase("Capturing CUDA graph shapes"), - Some("warmup") - ); - assert_eq!( - classify_startup_phase("Warming up the engine"), - Some("warmup") - ); - // Ordinary chatter carries no phase signal. - assert_eq!( - classify_startup_phase("Uvicorn running on http://..."), - None - ); - } - - #[test] - fn classify_startup_phase_emits_only_dashboard_known_tokens() { - // These tokens are the wire contract with the dashboard's - // `StartupPhase::from_token` (rocm-dash-core); emitting anything else - // would be silently dropped there. rocmd can't link that crate, so the - // contract is pinned here by literal. - for line in [ - "Downloading shards", - "Loading model weights", - "Capturing CUDA graph", - ] { - let token = classify_startup_phase(line).expect("line is a phase signal"); - assert!( - matches!(token, "downloading" | "loading" | "warmup"), - "token {token:?} must be one the dashboard understands" - ); - } - } - - #[test] - fn read_new_log_phase_advances_and_tracks_latest() { - use std::io::Write as _; - // Workspace-local test root (rooted at CARGO_MANIFEST_DIR, not the - // ambient temp dir) — same helper the other rocmd tests use. - let dir = unique_test_root(&format!("rocmd-phase-{}", std::process::id())); - let log = dir.join("svc.log"); - std::fs::write(&log, "boot\nDownloading shards: 100%\n").unwrap(); - - let mut pos = 0_u64; - assert_eq!(read_new_log_phase(&log, &mut pos), Some("downloading")); - // No new bytes → no phase, cursor unchanged. - let after_first = pos; - assert_eq!(read_new_log_phase(&log, &mut pos), None); - assert_eq!(pos, after_first); - - // Appending a later stage advances the phase. - let mut f = std::fs::OpenOptions::new().append(true).open(&log).unwrap(); - writeln!(f, "Loading model weights took 3s").unwrap(); - assert_eq!(read_new_log_phase(&log, &mut pos), Some("loading")); - - let _ = std::fs::remove_dir_all(&dir); - } - - #[test] - fn engine_serve_http_args_forward_engine_recipe_json() { - let engine_recipe_json = r#"{"contract_version":"0.1.0","engine":"vllm","required_flags":["--enable-auto-tool-choice"]}"#; - let args = engine_serve_http_args( - "vllm", - "svc-1", - "Qwen/Qwen3.5-4B", - "127.0.0.1", - 11435, - "gpu_required", - &[], - Some("therock-release:gfx120X-all"), - Some("env-1"), - Some(engine_recipe_json), - Path::new("state.json"), - ); - - assert!( - args.windows(2) - .any(|pair| pair[0] == "--engine-recipe-json" && pair[1] == engine_recipe_json) - ); - assert!( - args.windows(2).any(|pair| { - pair[0] == "--runtime-id" && pair[1] == "therock-release:gfx120X-all" - }) - ); - assert!( - args.windows(2) - .any(|pair| pair[0] == "--state-path" && pair[1] == "state.json") - ); - } - - #[test] - fn engine_serve_http_args_emit_gpu_indices_when_pinned() { - let args = engine_serve_http_args( - "vllm", - "svc-1", - "Qwen/Qwen3.5-4B", - "127.0.0.1", - 11435, - "gpu_required", - &[1], - None, - None, - None, - Path::new("state.json"), - ); - - assert!( - args.windows(2) - .any(|pair| pair[0] == "--gpu" && pair[1] == "1") - ); - - let auto = engine_serve_http_args( - "vllm", - "svc-1", - "Qwen/Qwen3.5-4B", - "127.0.0.1", - 11435, - "gpu_required", - &[], - None, - None, - None, - Path::new("state.json"), - ); - assert!(!auto.iter().any(|arg| arg == "--gpu")); - } + use crate::test_support::{UNCONFIRMED_STOP_PID, seed_keyed_service, temp_app_paths}; + use crate::watchers::stop_requested; + use std::fs; + #[cfg(unix)] + use std::process::{Command as ProcessCommand, Stdio}; #[tokio::test] async fn local_webhook_requires_enabled_automation_loop() { @@ -2112,174 +1369,4 @@ mod tests { ); Ok(()) } - - /// The identity check is only as good as the token the launcher writes down. - /// `rocm_core::identity_state` treats a missing token as a best-effort match - /// — the legacy `(None, _)` branch — so a record written without one is - /// signalled blind, which is the original defect restored in full. Every - /// other test here hands itself a record with the token already set, and so - /// proves nothing about the code that is supposed to put it there. - /// - /// This drives the real `supervise_service` and reads the manifest it - /// persisted. It is stopped at that first write by making the service log - /// path a directory, so the `fs::File::create` immediately after it fails: - /// going further reaches the engine spawn, and the failure path past that - /// calls `std::process::exit`, which inside a test binary would take the - /// whole suite with it. - /// - /// Linux-only: `rocm_core::process_start_ticks` reads `/proc` and returns - /// `None` everywhere else, so on other platforms there is no token to write. - #[cfg(target_os = "linux")] - #[test] - fn supervise_service_persists_the_supervisor_identity_token() -> Result<()> { - let (root, paths) = temp_app_paths("supervise-records-identity"); - paths.ensure()?; - fs::create_dir_all(paths.services_dir())?; - - let service_id = "svc-supervise-identity"; - let log_path = identity_probe_record(&paths, service_id, 11443).log_path; - // A directory where the supervisor expects to create a file. - fs::create_dir_all(&log_path)?; - - let outcome = supervise_service( - &paths, - service_id.to_owned(), - "llamacpp".to_owned(), - "a-model".to_owned(), - "a-model".to_owned(), - None, - None, - "127.0.0.1".to_owned(), - 11443, - "gpu_required".to_owned(), - None, - None, - ); - let persisted = load_service_record(&paths, service_id); - fs::remove_dir_all(root).ok(); - - let error = - outcome.expect_err("the log path is a directory, so the engine spawn is unreachable"); - assert!( - format!("{error:#}").contains("failed to create"), - "stopped somewhere other than the log file: {error:#}" - ); - assert!( - persisted?.supervisor_start_ticks.is_some(), - "supervise_service must persist the supervisor's start-time token beside its PID" - ); - Ok(()) - } - - /// The supervisor's last write lands after its engine child exits — which - /// is what a stop that could not verify the supervisor, but did kill the - /// engine, leaves behind. Writing back the record the supervisor built at - /// startup would erase the stop marker the stop persisted, and a `failed` - /// record with no marker is exactly what the daemon's recovery restarts. - #[test] - fn supervisor_exit_write_keeps_a_standing_stop_marker() -> Result<()> { - let (root, paths) = temp_app_paths("supervise-exit-keeps-marker"); - paths.ensure()?; - let service_id = "svc-supervise-exit-marker"; - // The supervisor's own copy, as `supervise_service` builds it: no marker. - let mut supervisor = identity_probe_record(&paths, service_id, 11446); - supervisor.record_supervisor_identity(std::process::id()); - supervisor.status = "running".to_owned(); - // What is on disk once a stop has run: same processes, marker set. - let mut on_disk = supervisor.clone(); - on_disk.stop_requested_unix_ms = Some(1); - on_disk.write()?; - - record_supervised_exit(&paths, &supervisor, false)?; - let persisted = load_service_record(&paths, service_id); - let recoverable = find_recoverable_service(&paths); - fs::remove_dir_all(root).ok(); - - let persisted = persisted?; - assert_eq!( - persisted.status, "failed", - "the supervisor's own field lands" - ); - assert_eq!( - persisted.stop_requested_unix_ms, - Some(1), - "the supervisor's exit write must not withdraw a standing stop" - ); - assert!( - recoverable?.is_none(), - "a stopped service must not become a recovery candidate" - ); - Ok(()) - } - - /// A record that names another supervisor — a confirmed stop cleared the - /// PIDs, or a restart recorded its own — or that was removed outright, is - /// no longer this supervisor's to write. - #[test] - fn supervisor_exit_write_leaves_a_record_it_no_longer_owns() -> Result<()> { - let (root, paths) = temp_app_paths("supervise-exit-not-owner"); - paths.ensure()?; - let service_id = "svc-supervise-exit-not-owner"; - let mut supervisor = identity_probe_record(&paths, service_id, 11447); - supervisor.record_supervisor_identity(std::process::id()); - supervisor.status = "running".to_owned(); - // A confirmed stop's final write: status stopped, PIDs cleared. - let mut stopped = supervisor.clone(); - stopped.status = "stopped".to_owned(); - stopped.supervisor_pid = 0; - stopped.supervisor_start_ticks = None; - stopped.write()?; - - record_supervised_exit(&paths, &supervisor, false)?; - let after_stop = load_service_record(&paths, service_id); - - fs::remove_file(paths.service_manifest_path(service_id))?; - record_supervised_exit(&paths, &supervisor, false)?; - let resurrected = paths.service_manifest_path(service_id).exists(); - fs::remove_dir_all(root).ok(); - - assert_eq!(after_stop?.status, "stopped"); - assert!(!resurrected, "a removed record must not be written back"); - Ok(()) - } - - /// The supervisor's first write rebuilds the record from its arguments. - /// The restart that launched it cleared any stop marker first, so a marker - /// on disk here was set by a stop issued since, and must survive. - #[cfg(target_os = "linux")] - #[test] - fn supervise_service_first_write_keeps_a_standing_stop_marker() -> Result<()> { - let (root, paths) = temp_app_paths("supervise-first-write-keeps-marker"); - paths.ensure()?; - fs::create_dir_all(paths.services_dir())?; - - let service_id = "svc-supervise-first-write-marker"; - let mut seeded = identity_probe_record(&paths, service_id, 11448); - seeded.stop_requested_unix_ms = Some(1); - seeded.write()?; - // Stop `supervise_service` right after its first write; see - // `supervise_service_persists_the_supervisor_identity_token`. - fs::create_dir_all(&seeded.log_path)?; - - let outcome = supervise_service( - &paths, - service_id.to_owned(), - "llamacpp".to_owned(), - "a-model".to_owned(), - "a-model".to_owned(), - None, - None, - "127.0.0.1".to_owned(), - 11448, - "gpu_required".to_owned(), - None, - None, - ); - let persisted = load_service_record(&paths, service_id); - fs::remove_dir_all(root).ok(); - - assert!(outcome.is_err(), "the log path is a directory"); - assert_eq!(persisted?.stop_requested_unix_ms, Some(1)); - Ok(()) - } } diff --git a/apps/rocmd/src/supervise.rs b/apps/rocmd/src/supervise.rs new file mode 100644 index 0000000000..3947a01a64 --- /dev/null +++ b/apps/rocmd/src/supervise.rs @@ -0,0 +1,930 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +use anyhow::{Context, Result}; +use rocm_core::{AppPaths, ManagedServiceRecord, daemon_binary_path}; +use std::fs; +use std::path::Path; +use std::process::{Command as ProcessCommand, Stdio}; +use std::thread; +use std::time::Duration; + +fn parse_gpu_indices_arg(value: Option<&str>) -> Result> { + let Some(raw) = value else { + return Ok(Vec::new()); + }; + match rocm_engine_protocol::GpuSelection::parse_cli_value(raw).map_err(anyhow::Error::msg)? { + rocm_engine_protocol::GpuSelection::Auto => Ok(Vec::new()), + rocm_engine_protocol::GpuSelection::Index(index) => Ok(vec![index]), + } +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn supervise_service( + paths: &AppPaths, + service_id: String, + engine: String, + model_ref: String, + canonical_model_id: String, + runtime_id: Option, + env_id: Option, + host: String, + port: u16, + device_policy: String, + gpu: Option, + engine_recipe_json: Option, +) -> Result<()> { + paths.ensure()?; + fs::create_dir_all(paths.engine_logs_dir(&engine))?; + fs::create_dir_all(paths.engine_state_dir(&engine))?; + fs::create_dir_all(paths.services_dir())?; + + let gpu_indices = parse_gpu_indices_arg(gpu.as_deref())?; + let _ = daemon_binary_path(); + + let mut record = ManagedServiceRecord::new( + paths, + service_id, + engine.clone(), + model_ref, + canonical_model_id.clone(), + host, + port, + "managed", + std::process::id(), + runtime_id.clone(), + env_id.clone(), + Some(device_policy.clone()), + ); + // `new` recorded this PID; this pairs it with its start-time token. + record.record_supervisor_identity(std::process::id()); + record.gpu_indices = gpu_indices; + record.engine_recipe_json = engine_recipe_json.clone(); + // Carried over from whatever is on disk. `ManagedServiceRecord::new` starts + // this false, so rebuilding a record here without restoring it would not + // just skip the check now — it would write the weakened record back and + // disarm every later `rocm services restart` as well. + // + // Propagated, not defaulted. This read arms the guard below, so it is not + // best-effort the way an identical-looking call feeding a printed warning + // would be. `load_managed_services` already *skips* unparseable records, so + // an `Err` here is a real I/O failure — and a missing directory is `Ok` + // anyway. Swallowing it would say "no service ever required a key", the + // key-file fallback is false precisely when a service has been stopped, and + // the weakened record would then be written back at the bottom of this + // function. That is the outcome the comment above says must not happen. + let existing = crate::persistence::load_managed_services(paths) + .context( + "could not read the service registry to check whether this service requires an \ + endpoint API key; refusing to recover it rather than assume it does not", + )? + .into_iter() + .find(|existing| existing.service_id == record.service_id); + let previously_required = existing + .as_ref() + .is_some_and(|existing| existing.requires_api_key); + // Only what the registry recorded. The `|| key-file-is-present` clause that + // used to be here re-derived the flag the same way `spawn_managed_engine_child` + // did, and was wrong for the same reason: a public bind always has a key file + // whether or not auth was ever demanded, so recovery re-armed this on services + // that never asked for it and refused them with the wrong remediation. + record.requires_api_key = previously_required; + // A standing stop request is carried over too. The restart that launched + // this supervisor clears the marker before it does so, so one found here was + // set by a stop issued since — and `ManagedServiceRecord::new` starting it + // at `None` would quietly withdraw that stop and hand the service straight + // back to the daemon's recovery. + record.stop_requested_unix_ms = existing.and_then(|existing| existing.stop_requested_unix_ms); + // Refuse a keyless public respawn before the manifest write, so a refused + // attempt leaves the recorded restart_count and timestamps intact instead of + // clobbering them with a record no live process will ever back. The spawn + // site below re-checks against the key actually threaded onto the command. + crate::common::ensure_public_service_has_endpoint_key( + &record.host, + rocm_engine_protocol::endpoint_key_file_if_present(paths, &record.service_id) + .and_then(|path| rocm_engine_protocol::endpoint_api_key_file_if_valid(&path)) + .is_some(), + record.requires_api_key, + )?; + record.write()?; + + let log_file = fs::File::create(&record.log_path) + .with_context(|| format!("failed to create {}", record.log_path.display()))?; + let log_file_err = log_file + .try_clone() + .context("failed to clone service log file handle")?; + + let rocm_binary = + std::env::current_exe().context("failed to resolve current rocm executable path")?; + let mut command = ProcessCommand::new(rocm_binary); + command + .args(engine_serve_http_args( + &engine, + &record.service_id, + &canonical_model_id, + &record.host, + record.port, + &device_policy, + &record.gpu_indices, + runtime_id.as_deref(), + env_id.as_deref(), + engine_recipe_json.as_deref(), + &record.engine_state_path, + )) + .stdin(Stdio::null()) + .stdout(Stdio::from(log_file)) + .stderr(Stdio::from(log_file_err)); + // Re-thread the endpoint key file (public bind only) onto the engine child, + // same as the initial `rocm serve` spawn. This path also runs on daemon + // recovery (`restart_managed_service` re-execs `rocmd supervise`), so + // without this a previously-authenticated public service would come back + // up anonymous after a crash/recover cycle. + // If the key is gone the child would listen on the recorded public host with + // no auth, so fail closed instead — an unreachable service is recoverable, + // an anonymous public one is not. + let endpoint_key_applied = + crate::common::apply_endpoint_key_env(&mut command, paths, &record.service_id); + crate::common::ensure_public_service_has_endpoint_key( + &record.host, + endpoint_key_applied, + record.requires_api_key, + )?; + let mut child = command + .spawn() + .with_context(|| format!("failed to spawn engine supervisor child for {engine}"))?; + + // Every write from here on goes through `update_supervised_record`: it + // applies only this supervisor's own fields to the record as it is on disk + // now, so a stop marker, or anything else another writer persisted meanwhile, + // survives it. + // + // Captured while the child is known alive, so a later stop verifies this + // exact process instead of whatever has since inherited its PID. + let engine_pid = child.id(); + update_supervised_record(paths, &record, |current| { + current.record_engine_identity(engine_pid); + current.status = "running".to_owned(); + })?; + + let became_ready = wait_for_service_ready( + paths, + &record.engine, + &record.service_id, + &record.log_path, + Duration::from_mins(3), + |phase| { + let _ = update_supervised_record(paths, &record, |current| { + current.startup_phase = Some(phase.to_owned()); + }); + }, + ); + if became_ready { + update_supervised_record(paths, &record, |current| { + current.status = "ready".to_owned(); + // The phase only describes the coming-up window; clear it once ready. + current.startup_phase = None; + })?; + } + + let exit_status = child.wait().context("failed waiting for engine child")?; + record_supervised_exit(paths, &record, exit_status.success())?; + + if exit_status.success() { + Ok(()) + } else { + std::process::exit(exit_status.code().unwrap_or(1)); + } +} + +/// Apply one of [`supervise_service`]'s own changes to its service's record as +/// it is on disk now, rather than writing back the copy the supervisor built at +/// startup. +/// +/// `write` replaces the whole file, and the supervisor's copy is stale the +/// moment anyone else writes: a stop records `stop_requested_unix_ms` before it +/// signals anything, and `rocm`'s liveness refresh and the daemon's restart +/// write here too. Writing the startup snapshot back would erase that stop +/// marker — and a record reading `failed` with no marker is exactly what the +/// daemon's recovery restarts, undoing the stop. So `update` touches only the +/// fields the supervisor owns, and never the marker. +/// +/// Nothing is written when the record no longer belongs to this supervisor: +/// +/// - it is gone — `rocm services remove` deleted it, and writing would +/// resurrect a service the operator removed; +/// - it names a different supervisor process — a confirmed stop clears the +/// recorded PIDs, and a restart records its own. Either way the record now +/// describes something this supervisor is not, and its view of the service +/// (`running`, `failed`, …) would overwrite a truer one. +pub(crate) fn update_supervised_record( + paths: &AppPaths, + supervisor: &ManagedServiceRecord, + update: impl FnOnce(&mut ManagedServiceRecord), +) -> Result<()> { + if !paths.service_manifest_path(&supervisor.service_id).exists() { + return Ok(()); + } + let mut current = crate::persistence::load_service_record(paths, &supervisor.service_id)?; + if current.supervisor_pid != supervisor.supervisor_pid + || current.supervisor_start_ticks != supervisor.supervisor_start_ticks + { + return Ok(()); + } + update(&mut current); + current.write() +} + +/// [`supervise_service`]'s last write: the engine child has exited, so record +/// how. Separate so a test can drive exactly this write. +fn record_supervised_exit( + paths: &AppPaths, + supervisor: &ManagedServiceRecord, + success: bool, +) -> Result<()> { + update_supervised_record(paths, supervisor, |current| { + current.status = if success { "stopped" } else { "failed" }.to_owned(); + }) +} + +#[allow(clippy::too_many_arguments)] +fn engine_serve_http_args( + engine: &str, + service_id: &str, + canonical_model_id: &str, + host: &str, + port: u16, + device_policy: &str, + gpu_indices: &[u32], + runtime_id: Option<&str>, + env_id: Option<&str>, + engine_recipe_json: Option<&str>, + state_path: &Path, +) -> Vec { + let mut args = vec![ + "__engine-serve-http".to_owned(), + engine.to_owned(), + service_id.to_owned(), + canonical_model_id.to_owned(), + "--host".to_owned(), + host.to_owned(), + "--port".to_owned(), + port.to_string(), + "--device-policy".to_owned(), + device_policy.to_owned(), + ]; + if let Some(csv) = rocm_engine_protocol::gpu_indices_to_csv(gpu_indices) { + args.extend(["--gpu".to_owned(), csv]); + } + args.extend(crate::common::optional_arg("--runtime-id", runtime_id)); + args.extend(crate::common::optional_arg("--env-id", env_id)); + args.extend(crate::common::optional_arg( + "--engine-recipe-json", + engine_recipe_json, + )); + args.extend(["--state-path".to_owned(), state_path.display().to_string()]); + args +} + +/// Keep only the final visible segment of a `\r`-redrawn progress line. +/// +/// Progress tools (pip, tqdm, Hugging Face) redraw a line in place with a bare +/// carriage return and no newline, so the segment after the last `\r` is its +/// final visible state. Lines without `\r` pass through unchanged. (Same +/// collapse rule the dashboard job console applies to streamed job output.) +fn last_cr_segment(line: &str) -> &str { + line.rsplit('\r').next().unwrap_or(line) +} + +/// Classify a single serve-log line into a coarse startup phase token +/// (`downloading`/`loading`/`warmup`), or `None` when the line carries no phase +/// signal. Case-insensitive substring match over the common vLLM / llama.cpp / +/// Hugging Face startup vocabulary. Checked warmup → loading → downloading so +/// the latest lifecycle stage a line mentions wins. +fn classify_startup_phase(line: &str) -> Option<&'static str> { + let lower = line.to_ascii_lowercase(); + if lower.contains("capturing cuda graph") + || lower.contains("capturing the model") + || lower.contains("warming up") + || lower.contains("warmup") + { + Some("warmup") + } else if lower.contains("loading weights") + || lower.contains("loading model") + || lower.contains("load_tensors") + || lower.contains("llama_model_loader") + || lower.contains("model loading took") + { + Some("loading") + } else if lower.contains("downloading") || lower.contains("fetching") { + Some("downloading") + } else { + None + } +} + +/// Read log bytes appended since `*pos`, advance `*pos`, and return the most +/// recent recognizable startup phase in that new output (later lines win, so a +/// download → load → warmup progression advances naturally). +/// +/// Best-effort: any I/O error (file not created yet, transient read) yields +/// `None`. A shrunk file (rotation/truncation) resets the cursor to the top. +fn read_new_log_phase(log_path: &Path, pos: &mut u64) -> Option<&'static str> { + use std::io::{Read, Seek, SeekFrom}; + let mut file = fs::File::open(log_path).ok()?; + let len = file.metadata().ok()?.len(); + if len < *pos { + *pos = 0; + } + if len == *pos { + return None; + } + file.seek(SeekFrom::Start(*pos)).ok()?; + let mut bytes = Vec::new(); + let read = file.read_to_end(&mut bytes).ok()?; + *pos += read as u64; + let text = String::from_utf8_lossy(&bytes); + let mut phase = None; + for line in text.lines() { + if let Some(found) = classify_startup_phase(last_cr_segment(line)) { + phase = Some(found); + } + } + phase +} + +fn engine_healthcheck_ready(paths: &AppPaths, engine: &str, service_id: &str) -> Result { + Ok(crate::common::healthcheck_response_ready( + &crate::common::engine_healthcheck_response(paths, engine, service_id)?, + )) +} + +/// Poll a freshly-spawned service until its healthcheck reports ready (or the +/// timeout elapses), tailing its log file meanwhile and reporting each coarse +/// startup phase transition via `on_phase`. +fn wait_for_service_ready( + paths: &AppPaths, + engine: &str, + service_id: &str, + log_path: &Path, + timeout: Duration, + mut on_phase: impl FnMut(&str), +) -> bool { + let start = std::time::Instant::now(); + let mut log_pos: u64 = 0; + let mut last_phase: Option<&'static str> = None; + while start.elapsed() < timeout { + if let Some(phase) = read_new_log_phase(log_path, &mut log_pos) + && last_phase != Some(phase) + { + last_phase = Some(phase); + on_phase(phase); + } + if engine_healthcheck_ready(paths, engine, service_id).unwrap_or(false) { + return true; + } + thread::sleep(Duration::from_millis(200)); + } + false +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::persistence::load_service_record; + use crate::test_support::{identity_probe_record, temp_app_paths, unique_test_root}; + use crate::watchers::find_recoverable_service; + + /// Drive `supervise_service` far enough to reach the key guard, and return + /// what it did. + /// + /// The guard sits before the manifest write and well before any spawn, so a + /// refusal returns without starting a process — which is what makes the real + /// call site testable at all. The arguments below are the shape a recovery + /// re-exec passes: a loopback bind, no GPU, no recipe. + /// + /// This exists because testing `ensure_public_service_has_endpoint_key` + /// directly with literal arguments cannot catch the defect that actually + /// happened twice in this crate's history — the guard being *wired up* with + /// the wrong value at its call site. + /// + /// Bounded, and the bound is the assertion. A guard that fails to refuse + /// does not return an error — it falls through to the engine spawn and + /// supervises a child that never exits, so an unbounded call would hang the + /// suite instead of failing it. Both callers below are regression tests for + /// a fail-*open*, which is exactly the shape that turns into a hang. + fn supervise_at_the_guard(paths: &AppPaths, service_id: &str) -> Result<()> { + let paths = paths.clone(); + let service_id = service_id.to_owned(); + let (sender, receiver) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let outcome = supervise_service( + &paths, + service_id, + "llamacpp".to_owned(), + "a-model".to_owned(), + "a-model".to_owned(), + None, + None, + "127.0.0.1".to_owned(), + 11434, + "gpu_required".to_owned(), + None, + None, + ); + let _ = sender.send(outcome.map_err(|error| format!("{error:#}"))); + }); + receiver + .recv_timeout(std::time::Duration::from_secs(30)) + .unwrap_or_else(|_| { + panic!( + "supervise_service did not return within 30s: the key guard let the call \ + through and it reached the engine spawn, which is the fail-open this test \ + exists to catch" + ) + }) + .map_err(anyhow::Error::msg) + } + + /// Write a service record into the registry the way a live service would + /// have left it behind. + fn seed_registry(paths: &AppPaths, service_id: &str, requires_api_key: bool) { + fs::create_dir_all(paths.services_dir()).unwrap(); + let mut record = ManagedServiceRecord::new( + paths, + service_id.to_owned(), + "llamacpp".to_owned(), + "a-model".to_owned(), + "a-model".to_owned(), + "127.0.0.1".to_owned(), + 11434, + "managed", + std::process::id(), + None, + None, + Some("gpu_required".to_owned()), + ); + record.requires_api_key = requires_api_key; + record.write().unwrap(); + } + + /// Drive `supervise_service` and report whether the key guard let it past. + /// + /// Decided on what the call *returns*, not on any file. The obvious + /// observable — the manifest appearing — is useless here, because + /// `seed_registry` has already written one, so polling for it passes + /// whatever the guard does. That mistake was made first and caught by + /// mutating the code the test claims to protect. + /// + /// A call the guard admits does not return: it carries on to the engine + /// spawn. So the guard's refusal is the only thing that comes back quickly, + /// and it is identified by its message rather than by the mere fact of an + /// error — a later, unrelated failure must not read as a refusal. + fn guard_admits(paths: &AppPaths, service_id: &str) -> bool { + let owned_paths = paths.clone(); + let owned_id = service_id.to_owned(); + let (sender, receiver) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let outcome = supervise_service( + &owned_paths, + owned_id, + "llamacpp".to_owned(), + "a-model".to_owned(), + "a-model".to_owned(), + None, + None, + "127.0.0.1".to_owned(), + 11434, + "gpu_required".to_owned(), + None, + None, + ); + let _ = sender.send(outcome.map_err(|error| format!("{error:#}"))); + }); + + match receiver.recv_timeout(std::time::Duration::from_secs(10)) { + // The key guard refused, by its own words. + Ok(Err(rendered)) if rendered.contains("without authentication") => false, + // Anything else means it got past the guard: it either finished, or + // failed later for a reason that is not this guard, or is still + // running because it reached the spawn. + _ => true, + } + } + + #[test] + fn a_service_that_never_required_a_key_is_not_refused_for_lacking_one() { + // The other direction of the guard, and the one no test covered. + // Hardcoding `record.requires_api_key = true` at the restore site passes + // every other test in this crate, because they all seed a service that + // *does* require a key. This is the case that catches it. + // + // Two records are seeded, not one: with a single record the + // `existing.service_id == record.service_id` half of the lookup does + // nothing, so dropping that comparison would go unnoticed and one + // service's requirement would leak onto another's. + let (root, paths) = temp_app_paths("supervise-no-key-needed"); + seed_registry(&paths, "svc-needs-key", true); + seed_registry(&paths, "svc-plain", false); + + assert!( + guard_admits(&paths, "svc-plain"), + "a loopback service that never asked for a key must not be refused for lacking one" + ); + + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn supervising_a_service_that_required_a_key_refuses_when_the_key_is_gone() { + // The real call site, not the guard in isolation. `supervise_service` + // rebuilds the record with `ManagedServiceRecord::new`, which starts + // `requires_api_key` false, and restores it from the registry. Passing + // the wrong value here — a literal, or the freshly-built field before it + // is restored — is exactly the miswiring that shipped twice in this + // crate and that a literal-argument unit test cannot see. + let (root, paths) = temp_app_paths("supervise-requires-key"); + seed_registry(&paths, "svc-needs-key", true); + + let error = supervise_at_the_guard(&paths, "svc-needs-key") + .expect_err("a service that required a key must not be recovered without one"); + assert!( + format!("{error:#}").contains("without authentication"), + "{error:#}" + ); + + // And the refusal must not have weakened what is on disk. The guard runs + // before `record.write()` precisely so a refused attempt leaves the + // recorded requirement armed for the next attempt. + let stored = crate::persistence::load_managed_services(&paths).unwrap(); + let stored = stored + .iter() + .find(|candidate| candidate.service_id == "svc-needs-key") + .expect("the seeded record must survive a refused recovery"); + assert!( + stored.requires_api_key, + "a refused recovery must not disarm the requirement" + ); + + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn a_registry_that_cannot_be_read_refuses_recovery_rather_than_assuming_no_key() { + // `load_managed_services` already skips records it cannot parse, so an + // `Err` from it is a real I/O failure — and a missing directory is `Ok`. + // Defaulting it away therefore says "no service ever required a key", + // which is fail-open on an auth gate and, worse, gets written back. + // + // The failure is provoked portably: a directory named like a record + // makes the `fs::read` inside the loop fail rather than the read_dir. + let (root, paths) = temp_app_paths("supervise-unreadable-registry"); + fs::create_dir_all(paths.services_dir().join("not-a-record.json")).unwrap(); + + let error = supervise_at_the_guard(&paths, "svc-unknown") + .expect_err("an unreadable registry must refuse, not assume no key was required"); + let rendered = format!("{error:#}"); + assert!( + rendered.contains("could not read the service registry"), + "{rendered}" + ); + + // Nothing was written: a registry we could not read is not a registry we + // may add a weakened record to. + assert!( + !paths.service_manifest_path("svc-unknown").exists(), + "a refused recovery must not persist a record" + ); + + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn last_cr_segment_keeps_final_progress_redraw() { + // A tqdm/HF-style in-place redraw collapses to its last segment. + assert_eq!( + last_cr_segment("Downloading: 10%\rDownloading: 55%\rDownloading: 100%"), + "Downloading: 100%" + ); + // A plain line is unchanged. + assert_eq!( + last_cr_segment("Loading model weights"), + "Loading model weights" + ); + } + + #[test] + fn classify_startup_phase_maps_engine_vocabulary() { + assert_eq!( + classify_startup_phase("Downloading shards: 100%"), + Some("downloading") + ); + assert_eq!( + classify_startup_phase("Fetching 12 files"), + Some("downloading") + ); + assert_eq!( + classify_startup_phase("INFO: Loading model weights took 4.2s"), + Some("loading") + ); + assert_eq!( + classify_startup_phase("llama_model_loader: loaded meta data"), + Some("loading") + ); + assert_eq!( + classify_startup_phase("Capturing CUDA graph shapes"), + Some("warmup") + ); + assert_eq!( + classify_startup_phase("Warming up the engine"), + Some("warmup") + ); + // Ordinary chatter carries no phase signal. + assert_eq!( + classify_startup_phase("Uvicorn running on http://..."), + None + ); + } + + #[test] + fn classify_startup_phase_emits_only_dashboard_known_tokens() { + // These tokens are the wire contract with the dashboard's + // `StartupPhase::from_token` (rocm-dash-core); emitting anything else + // would be silently dropped there. rocmd can't link that crate, so the + // contract is pinned here by literal. + for line in [ + "Downloading shards", + "Loading model weights", + "Capturing CUDA graph", + ] { + let token = classify_startup_phase(line).expect("line is a phase signal"); + assert!( + matches!(token, "downloading" | "loading" | "warmup"), + "token {token:?} must be one the dashboard understands" + ); + } + } + + #[test] + fn read_new_log_phase_advances_and_tracks_latest() { + use std::io::Write as _; + // Workspace-local test root (rooted at CARGO_MANIFEST_DIR, not the + // ambient temp dir) — same helper the other rocmd tests use. + let dir = unique_test_root(&format!("rocmd-phase-{}", std::process::id())); + let log = dir.join("svc.log"); + std::fs::write(&log, "boot\nDownloading shards: 100%\n").unwrap(); + + let mut pos = 0_u64; + assert_eq!(read_new_log_phase(&log, &mut pos), Some("downloading")); + // No new bytes → no phase, cursor unchanged. + let after_first = pos; + assert_eq!(read_new_log_phase(&log, &mut pos), None); + assert_eq!(pos, after_first); + + // Appending a later stage advances the phase. + let mut f = std::fs::OpenOptions::new().append(true).open(&log).unwrap(); + writeln!(f, "Loading model weights took 3s").unwrap(); + assert_eq!(read_new_log_phase(&log, &mut pos), Some("loading")); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn engine_serve_http_args_forward_engine_recipe_json() { + let engine_recipe_json = r#"{"contract_version":"0.1.0","engine":"vllm","required_flags":["--enable-auto-tool-choice"]}"#; + let args = engine_serve_http_args( + "vllm", + "svc-1", + "Qwen/Qwen3.5-4B", + "127.0.0.1", + 11435, + "gpu_required", + &[], + Some("therock-release:gfx120X-all"), + Some("env-1"), + Some(engine_recipe_json), + Path::new("state.json"), + ); + + assert!( + args.windows(2) + .any(|pair| pair[0] == "--engine-recipe-json" && pair[1] == engine_recipe_json) + ); + assert!( + args.windows(2).any(|pair| { + pair[0] == "--runtime-id" && pair[1] == "therock-release:gfx120X-all" + }) + ); + assert!( + args.windows(2) + .any(|pair| pair[0] == "--state-path" && pair[1] == "state.json") + ); + } + + #[test] + fn engine_serve_http_args_emit_gpu_indices_when_pinned() { + let args = engine_serve_http_args( + "vllm", + "svc-1", + "Qwen/Qwen3.5-4B", + "127.0.0.1", + 11435, + "gpu_required", + &[1], + None, + None, + None, + Path::new("state.json"), + ); + + assert!( + args.windows(2) + .any(|pair| pair[0] == "--gpu" && pair[1] == "1") + ); + + let auto = engine_serve_http_args( + "vllm", + "svc-1", + "Qwen/Qwen3.5-4B", + "127.0.0.1", + 11435, + "gpu_required", + &[], + None, + None, + None, + Path::new("state.json"), + ); + assert!(!auto.iter().any(|arg| arg == "--gpu")); + } + + /// The identity check is only as good as the token the launcher writes down. + /// `rocm_core::identity_state` treats a missing token as a best-effort match + /// — the legacy `(None, _)` branch — so a record written without one is + /// signalled blind, which is the original defect restored in full. Every + /// other test here hands itself a record with the token already set, and so + /// proves nothing about the code that is supposed to put it there. + /// + /// This drives the real `supervise_service` and reads the manifest it + /// persisted. It is stopped at that first write by making the service log + /// path a directory, so the `fs::File::create` immediately after it fails: + /// going further reaches the engine spawn, and the failure path past that + /// calls `std::process::exit`, which inside a test binary would take the + /// whole suite with it. + /// + /// Linux-only: `rocm_core::process_start_ticks` reads `/proc` and returns + /// `None` everywhere else, so on other platforms there is no token to write. + #[cfg(target_os = "linux")] + #[test] + fn supervise_service_persists_the_supervisor_identity_token() -> Result<()> { + let (root, paths) = temp_app_paths("supervise-records-identity"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-supervise-identity"; + let log_path = identity_probe_record(&paths, service_id, 11443).log_path; + // A directory where the supervisor expects to create a file. + fs::create_dir_all(&log_path)?; + + let outcome = supervise_service( + &paths, + service_id.to_owned(), + "llamacpp".to_owned(), + "a-model".to_owned(), + "a-model".to_owned(), + None, + None, + "127.0.0.1".to_owned(), + 11443, + "gpu_required".to_owned(), + None, + None, + ); + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let error = + outcome.expect_err("the log path is a directory, so the engine spawn is unreachable"); + assert!( + format!("{error:#}").contains("failed to create"), + "stopped somewhere other than the log file: {error:#}" + ); + assert!( + persisted?.supervisor_start_ticks.is_some(), + "supervise_service must persist the supervisor's start-time token beside its PID" + ); + Ok(()) + } + + /// The supervisor's last write lands after its engine child exits — which + /// is what a stop that could not verify the supervisor, but did kill the + /// engine, leaves behind. Writing back the record the supervisor built at + /// startup would erase the stop marker the stop persisted, and a `failed` + /// record with no marker is exactly what the daemon's recovery restarts. + #[test] + fn supervisor_exit_write_keeps_a_standing_stop_marker() -> Result<()> { + let (root, paths) = temp_app_paths("supervise-exit-keeps-marker"); + paths.ensure()?; + let service_id = "svc-supervise-exit-marker"; + // The supervisor's own copy, as `supervise_service` builds it: no marker. + let mut supervisor = identity_probe_record(&paths, service_id, 11446); + supervisor.record_supervisor_identity(std::process::id()); + supervisor.status = "running".to_owned(); + // What is on disk once a stop has run: same processes, marker set. + let mut on_disk = supervisor.clone(); + on_disk.stop_requested_unix_ms = Some(1); + on_disk.write()?; + + record_supervised_exit(&paths, &supervisor, false)?; + let persisted = load_service_record(&paths, service_id); + let recoverable = find_recoverable_service(&paths); + fs::remove_dir_all(root).ok(); + + let persisted = persisted?; + assert_eq!( + persisted.status, "failed", + "the supervisor's own field lands" + ); + assert_eq!( + persisted.stop_requested_unix_ms, + Some(1), + "the supervisor's exit write must not withdraw a standing stop" + ); + assert!( + recoverable?.is_none(), + "a stopped service must not become a recovery candidate" + ); + Ok(()) + } + + /// A record that names another supervisor — a confirmed stop cleared the + /// PIDs, or a restart recorded its own — or that was removed outright, is + /// no longer this supervisor's to write. + #[test] + fn supervisor_exit_write_leaves_a_record_it_no_longer_owns() -> Result<()> { + let (root, paths) = temp_app_paths("supervise-exit-not-owner"); + paths.ensure()?; + let service_id = "svc-supervise-exit-not-owner"; + let mut supervisor = identity_probe_record(&paths, service_id, 11447); + supervisor.record_supervisor_identity(std::process::id()); + supervisor.status = "running".to_owned(); + // A confirmed stop's final write: status stopped, PIDs cleared. + let mut stopped = supervisor.clone(); + stopped.status = "stopped".to_owned(); + stopped.supervisor_pid = 0; + stopped.supervisor_start_ticks = None; + stopped.write()?; + + record_supervised_exit(&paths, &supervisor, false)?; + let after_stop = load_service_record(&paths, service_id); + + fs::remove_file(paths.service_manifest_path(service_id))?; + record_supervised_exit(&paths, &supervisor, false)?; + let resurrected = paths.service_manifest_path(service_id).exists(); + fs::remove_dir_all(root).ok(); + + assert_eq!(after_stop?.status, "stopped"); + assert!(!resurrected, "a removed record must not be written back"); + Ok(()) + } + + /// The supervisor's first write rebuilds the record from its arguments. + /// The restart that launched it cleared any stop marker first, so a marker + /// on disk here was set by a stop issued since, and must survive. + #[cfg(target_os = "linux")] + #[test] + fn supervise_service_first_write_keeps_a_standing_stop_marker() -> Result<()> { + let (root, paths) = temp_app_paths("supervise-first-write-keeps-marker"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-supervise-first-write-marker"; + let mut seeded = identity_probe_record(&paths, service_id, 11448); + seeded.stop_requested_unix_ms = Some(1); + seeded.write()?; + // Stop `supervise_service` right after its first write; see + // `supervise_service_persists_the_supervisor_identity_token`. + fs::create_dir_all(&seeded.log_path)?; + + let outcome = supervise_service( + &paths, + service_id.to_owned(), + "llamacpp".to_owned(), + "a-model".to_owned(), + "a-model".to_owned(), + None, + None, + "127.0.0.1".to_owned(), + 11448, + "gpu_required".to_owned(), + None, + None, + ); + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + assert!(outcome.is_err(), "the log path is a directory"); + assert_eq!(persisted?.stop_requested_unix_ms, Some(1)); + Ok(()) + } +} diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index 54e985ea14..878888038c 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -1539,7 +1539,7 @@ pub(crate) fn restart_managed_service( // Not a write-back of `record`: a stop issued since the spawn has // persisted its marker, and a `failed` record without one is what // recovery restarts. Same discipline as the supervisor's own writes. - crate::service::update_supervised_record(paths, record, |current| { + crate::supervise::update_supervised_record(paths, record, |current| { current.status = "failed".to_owned(); })?; anyhow::bail!( @@ -1573,7 +1573,7 @@ fn record_spawned_supervisor( record.record_supervisor_identity(pid); let (supervisor_pid, supervisor_start_ticks) = (record.supervisor_pid, record.supervisor_start_ticks); - crate::service::update_supervised_record(paths, &unrecorded, |current| { + crate::supervise::update_supervised_record(paths, &unrecorded, |current| { current.supervisor_pid = supervisor_pid; current.supervisor_start_ticks = supervisor_start_ticks; }) diff --git a/crates/rocm-dash-daemon/src/registry.rs b/crates/rocm-dash-daemon/src/registry.rs index cf321d757e..6f02aa8f4c 100644 --- a/crates/rocm-dash-daemon/src/registry.rs +++ b/crates/rocm-dash-daemon/src/registry.rs @@ -89,7 +89,7 @@ pub fn is_scrapeable_status(status: &str) -> bool { /// for safety. /// /// **`running` + `startup_phase` overlap:** the rocm-cli supervisor -/// (`apps/rocmd/src/lib.rs::supervise_service`) flips the on-disk record to +/// (`apps/rocmd/src/supervise.rs::supervise_service`) flips the on-disk record to /// `running` immediately after spawning the engine child — *before* it starts /// polling and parsing `startup_phase` from the serve log — and only clears /// `startup_phase` once the service reaches `ready`. So a record can be @@ -432,7 +432,7 @@ mod tests { #[test] fn discovered_from_record_surfaces_startup_phase_while_running() { - // The real producer (`apps/rocmd/src/lib.rs::supervise_service`) flips + // The real producer (`apps/rocmd/src/supervise.rs::supervise_service`) flips // the on-disk record to `running` *before* it starts polling and // parsing `startup_phase` from the serve log, and only clears // `startup_phase` once the service reaches `ready`. Exercise that diff --git a/docs/architecture.md b/docs/architecture.md index d00028d482..802cb6de13 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -89,7 +89,8 @@ Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): - [`persistence.rs`](../apps/rocmd/src/persistence.rs) — `record_event`/`load_managed_services`/`load_service_record`, the automation-event/audit-log and managed-service-registry I/O shared across the daemon's sandbox, MCP, service-lifecycle, and watcher code. - [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) — bubblewrap/native sandbox execution, atomic-write helpers, artifact prefetch policy gating, and the sandbox-tool result shaping for `check_updates`/`driver_plan`/`stop_server`. - [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs) — the sandbox-tool vocabulary: the `SandboxToolArg` value enum naming each sandbox tool, and `SandboxToolPolicy`, the artifact-download policy built from the `--allow-*` flags (including the Hugging Face token read from the environment when authenticated downloads are allowed). It uses no other module of the crate, so modules that need the vocabulary do not import `cli.rs`. -- [`service.rs`](../apps/rocmd/src/service.rs) — managed-service PID lifecycle/stop, `run_daemon`'s foreground loop, `supervise_service`'s spawn-and-recover path, and serve-log startup-phase polling. +- [`service.rs`](../apps/rocmd/src/service.rs) — managed-service PID lifecycle/stop, and `run_daemon`'s foreground loop with `print_status`. +- [`supervise.rs`](../apps/rocmd/src/supervise.rs) — the `rocmd supervise` side of a managed service: `supervise_service` (launch the engine child, record its identity, wait for readiness, record the exit), `update_supervised_record` (the re-read-then-write update that every write after the first goes through, which leaves a stop marker or another supervisor's record alone), and the serve-log startup-phase polling and readiness wait it uses. Its production code uses only [`common.rs`](../apps/rocmd/src/common.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs). `cli.rs` calls `supervise_service`, and `watchers.rs` calls `update_supervised_record`. - [`watchers.rs`](../apps/rocmd/src/watchers.rs) — event collection/dispatch for all built-in watchers (TheRock update, GPU metrics/thermal-pressure, cache-warm, driver-upgrade, server-recover), plus managed-service recovery classification and automation-proposal queuing. - [`webhook.rs`](../apps/rocmd/src/webhook.rs) — the local webhook source: its axum routes, request validation, and watcher-kind allow-list. From 35a4494ecbfe5e14d097572ed1251c029ad6f4d0 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 2 Nov 2026 10:57:34 +0000 Subject: [PATCH 05/13] refactor(rocmd): move the recovery respawn into supervise.rs Move restart_managed_service, record_spawned_supervisor, recovery_supervise_args and both cfg variants of detached_rocmd_command out of watchers.rs into supervise.rs, next to supervise_service, which is what recovery_supervise_args builds the arguments for. The six tests that exercise only these symbols move with them (recovery_supervise_args_preserve_engine_recipe_json, restart_managed_service_persists_the_supervisor_identity_token, restart_managed_service_drops_the_replaced_runs_engine_pid, restart_failure_write_keeps_a_stop_marker_set_meanwhile, restart_records_its_supervisor_without_erasing_a_stop_marker, restart_leaves_a_record_naming_another_supervisor_or_none; three are Linux-only). watchers.rs and sandbox.rs now import restart_managed_service from supervise.rs, so supervise.rs no longer depends on watchers.rs in production code. Part of the rocmd module-cycle follow-up work. Besides the moved lines and import wiring, this commit changes: update_supervised_record becomes private, since its only remaining callers are now inside supervise.rs; the imports that only the moved code used are dropped from watchers.rs, and its test module takes its own use std::fs; the supervise.rs test module imports UNCONFIRMED_STOP_PID only on Linux, since only a Linux-only test uses it; docs/architecture.md describes the recovery respawn in the supervise.rs entry. Signed-off-by: Jussi Elo --- apps/rocmd/src/sandbox.rs | 2 +- apps/rocmd/src/supervise.rs | 393 ++++++++++++++++++++++++++++++++++- apps/rocmd/src/watchers.rs | 395 +----------------------------------- docs/architecture.md | 2 +- 4 files changed, 396 insertions(+), 396 deletions(-) diff --git a/apps/rocmd/src/sandbox.rs b/apps/rocmd/src/sandbox.rs index 0c64f8ce8d..01b6d0ab71 100644 --- a/apps/rocmd/src/sandbox.rs +++ b/apps/rocmd/src/sandbox.rs @@ -6,7 +6,7 @@ use crate::common; use crate::persistence::load_managed_services; use crate::sandbox_tool::{SandboxToolArg, SandboxToolPolicy}; use crate::service::{stop_managed_service, stop_report_confirmed}; -use crate::watchers::restart_managed_service; +use crate::supervise::restart_managed_service; use anyhow::{Context, Result, bail}; use rocm_core::{ AppPaths, AuditEventRecord, CommandCapture, ExamineSummary, ModelRecipeArtifactRecord, diff --git a/apps/rocmd/src/supervise.rs b/apps/rocmd/src/supervise.rs index 3947a01a64..7719de84c6 100644 --- a/apps/rocmd/src/supervise.rs +++ b/apps/rocmd/src/supervise.rs @@ -217,7 +217,7 @@ pub(crate) fn supervise_service( /// recorded PIDs, and a restart records its own. Either way the record now /// describes something this supervisor is not, and its view of the service /// (`running`, `failed`, …) would overwrite a truer one. -pub(crate) fn update_supervised_record( +fn update_supervised_record( paths: &AppPaths, supervisor: &ManagedServiceRecord, update: impl FnOnce(&mut ManagedServiceRecord), @@ -388,10 +388,165 @@ fn wait_for_service_ready( false } +pub(crate) fn restart_managed_service( + paths: &AppPaths, + record: &mut ManagedServiceRecord, +) -> Result<()> { + let rocmd_binary = + std::env::current_exe().context("failed to resolve current rocmd executable path")?; + let log_file = fs::OpenOptions::new() + .create(true) + .append(true) + .open(&record.log_path) + .with_context(|| format!("failed to open {}", record.log_path.display()))?; + let log_file_err = log_file + .try_clone() + .context("failed to clone service log file handle")?; + + record.status = "recovering".to_owned(); + // Counts the restart and drops the previous run's inference verification. + // The respawned child writes a fresh record of its own, and "recovering" is + // outside the statuses that probe, so a stale verdict would not currently be + // acted on — but this record is written again below, after the spawn, and + // that write can land after the child's. Clearing here keeps the invariant + // true at the one site that reuses a record across restarts. + record.reset_for_restart(); + // No supervisor exists until the spawn below succeeds, so none is recorded. + // Recording this daemon's own PID here, as a stand-in, left the manifest + // naming the daemon as the service's supervisor whenever the spawn failed: + // a stop run from any other process (`rocmd sandbox-tool stop_server`, + // `rocm services stop`) verifies that identity, finds it live, and + // terminates the daemon's whole process tree. The spawned supervisor's + // identity is recorded right after the spawn. + record.supervisor_pid = 0; + record.supervisor_start_ticks = None; + // The engine PID belonged to the run being replaced, and nothing here + // starts an engine: the supervisor spawned below records its own, together + // with its token. Carried forward, the old PID would sit beside a + // supervisor it no longer matches — and on a record `rocm` wrote, with no + // token at all, so the next stop could only signal it blind. + record.engine_pid = None; + record.engine_start_ticks = None; + // Restarting is a request to run, so it supersedes any earlier stop request, + // as `rocm services restart` treats it too. + record.stop_requested_unix_ms = None; + record.write()?; + + let mut child = detached_rocmd_command(&rocmd_binary) + .args(recovery_supervise_args(record)) + .stdin(Stdio::null()) + .stdout(Stdio::from(log_file)) + .stderr(Stdio::from(log_file_err)) + .spawn() + .context("failed to spawn recovery supervisor")?; + + record_spawned_supervisor(paths, record, child.id())?; + + thread::sleep(Duration::from_millis(200)); + if let Some(status) = child + .try_wait() + .context("failed to check recovery supervisor startup state")? + { + // Not a write-back of `record`: a stop issued since the spawn has + // persisted its marker, and a `failed` record without one is what + // recovery restarts. Same discipline as the supervisor's own writes. + update_supervised_record(paths, record, |current| { + current.status = "failed".to_owned(); + })?; + anyhow::bail!( + "recovery supervisor exited immediately with status {status}; inspect {}", + record.log_path.display() + ); + } + + Ok(()) +} + +/// Record the supervisor [`restart_managed_service`] just spawned, as `pid`. +/// +/// Between the restart's first write — which clears the recorded PIDs, since +/// no supervisor exists yet — and this one, the record names no process at +/// all. A stop that starts in that window persists its marker first and then +/// finds nothing to signal; writing the restart's whole copy back here would +/// erase that marker while the new supervisor comes up, and leave the stop's +/// own final write comparing against a record that no longer shows the +/// process it has to answer for. So only the supervisor's identity is applied, +/// to the record as it is on disk now, through the same ownership guard as the +/// supervisor's own writes: against the copy from before this assignment, so +/// it is the cleared PID that has to still be there. If the new supervisor has +/// already recorded itself, or the record was removed, nothing is written. +fn record_spawned_supervisor( + paths: &AppPaths, + record: &mut ManagedServiceRecord, + pid: u32, +) -> Result<()> { + let unrecorded = record.clone(); + record.record_supervisor_identity(pid); + let (supervisor_pid, supervisor_start_ticks) = + (record.supervisor_pid, record.supervisor_start_ticks); + update_supervised_record(paths, &unrecorded, |current| { + current.supervisor_pid = supervisor_pid; + current.supervisor_start_ticks = supervisor_start_ticks; + }) +} + +fn recovery_supervise_args(record: &ManagedServiceRecord) -> Vec { + let mut args = vec![ + "supervise".to_owned(), + record.service_id.clone(), + "--engine".to_owned(), + record.engine.clone(), + "--model-ref".to_owned(), + record.model_ref.clone(), + "--canonical-model-id".to_owned(), + record.canonical_model_id.clone(), + "--host".to_owned(), + record.host.clone(), + "--port".to_owned(), + record.port.to_string(), + "--device-policy".to_owned(), + record + .device_policy + .as_deref() + .unwrap_or("gpu_required") + .to_owned(), + ]; + args.extend(crate::common::optional_arg( + "--runtime-id", + record.runtime_id.as_deref(), + )); + args.extend(crate::common::optional_arg( + "--env-id", + record.env_id.as_deref(), + )); + if let Some(csv) = rocm_engine_protocol::gpu_indices_to_csv(&record.gpu_indices) { + args.extend(["--gpu".to_owned(), csv]); + } + args.extend(crate::common::optional_arg( + "--engine-recipe-json", + record.engine_recipe_json.as_deref(), + )); + args +} + +#[cfg(unix)] +fn detached_rocmd_command(rocmd_binary: &std::path::Path) -> ProcessCommand { + let mut command = ProcessCommand::new("setsid"); + command.arg(rocmd_binary); + command +} + +#[cfg(not(unix))] +fn detached_rocmd_command(rocmd_binary: &std::path::Path) -> ProcessCommand { + ProcessCommand::new(rocmd_binary) +} + #[cfg(test)] mod tests { use super::*; use crate::persistence::load_service_record; + #[cfg(target_os = "linux")] + use crate::test_support::UNCONFIRMED_STOP_PID; use crate::test_support::{identity_probe_record, temp_app_paths, unique_test_root}; use crate::watchers::find_recoverable_service; @@ -927,4 +1082,240 @@ mod tests { assert_eq!(persisted?.stop_requested_unix_ms, Some(1)); Ok(()) } + + #[test] + fn recovery_supervise_args_preserve_engine_recipe_json() { + let (_root, paths) = temp_app_paths("recovery-engine-recipe"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-1", + "vllm", + "qwen", + "Qwen/Qwen3.5-4B", + "127.0.0.1", + 11435, + "managed", + 123, + Some("therock-release:gfx120X-all".to_owned()), + Some("env-1".to_owned()), + Some("gpu_required".to_owned()), + ); + let engine_recipe_json = r#"{"contract_version":"0.1.0","engine":"vllm","required_flags":["--enable-auto-tool-choice"]}"#; + record.engine_recipe_json = Some(engine_recipe_json.to_owned()); + + let args = recovery_supervise_args(&record); + + assert!( + args.windows(2) + .any(|pair| pair[0] == "--engine-recipe-json" && pair[1] == engine_recipe_json) + ); + assert!( + args.windows(2) + .any(|pair| { pair[0] == "--canonical-model-id" && pair[1] == "Qwen/Qwen3.5-4B" }) + ); + } + + /// The same guarantee for the daemon's recovery path, which records a PID of + /// its own. See `supervise_service_persists_the_supervisor_identity_token` + /// for why an unrecorded token is the defect rather than a cosmetic gap. + /// + /// The restart re-execs `rocmd supervise`, which here is the test binary: + /// it rejects those arguments and exits at once, so the restart reports + /// failure — after it has written the record this test reads. + #[cfg(target_os = "linux")] + #[test] + fn restart_managed_service_persists_the_supervisor_identity_token() -> Result<()> { + let (root, paths) = temp_app_paths("restart-records-identity"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-restart-identity"; + let mut record = identity_probe_record(&paths, service_id, 11444); + record.status = "failed".to_owned(); + record.write()?; + assert_eq!( + load_service_record(&paths, service_id)?.supervisor_start_ticks, + None, + "precondition: the seeded record carries no token" + ); + + let _ = restart_managed_service(&paths, &mut record); + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + assert!( + persisted?.supervisor_start_ticks.is_some(), + "restart_managed_service must persist the supervisor's start-time token beside its PID" + ); + Ok(()) + } + + /// A restart moves the supervisor PID; the engine PID it carried belonged + /// to the run being replaced. On a record `rocm serve --background` wrote, + /// that PID has no token, so carrying it past the point where the two PIDs + /// diverge leaves the next stop an entry it can only signal blind. + /// + /// Linux-only for the same reason as the test above: the restart re-execs + /// the test binary, which is only known to reject those arguments there. + #[cfg(target_os = "linux")] + #[test] + fn restart_managed_service_drops_the_replaced_runs_engine_pid() -> Result<()> { + let (root, paths) = temp_app_paths("restart-drops-engine-pid"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-restart-drops-engine-pid"; + let mut record = identity_probe_record(&paths, service_id, 11455); + // The shape `rocm serve --background` writes: one PID in both roles. + record.supervisor_pid = UNCONFIRMED_STOP_PID; + record.engine_pid = Some(UNCONFIRMED_STOP_PID); + record.engine_start_ticks = None; + record.stop_requested_unix_ms = Some(1); + record.status = "failed".to_owned(); + record.write()?; + + let _ = restart_managed_service(&paths, &mut record); + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let persisted = persisted?; + assert_ne!(persisted.supervisor_pid, UNCONFIRMED_STOP_PID); + assert_eq!( + persisted.engine_pid, None, + "the replaced run's engine PID must not outlive the restart" + ); + assert_eq!(persisted.engine_start_ticks, None); + assert_eq!( + persisted.stop_requested_unix_ms, None, + "a restart supersedes an earlier stop request" + ); + Ok(()) + } + + /// A stop that lands while a restart waits to see whether its supervisor + /// survived startup must outlive the restart's `failed` write: a `failed` + /// record without the marker is exactly what recovery restarts. + /// + /// The restarted supervisor is the test binary, which rejects the + /// `supervise` arguments and exits at once, so the restart reaches that + /// write. A thread plays the stop: once the restart has recorded the + /// spawned supervisor, it persists a marker, as a stop does first. + #[cfg(target_os = "linux")] + #[test] + fn restart_failure_write_keeps_a_stop_marker_set_meanwhile() -> Result<()> { + let (root, paths) = temp_app_paths("restart-failure-keeps-marker"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-restart-failure-marker"; + let mut record = identity_probe_record(&paths, service_id, 11456); + record.status = "failed".to_owned(); + record.write()?; + + let stop = { + let paths = paths.clone(); + thread::spawn(move || -> Result { + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while std::time::Instant::now() < deadline { + // `write` is not atomic, so a read can land mid-write and + // see a truncated file; just look again. + let Ok(mut on_disk) = load_service_record(&paths, service_id) else { + thread::sleep(Duration::from_millis(5)); + continue; + }; + if on_disk.supervisor_pid != 0 && on_disk.supervisor_pid != std::process::id() { + on_disk.stop_requested_unix_ms = Some(1); + on_disk.write()?; + return Ok(true); + } + thread::sleep(Duration::from_millis(5)); + } + Ok(false) + }) + }; + let outcome = restart_managed_service(&paths, &mut record); + let stop_landed = stop.join().expect("stop thread"); + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + assert!( + stop_landed?, + "precondition: the stop ran during the restart" + ); + assert!(outcome.is_err(), "the test binary is not a supervisor"); + let persisted = persisted?; + assert_eq!(persisted.status, "failed"); + assert_eq!( + persisted.stop_requested_unix_ms, + Some(1), + "the restart's failure write must not withdraw a stop issued meanwhile" + ); + Ok(()) + } + + /// Between a restart's first write, which clears the recorded PIDs, and the + /// one recording the supervisor it spawned, a stop can persist its marker. + /// Writing the restart's own copy back would erase it while the new + /// supervisor comes up — so only the identity is applied. + #[test] + fn restart_records_its_supervisor_without_erasing_a_stop_marker() -> Result<()> { + let (root, paths) = temp_app_paths("restart-spawn-keeps-marker"); + paths.ensure()?; + let service_id = "svc-restart-spawn-marker"; + // The restart's copy after its first write: no supervisor, no marker. + let mut restarting = identity_probe_record(&paths, service_id, 11457); + restarting.status = "recovering".to_owned(); + restarting.supervisor_pid = 0; + restarting.supervisor_start_ticks = None; + // A stop started since: same (empty) processes, marker set. + let mut on_disk = restarting.clone(); + on_disk.stop_requested_unix_ms = Some(1); + on_disk.write()?; + + record_spawned_supervisor(&paths, &mut restarting, std::process::id())?; + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let persisted = persisted?; + assert_eq!(restarting.supervisor_pid, std::process::id()); + assert_eq!(persisted.supervisor_pid, std::process::id()); + assert_eq!( + persisted.supervisor_start_ticks, + restarting.supervisor_start_ticks + ); + assert_eq!( + persisted.stop_requested_unix_ms, + Some(1), + "recording the spawned supervisor must not withdraw a standing stop" + ); + Ok(()) + } + + /// The spawned supervisor may record itself before the restart does, and + /// the record may be removed outright; either way the restart's write is + /// not the one to land. + #[test] + fn restart_leaves_a_record_naming_another_supervisor_or_none() -> Result<()> { + let (root, paths) = temp_app_paths("restart-spawn-not-owner"); + paths.ensure()?; + let service_id = "svc-restart-spawn-not-owner"; + let mut restarting = identity_probe_record(&paths, service_id, 11458); + restarting.supervisor_pid = 0; + restarting.supervisor_start_ticks = None; + let mut on_disk = restarting.clone(); + on_disk.supervisor_pid = 4_242_424; + on_disk.write()?; + + record_spawned_supervisor(&paths, &mut restarting.clone(), std::process::id())?; + let after_supervisor = load_service_record(&paths, service_id); + + fs::remove_file(paths.service_manifest_path(service_id))?; + record_spawned_supervisor(&paths, &mut restarting, std::process::id())?; + let resurrected = paths.service_manifest_path(service_id).exists(); + fs::remove_dir_all(root).ok(); + + assert_eq!(after_supervisor?.supervisor_pid, 4_242_424); + assert!(!resurrected, "a removed record must not be written back"); + Ok(()) + } } diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index 878888038c..1478957089 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -3,6 +3,7 @@ // SPDX-License-Identifier: MIT use crate::persistence::load_service_record; +use crate::supervise::restart_managed_service; use anyhow::{Context, Result, bail}; use rocm_core::{ AppPaths, AutomationProposalRecord, AutomationRuntimeState, AutomationTriggerEvent, @@ -12,9 +13,6 @@ use rocm_core::{ }; use serde_json::Value; use serde_json::json; -use std::fs; -use std::process::{Command as ProcessCommand, Stdio}; -use std::thread; use std::time::Duration; const SERVER_RECOVER_BACKOFF_MS: u128 = 30_000; @@ -1477,147 +1475,6 @@ fn manifest_service_recovery_reason( } } -pub(crate) fn restart_managed_service( - paths: &AppPaths, - record: &mut ManagedServiceRecord, -) -> Result<()> { - let rocmd_binary = - std::env::current_exe().context("failed to resolve current rocmd executable path")?; - let log_file = fs::OpenOptions::new() - .create(true) - .append(true) - .open(&record.log_path) - .with_context(|| format!("failed to open {}", record.log_path.display()))?; - let log_file_err = log_file - .try_clone() - .context("failed to clone service log file handle")?; - - record.status = "recovering".to_owned(); - // Counts the restart and drops the previous run's inference verification. - // The respawned child writes a fresh record of its own, and "recovering" is - // outside the statuses that probe, so a stale verdict would not currently be - // acted on — but this record is written again below, after the spawn, and - // that write can land after the child's. Clearing here keeps the invariant - // true at the one site that reuses a record across restarts. - record.reset_for_restart(); - // No supervisor exists until the spawn below succeeds, so none is recorded. - // Recording this daemon's own PID here, as a stand-in, left the manifest - // naming the daemon as the service's supervisor whenever the spawn failed: - // a stop run from any other process (`rocmd sandbox-tool stop_server`, - // `rocm services stop`) verifies that identity, finds it live, and - // terminates the daemon's whole process tree. The spawned supervisor's - // identity is recorded right after the spawn. - record.supervisor_pid = 0; - record.supervisor_start_ticks = None; - // The engine PID belonged to the run being replaced, and nothing here - // starts an engine: the supervisor spawned below records its own, together - // with its token. Carried forward, the old PID would sit beside a - // supervisor it no longer matches — and on a record `rocm` wrote, with no - // token at all, so the next stop could only signal it blind. - record.engine_pid = None; - record.engine_start_ticks = None; - // Restarting is a request to run, so it supersedes any earlier stop request, - // as `rocm services restart` treats it too. - record.stop_requested_unix_ms = None; - record.write()?; - - let mut child = detached_rocmd_command(&rocmd_binary) - .args(recovery_supervise_args(record)) - .stdin(Stdio::null()) - .stdout(Stdio::from(log_file)) - .stderr(Stdio::from(log_file_err)) - .spawn() - .context("failed to spawn recovery supervisor")?; - - record_spawned_supervisor(paths, record, child.id())?; - - thread::sleep(Duration::from_millis(200)); - if let Some(status) = child - .try_wait() - .context("failed to check recovery supervisor startup state")? - { - // Not a write-back of `record`: a stop issued since the spawn has - // persisted its marker, and a `failed` record without one is what - // recovery restarts. Same discipline as the supervisor's own writes. - crate::supervise::update_supervised_record(paths, record, |current| { - current.status = "failed".to_owned(); - })?; - anyhow::bail!( - "recovery supervisor exited immediately with status {status}; inspect {}", - record.log_path.display() - ); - } - - Ok(()) -} - -/// Record the supervisor [`restart_managed_service`] just spawned, as `pid`. -/// -/// Between the restart's first write — which clears the recorded PIDs, since -/// no supervisor exists yet — and this one, the record names no process at -/// all. A stop that starts in that window persists its marker first and then -/// finds nothing to signal; writing the restart's whole copy back here would -/// erase that marker while the new supervisor comes up, and leave the stop's -/// own final write comparing against a record that no longer shows the -/// process it has to answer for. So only the supervisor's identity is applied, -/// to the record as it is on disk now, through the same ownership guard as the -/// supervisor's own writes: against the copy from before this assignment, so -/// it is the cleared PID that has to still be there. If the new supervisor has -/// already recorded itself, or the record was removed, nothing is written. -fn record_spawned_supervisor( - paths: &AppPaths, - record: &mut ManagedServiceRecord, - pid: u32, -) -> Result<()> { - let unrecorded = record.clone(); - record.record_supervisor_identity(pid); - let (supervisor_pid, supervisor_start_ticks) = - (record.supervisor_pid, record.supervisor_start_ticks); - crate::supervise::update_supervised_record(paths, &unrecorded, |current| { - current.supervisor_pid = supervisor_pid; - current.supervisor_start_ticks = supervisor_start_ticks; - }) -} - -fn recovery_supervise_args(record: &ManagedServiceRecord) -> Vec { - let mut args = vec![ - "supervise".to_owned(), - record.service_id.clone(), - "--engine".to_owned(), - record.engine.clone(), - "--model-ref".to_owned(), - record.model_ref.clone(), - "--canonical-model-id".to_owned(), - record.canonical_model_id.clone(), - "--host".to_owned(), - record.host.clone(), - "--port".to_owned(), - record.port.to_string(), - "--device-policy".to_owned(), - record - .device_policy - .as_deref() - .unwrap_or("gpu_required") - .to_owned(), - ]; - args.extend(crate::common::optional_arg( - "--runtime-id", - record.runtime_id.as_deref(), - )); - args.extend(crate::common::optional_arg( - "--env-id", - record.env_id.as_deref(), - )); - if let Some(csv) = rocm_engine_protocol::gpu_indices_to_csv(&record.gpu_indices) { - args.extend(["--gpu".to_owned(), csv]); - } - args.extend(crate::common::optional_arg( - "--engine-recipe-json", - record.engine_recipe_json.as_deref(), - )); - args -} - fn queue_proposal( paths: &AppPaths, watcher_id: &str, @@ -1689,26 +1546,14 @@ fn proposal_arguments_for_action(action: &str, service_id: Option<&str>) -> Valu } } -#[cfg(unix)] -fn detached_rocmd_command(rocmd_binary: &std::path::Path) -> ProcessCommand { - let mut command = ProcessCommand::new("setsid"); - command.arg(rocmd_binary); - command -} - -#[cfg(not(unix))] -fn detached_rocmd_command(rocmd_binary: &std::path::Path) -> ProcessCommand { - ProcessCommand::new(rocmd_binary) -} - #[cfg(test)] mod tests { use super::*; - use crate::test_support::identity_probe_record; use crate::test_support::{ UNCONFIRMED_STOP_PID, seed_keyed_service, stop_with_outcome, temp_app_paths, }; use rocm_core::{AuditEventRecord, AutomationEventRecord}; + use std::fs; fn test_watcher_snapshot( id: &str, @@ -3464,38 +3309,6 @@ mod tests { Ok(()) } - #[test] - fn recovery_supervise_args_preserve_engine_recipe_json() { - let (_root, paths) = temp_app_paths("recovery-engine-recipe"); - let mut record = ManagedServiceRecord::new( - &paths, - "svc-1", - "vllm", - "qwen", - "Qwen/Qwen3.5-4B", - "127.0.0.1", - 11435, - "managed", - 123, - Some("therock-release:gfx120X-all".to_owned()), - Some("env-1".to_owned()), - Some("gpu_required".to_owned()), - ); - let engine_recipe_json = r#"{"contract_version":"0.1.0","engine":"vllm","required_flags":["--enable-auto-tool-choice"]}"#; - record.engine_recipe_json = Some(engine_recipe_json.to_owned()); - - let args = recovery_supervise_args(&record); - - assert!( - args.windows(2) - .any(|pair| pair[0] == "--engine-recipe-json" && pair[1] == engine_recipe_json) - ); - assert!( - args.windows(2) - .any(|pair| { pair[0] == "--canonical-model-id" && pair[1] == "Qwen/Qwen3.5-4B" }) - ); - } - #[test] fn watcher_policy_maps_modes_to_decisions() { assert_eq!( @@ -3516,41 +3329,6 @@ mod tests { ); } - /// The same guarantee for the daemon's recovery path, which records a PID of - /// its own. See `supervise_service_persists_the_supervisor_identity_token` - /// for why an unrecorded token is the defect rather than a cosmetic gap. - /// - /// The restart re-execs `rocmd supervise`, which here is the test binary: - /// it rejects those arguments and exits at once, so the restart reports - /// failure — after it has written the record this test reads. - #[cfg(target_os = "linux")] - #[test] - fn restart_managed_service_persists_the_supervisor_identity_token() -> Result<()> { - let (root, paths) = temp_app_paths("restart-records-identity"); - paths.ensure()?; - fs::create_dir_all(paths.services_dir())?; - - let service_id = "svc-restart-identity"; - let mut record = identity_probe_record(&paths, service_id, 11444); - record.status = "failed".to_owned(); - record.write()?; - assert_eq!( - load_service_record(&paths, service_id)?.supervisor_start_ticks, - None, - "precondition: the seeded record carries no token" - ); - - let _ = restart_managed_service(&paths, &mut record); - let persisted = load_service_record(&paths, service_id); - fs::remove_dir_all(root).ok(); - - assert!( - persisted?.supervisor_start_ticks.is_some(), - "restart_managed_service must persist the supervisor's start-time token beside its PID" - ); - Ok(()) - } - /// An unconfirmed stop leaves the status at `ready` on purpose, which is /// exactly what the daemon's recovery scan looks for. The stop request it /// records is the only thing telling it apart from a crash, so recovery @@ -3647,173 +3425,4 @@ mod tests { assert!(event_without_request); Ok(()) } - - /// A restart moves the supervisor PID; the engine PID it carried belonged - /// to the run being replaced. On a record `rocm serve --background` wrote, - /// that PID has no token, so carrying it past the point where the two PIDs - /// diverge leaves the next stop an entry it can only signal blind. - /// - /// Linux-only for the same reason as the test above: the restart re-execs - /// the test binary, which is only known to reject those arguments there. - #[cfg(target_os = "linux")] - #[test] - fn restart_managed_service_drops_the_replaced_runs_engine_pid() -> Result<()> { - let (root, paths) = temp_app_paths("restart-drops-engine-pid"); - paths.ensure()?; - fs::create_dir_all(paths.services_dir())?; - - let service_id = "svc-restart-drops-engine-pid"; - let mut record = identity_probe_record(&paths, service_id, 11455); - // The shape `rocm serve --background` writes: one PID in both roles. - record.supervisor_pid = UNCONFIRMED_STOP_PID; - record.engine_pid = Some(UNCONFIRMED_STOP_PID); - record.engine_start_ticks = None; - record.stop_requested_unix_ms = Some(1); - record.status = "failed".to_owned(); - record.write()?; - - let _ = restart_managed_service(&paths, &mut record); - let persisted = load_service_record(&paths, service_id); - fs::remove_dir_all(root).ok(); - - let persisted = persisted?; - assert_ne!(persisted.supervisor_pid, UNCONFIRMED_STOP_PID); - assert_eq!( - persisted.engine_pid, None, - "the replaced run's engine PID must not outlive the restart" - ); - assert_eq!(persisted.engine_start_ticks, None); - assert_eq!( - persisted.stop_requested_unix_ms, None, - "a restart supersedes an earlier stop request" - ); - Ok(()) - } - - /// A stop that lands while a restart waits to see whether its supervisor - /// survived startup must outlive the restart's `failed` write: a `failed` - /// record without the marker is exactly what recovery restarts. - /// - /// The restarted supervisor is the test binary, which rejects the - /// `supervise` arguments and exits at once, so the restart reaches that - /// write. A thread plays the stop: once the restart has recorded the - /// spawned supervisor, it persists a marker, as a stop does first. - #[cfg(target_os = "linux")] - #[test] - fn restart_failure_write_keeps_a_stop_marker_set_meanwhile() -> Result<()> { - let (root, paths) = temp_app_paths("restart-failure-keeps-marker"); - paths.ensure()?; - fs::create_dir_all(paths.services_dir())?; - - let service_id = "svc-restart-failure-marker"; - let mut record = identity_probe_record(&paths, service_id, 11456); - record.status = "failed".to_owned(); - record.write()?; - - let stop = { - let paths = paths.clone(); - thread::spawn(move || -> Result { - let deadline = std::time::Instant::now() + Duration::from_secs(10); - while std::time::Instant::now() < deadline { - // `write` is not atomic, so a read can land mid-write and - // see a truncated file; just look again. - let Ok(mut on_disk) = load_service_record(&paths, service_id) else { - thread::sleep(Duration::from_millis(5)); - continue; - }; - if on_disk.supervisor_pid != 0 && on_disk.supervisor_pid != std::process::id() { - on_disk.stop_requested_unix_ms = Some(1); - on_disk.write()?; - return Ok(true); - } - thread::sleep(Duration::from_millis(5)); - } - Ok(false) - }) - }; - let outcome = restart_managed_service(&paths, &mut record); - let stop_landed = stop.join().expect("stop thread"); - let persisted = load_service_record(&paths, service_id); - fs::remove_dir_all(root).ok(); - - assert!( - stop_landed?, - "precondition: the stop ran during the restart" - ); - assert!(outcome.is_err(), "the test binary is not a supervisor"); - let persisted = persisted?; - assert_eq!(persisted.status, "failed"); - assert_eq!( - persisted.stop_requested_unix_ms, - Some(1), - "the restart's failure write must not withdraw a stop issued meanwhile" - ); - Ok(()) - } - - /// Between a restart's first write, which clears the recorded PIDs, and the - /// one recording the supervisor it spawned, a stop can persist its marker. - /// Writing the restart's own copy back would erase it while the new - /// supervisor comes up — so only the identity is applied. - #[test] - fn restart_records_its_supervisor_without_erasing_a_stop_marker() -> Result<()> { - let (root, paths) = temp_app_paths("restart-spawn-keeps-marker"); - paths.ensure()?; - let service_id = "svc-restart-spawn-marker"; - // The restart's copy after its first write: no supervisor, no marker. - let mut restarting = identity_probe_record(&paths, service_id, 11457); - restarting.status = "recovering".to_owned(); - restarting.supervisor_pid = 0; - restarting.supervisor_start_ticks = None; - // A stop started since: same (empty) processes, marker set. - let mut on_disk = restarting.clone(); - on_disk.stop_requested_unix_ms = Some(1); - on_disk.write()?; - - record_spawned_supervisor(&paths, &mut restarting, std::process::id())?; - let persisted = load_service_record(&paths, service_id); - fs::remove_dir_all(root).ok(); - - let persisted = persisted?; - assert_eq!(restarting.supervisor_pid, std::process::id()); - assert_eq!(persisted.supervisor_pid, std::process::id()); - assert_eq!( - persisted.supervisor_start_ticks, - restarting.supervisor_start_ticks - ); - assert_eq!( - persisted.stop_requested_unix_ms, - Some(1), - "recording the spawned supervisor must not withdraw a standing stop" - ); - Ok(()) - } - - /// The spawned supervisor may record itself before the restart does, and - /// the record may be removed outright; either way the restart's write is - /// not the one to land. - #[test] - fn restart_leaves_a_record_naming_another_supervisor_or_none() -> Result<()> { - let (root, paths) = temp_app_paths("restart-spawn-not-owner"); - paths.ensure()?; - let service_id = "svc-restart-spawn-not-owner"; - let mut restarting = identity_probe_record(&paths, service_id, 11458); - restarting.supervisor_pid = 0; - restarting.supervisor_start_ticks = None; - let mut on_disk = restarting.clone(); - on_disk.supervisor_pid = 4_242_424; - on_disk.write()?; - - record_spawned_supervisor(&paths, &mut restarting.clone(), std::process::id())?; - let after_supervisor = load_service_record(&paths, service_id); - - fs::remove_file(paths.service_manifest_path(service_id))?; - record_spawned_supervisor(&paths, &mut restarting, std::process::id())?; - let resurrected = paths.service_manifest_path(service_id).exists(); - fs::remove_dir_all(root).ok(); - - assert_eq!(after_supervisor?.supervisor_pid, 4_242_424); - assert!(!resurrected, "a removed record must not be written back"); - Ok(()) - } } diff --git a/docs/architecture.md b/docs/architecture.md index 802cb6de13..6342300925 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -90,7 +90,7 @@ Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): - [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) — bubblewrap/native sandbox execution, atomic-write helpers, artifact prefetch policy gating, and the sandbox-tool result shaping for `check_updates`/`driver_plan`/`stop_server`. - [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs) — the sandbox-tool vocabulary: the `SandboxToolArg` value enum naming each sandbox tool, and `SandboxToolPolicy`, the artifact-download policy built from the `--allow-*` flags (including the Hugging Face token read from the environment when authenticated downloads are allowed). It uses no other module of the crate, so modules that need the vocabulary do not import `cli.rs`. - [`service.rs`](../apps/rocmd/src/service.rs) — managed-service PID lifecycle/stop, and `run_daemon`'s foreground loop with `print_status`. -- [`supervise.rs`](../apps/rocmd/src/supervise.rs) — the `rocmd supervise` side of a managed service: `supervise_service` (launch the engine child, record its identity, wait for readiness, record the exit), `update_supervised_record` (the re-read-then-write update that every write after the first goes through, which leaves a stop marker or another supervisor's record alone), and the serve-log startup-phase polling and readiness wait it uses. Its production code uses only [`common.rs`](../apps/rocmd/src/common.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs). `cli.rs` calls `supervise_service`, and `watchers.rs` calls `update_supervised_record`. +- [`supervise.rs`](../apps/rocmd/src/supervise.rs) — launching and re-launching a managed service's supervisor: `supervise_service` (the `rocmd supervise` entry point: launch the engine child, record its identity, wait for readiness, record the exit) with the serve-log startup-phase polling and readiness wait it uses, and `restart_managed_service` (the recovery respawn: re-launch `rocmd supervise` detached for a service and record the new supervisor). Writes after a supervisor's first go through the private `update_supervised_record`, which re-reads the record and leaves a stop marker or another supervisor's record alone. Its production code uses only [`common.rs`](../apps/rocmd/src/common.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs). `cli.rs` calls `supervise_service`; `watchers.rs` (the server-recover handler) and `sandbox.rs` (the `restart_server` tool) call `restart_managed_service`. - [`watchers.rs`](../apps/rocmd/src/watchers.rs) — event collection/dispatch for all built-in watchers (TheRock update, GPU metrics/thermal-pressure, cache-warm, driver-upgrade, server-recover), plus managed-service recovery classification and automation-proposal queuing. - [`webhook.rs`](../apps/rocmd/src/webhook.rs) — the local webhook source: its axum routes, request validation, and watcher-kind allow-list. From 11ed44fe2618d2d65ae832c9a6dc8c9b132603c4 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 2 Nov 2026 11:06:14 +0000 Subject: [PATCH 06/13] refactor(rocmd): move the daemon loop and status into daemon.rs Move run_daemon, print_status, build_runtime_state and both cfg variants of shutdown_signal out of service.rs, and WATCHER_TICK_INTERVAL out of lib.rs, into a new daemon.rs, together with the one test that exercises run_daemon on its own (local_webhook_requires_enabled_automation_loop). cli.rs calls the two entry points at their new path. service.rs now holds only the managed-service stop code, and with this commit the production modules of the crate no longer form an import cycle. Part of the rocmd module-cycle follow-up work. Besides the moved lines and import wiring, this commit changes: WATCHER_TICK_INTERVAL becomes a private constant of daemon.rs, its only user; the two path pins in apps/rocm/src/advised_commands.rs for the "rocmd automation supervisor started/stopped" log lines now name daemon.rs, where those lines live; docs/architecture.md gets an entry for the new file, rewords the service.rs entry to the stop code that remains, and drops the constant from the lib.rs entry. Signed-off-by: Jussi Elo --- apps/rocm/src/advised_commands.rs | 4 +- apps/rocmd/src/cli.rs | 4 +- apps/rocmd/src/daemon.rs | 305 ++++++++++++++++++++++++++++++ apps/rocmd/src/lib.rs | 5 +- apps/rocmd/src/service.rs | 294 +--------------------------- docs/architecture.md | 5 +- 6 files changed, 315 insertions(+), 302 deletions(-) create mode 100644 apps/rocmd/src/daemon.rs diff --git a/apps/rocm/src/advised_commands.rs b/apps/rocm/src/advised_commands.rs index e7c84a3420..ddf04c18e5 100644 --- a/apps/rocm/src/advised_commands.rs +++ b/apps/rocm/src/advised_commands.rs @@ -1201,12 +1201,12 @@ const NOT_INVOCATIONS: &[(&str, &str, &str)] = &[ "error message", ), ( - "apps/rocmd/src/service.rs", + "apps/rocmd/src/daemon.rs", "rocmd automation supervisor started", "log line", ), ( - "apps/rocmd/src/service.rs", + "apps/rocmd/src/daemon.rs", "rocmd automation supervisor stopped", "log line", ), diff --git a/apps/rocmd/src/cli.rs b/apps/rocmd/src/cli.rs index c332c9e044..8e9e0a2139 100644 --- a/apps/rocmd/src/cli.rs +++ b/apps/rocmd/src/cli.rs @@ -169,7 +169,7 @@ async fn run_cli(cli: Cli) -> Result<()> { Command::Run { automations_enabled, local_webhook_port, - } => crate::service::run_daemon(&paths, automations_enabled, local_webhook_port).await?, + } => crate::daemon::run_daemon(&paths, automations_enabled, local_webhook_port).await?, Command::Supervise { service_id, engine, @@ -197,7 +197,7 @@ async fn run_cli(cli: Cli) -> Result<()> { engine_recipe_json, )?, Command::Status => { - crate::service::print_status(&paths)?; + crate::daemon::print_status(&paths)?; } Command::BridgeSnapshot { pretty } => { print_bridge_snapshot(&paths, pretty)?; diff --git a/apps/rocmd/src/daemon.rs b/apps/rocmd/src/daemon.rs new file mode 100644 index 0000000000..9369b8f487 --- /dev/null +++ b/apps/rocmd/src/daemon.rs @@ -0,0 +1,305 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +use anyhow::{Context, Result, bail}; +use rocm_core::{ + AppPaths, AutomationRuntimeState, RocmCliConfig, WatcherRuntimeSnapshot, builtin_watchers, + unix_time_millis, +}; +use std::io::{self, Write}; +use std::time::Duration; +use tokio::time::{self, MissedTickBehavior}; + +const WATCHER_TICK_INTERVAL: Duration = Duration::from_secs(5); + +pub(crate) async fn run_daemon( + paths: &AppPaths, + automations_enabled: bool, + local_webhook_port: Option, +) -> Result<()> { + if local_webhook_port.is_some() && !automations_enabled { + bail!("local webhook source requires --automations-enabled"); + } + + let config = RocmCliConfig::load(paths)?; + let mut state = build_runtime_state(&config, automations_enabled); + let local_webhook = if let Some(port) = local_webhook_port { + Some(crate::webhook::start_local_webhook_source(port).await?) + } else { + None + }; + let (local_webhook_endpoint, mut local_webhook_receiver, local_webhook_task) = + match local_webhook { + Some(source) => ( + Some(source.endpoint), + Some(source.receiver), + Some(source.task), + ), + None => (None, None, None), + }; + state.local_webhook_endpoint = local_webhook_endpoint.clone(); + + println!("rocmd run"); + println!(" automations enabled: {automations_enabled}"); + println!( + " lifecycle: {}", + if automations_enabled { + "persistent" + } else { + "on-demand" + } + ); + println!(" config: {}", paths.config_path().display()); + println!(" state: {}", paths.automation_state_path().display()); + println!( + " local_webhook_endpoint: {}", + local_webhook_endpoint.as_deref().unwrap_or("disabled") + ); + let enabled_count = state + .active_watchers + .iter() + .filter(|watcher| watcher.enabled) + .count(); + println!(" enabled watchers: {enabled_count}"); + // This banner is the foreground-loop readiness contract used by callers and + // integration tests. Flush it before any persistent work so piped stdout on + // Windows cannot retain the line in a userspace buffer indefinitely. + io::stdout() + .flush() + .context("failed to flush rocmd run banner")?; + + if !automations_enabled { + println!( + " note: rerun with --automations-enabled to keep rocmd alive for watcher execution" + ); + return Ok(()); + } + + paths.ensure()?; + state.write(paths)?; + crate::persistence::record_event( + paths, + &mut state, + "rocmd", + "info", + "daemon_start", + "rocmd automation supervisor started", + None, + )?; + state.write(paths)?; + + crate::watchers::evaluate_watchers(paths, &config, &mut state)?; + state.last_tick_unix_ms = unix_time_millis(); + state.write(paths)?; + + let shutdown = shutdown_signal(); + tokio::pin!(shutdown); + + let mut ticker = time::interval(WATCHER_TICK_INTERVAL); + ticker.set_missed_tick_behavior(MissedTickBehavior::Delay); + + loop { + tokio::select! { + _ = ticker.tick() => { + let config = RocmCliConfig::load(paths)?; + crate::watchers::reconcile_watcher_snapshots(&config, &mut state); + crate::watchers::evaluate_watchers(paths, &config, &mut state)?; + state.last_tick_unix_ms = unix_time_millis(); + state.write(paths)?; + } + event = crate::webhook::receive_local_webhook_event(&mut local_webhook_receiver) => { + if let Some(event) = event { + let config = RocmCliConfig::load(paths)?; + crate::watchers::reconcile_watcher_snapshots(&config, &mut state); + crate::persistence::record_event( + paths, + &mut state, + "rocmd", + "info", + "local_webhook_event", + &format!( + "received local webhook event kind={} watcher_hint={}; dispatching through existing watcher policy; webhook payload grants no new action", + event.kind, + event.watcher_hint.as_deref().unwrap_or("") + ), + event.service_id.clone(), + )?; + if let Err(error) = + crate::watchers::evaluate_watchers_for_events(paths, &config, &mut state, &[event]) + { + crate::persistence::record_event( + paths, + &mut state, + "rocmd", + "error", + "local_webhook_dispatch_failed", + &format!( + "local webhook event could not be dispatched through watcher policy: {error}" + ), + None, + )?; + } + state.last_tick_unix_ms = unix_time_millis(); + state.write(paths)?; + } else { + local_webhook_receiver = None; + state.local_webhook_endpoint = None; + crate::persistence::record_event( + paths, + &mut state, + "rocmd", + "warn", + "local_webhook_stopped", + "local webhook source stopped; automation daemon continues without webhook ingestion", + None, + )?; + state.write(paths)?; + } + } + () = &mut shutdown => { + break; + } + } + } + + state.running = false; + state.last_tick_unix_ms = unix_time_millis(); + state.local_webhook_endpoint = None; + crate::persistence::record_event( + paths, + &mut state, + "rocmd", + "info", + "daemon_stop", + "rocmd automation supervisor stopped", + None, + )?; + state.write(paths)?; + if let Some(task) = local_webhook_task { + task.abort(); + } + Ok(()) +} + +pub(crate) fn print_status(paths: &AppPaths) -> Result<()> { + let config = RocmCliConfig::load(paths).unwrap_or_default(); + println!("rocmd status"); + println!(" config dir: {}", paths.config_dir.display()); + println!(" data dir: {}", paths.data_dir.display()); + println!(" policy: on-demand by default, persistent only with background features"); + println!( + " automations desired: {}", + if config.automation_daemon_enabled() { + "enabled" + } else { + "disabled" + } + ); + match AutomationRuntimeState::load(paths)? { + Some(state) => { + println!( + " automations runtime: {} pid={} last_tick_unix_ms={}", + if state.running { "running" } else { "stopped" }, + state.daemon_pid, + state.last_tick_unix_ms + ); + println!( + " local_webhook_endpoint: {}", + state + .local_webhook_endpoint + .as_deref() + .unwrap_or("disabled") + ); + for watcher in state + .active_watchers + .into_iter() + .filter(|watcher| watcher.enabled) + { + println!( + " watcher {} mode={} last_event={}", + watcher.id, + watcher.mode.as_str(), + watcher.last_event.as_deref().unwrap_or("") + ); + } + } + None => println!(" automations runtime: inactive"), + } + println!( + " automation events: {}", + paths.automation_events_path().display() + ); + println!(" audit events: {}", paths.audit_events_path().display()); + + let records = crate::persistence::load_managed_services(paths)?; + if records.is_empty() { + println!(" services: none"); + return Ok(()); + } + + for record in records { + println!( + " service {} engine={} status={} endpoint={}", + record.service_id, record.engine, record.status, record.endpoint_url + ); + } + + Ok(()) +} + +fn build_runtime_state( + config: &RocmCliConfig, + automations_enabled: bool, +) -> AutomationRuntimeState { + let now = unix_time_millis(); + let active_watchers = builtin_watchers() + .iter() + .map(|watcher| WatcherRuntimeSnapshot { + id: watcher.id.to_owned(), + enabled: config.watcher_enabled(watcher), + mode: config.effective_watcher_mode(watcher), + summary: watcher.summary.to_owned(), + last_event: None, + last_event_unix_ms: None, + }) + .collect(); + AutomationRuntimeState { + running: automations_enabled, + automations_enabled, + daemon_pid: std::process::id(), + started_at_unix_ms: now, + last_tick_unix_ms: now, + local_webhook_endpoint: None, + active_watchers, + } +} + +#[cfg(unix)] +async fn shutdown_signal() { + let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .expect("failed to register SIGTERM handler"); + tokio::select! { + _ = tokio::signal::ctrl_c() => {} + _ = term.recv() => {} + } +} + +#[cfg(not(unix))] +async fn shutdown_signal() { + let _ = tokio::signal::ctrl_c().await; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::temp_app_paths; + + #[tokio::test] + async fn local_webhook_requires_enabled_automation_loop() { + let (_root, paths) = temp_app_paths("local-webhook-requires-loop"); + let error = run_daemon(&paths, false, Some(0)).await.unwrap_err(); + + assert!(error.to_string().contains("requires --automations-enabled")); + } +} diff --git a/apps/rocmd/src/lib.rs b/apps/rocmd/src/lib.rs index 7db6b7297a..bff91c32b0 100644 --- a/apps/rocmd/src/lib.rs +++ b/apps/rocmd/src/lib.rs @@ -6,6 +6,7 @@ mod cli; mod common; +mod daemon; mod mcp; mod persistence; mod sandbox; @@ -18,7 +19,3 @@ mod watchers; mod webhook; pub use cli::{parse_args, run_bin_cli, run_from_args}; - -use std::time::Duration; - -const WATCHER_TICK_INTERVAL: Duration = Duration::from_secs(5); diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index 6459320431..316192263a 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -2,19 +2,14 @@ // // SPDX-License-Identifier: MIT -use anyhow::{Context, Result, bail}; -use rocm_core::{ - AppPaths, AutomationRuntimeState, ManagedServiceRecord, RocmCliConfig, WatcherRuntimeSnapshot, - builtin_watchers, unix_time_millis, -}; +use anyhow::{Context, Result}; +use rocm_core::{AppPaths, ManagedServiceRecord}; use rocm_engine_protocol::{EngineMethod, StopRequest, StopResponse}; use serde_json::Value; use serde_json::json; -use std::io::{self, Write}; #[cfg(windows)] use std::process::{Command as ProcessCommand, Stdio}; use std::time::Duration; -use tokio::time::{self, MissedTickBehavior}; pub(crate) fn stop_managed_service(paths: &AppPaths, service_id: &str) -> Result { stop_managed_service_with(paths, service_id, terminate_recorded_pid) @@ -385,283 +380,6 @@ fn terminate_recorded_pid(identity: &rocm_core::ProcessIdentity) -> rocm_core::T ) } -pub(crate) async fn run_daemon( - paths: &AppPaths, - automations_enabled: bool, - local_webhook_port: Option, -) -> Result<()> { - if local_webhook_port.is_some() && !automations_enabled { - bail!("local webhook source requires --automations-enabled"); - } - - let config = RocmCliConfig::load(paths)?; - let mut state = build_runtime_state(&config, automations_enabled); - let local_webhook = if let Some(port) = local_webhook_port { - Some(crate::webhook::start_local_webhook_source(port).await?) - } else { - None - }; - let (local_webhook_endpoint, mut local_webhook_receiver, local_webhook_task) = - match local_webhook { - Some(source) => ( - Some(source.endpoint), - Some(source.receiver), - Some(source.task), - ), - None => (None, None, None), - }; - state.local_webhook_endpoint = local_webhook_endpoint.clone(); - - println!("rocmd run"); - println!(" automations enabled: {automations_enabled}"); - println!( - " lifecycle: {}", - if automations_enabled { - "persistent" - } else { - "on-demand" - } - ); - println!(" config: {}", paths.config_path().display()); - println!(" state: {}", paths.automation_state_path().display()); - println!( - " local_webhook_endpoint: {}", - local_webhook_endpoint.as_deref().unwrap_or("disabled") - ); - let enabled_count = state - .active_watchers - .iter() - .filter(|watcher| watcher.enabled) - .count(); - println!(" enabled watchers: {enabled_count}"); - // This banner is the foreground-loop readiness contract used by callers and - // integration tests. Flush it before any persistent work so piped stdout on - // Windows cannot retain the line in a userspace buffer indefinitely. - io::stdout() - .flush() - .context("failed to flush rocmd run banner")?; - - if !automations_enabled { - println!( - " note: rerun with --automations-enabled to keep rocmd alive for watcher execution" - ); - return Ok(()); - } - - paths.ensure()?; - state.write(paths)?; - crate::persistence::record_event( - paths, - &mut state, - "rocmd", - "info", - "daemon_start", - "rocmd automation supervisor started", - None, - )?; - state.write(paths)?; - - crate::watchers::evaluate_watchers(paths, &config, &mut state)?; - state.last_tick_unix_ms = unix_time_millis(); - state.write(paths)?; - - let shutdown = shutdown_signal(); - tokio::pin!(shutdown); - - let mut ticker = time::interval(crate::WATCHER_TICK_INTERVAL); - ticker.set_missed_tick_behavior(MissedTickBehavior::Delay); - - loop { - tokio::select! { - _ = ticker.tick() => { - let config = RocmCliConfig::load(paths)?; - crate::watchers::reconcile_watcher_snapshots(&config, &mut state); - crate::watchers::evaluate_watchers(paths, &config, &mut state)?; - state.last_tick_unix_ms = unix_time_millis(); - state.write(paths)?; - } - event = crate::webhook::receive_local_webhook_event(&mut local_webhook_receiver) => { - if let Some(event) = event { - let config = RocmCliConfig::load(paths)?; - crate::watchers::reconcile_watcher_snapshots(&config, &mut state); - crate::persistence::record_event( - paths, - &mut state, - "rocmd", - "info", - "local_webhook_event", - &format!( - "received local webhook event kind={} watcher_hint={}; dispatching through existing watcher policy; webhook payload grants no new action", - event.kind, - event.watcher_hint.as_deref().unwrap_or("") - ), - event.service_id.clone(), - )?; - if let Err(error) = - crate::watchers::evaluate_watchers_for_events(paths, &config, &mut state, &[event]) - { - crate::persistence::record_event( - paths, - &mut state, - "rocmd", - "error", - "local_webhook_dispatch_failed", - &format!( - "local webhook event could not be dispatched through watcher policy: {error}" - ), - None, - )?; - } - state.last_tick_unix_ms = unix_time_millis(); - state.write(paths)?; - } else { - local_webhook_receiver = None; - state.local_webhook_endpoint = None; - crate::persistence::record_event( - paths, - &mut state, - "rocmd", - "warn", - "local_webhook_stopped", - "local webhook source stopped; automation daemon continues without webhook ingestion", - None, - )?; - state.write(paths)?; - } - } - () = &mut shutdown => { - break; - } - } - } - - state.running = false; - state.last_tick_unix_ms = unix_time_millis(); - state.local_webhook_endpoint = None; - crate::persistence::record_event( - paths, - &mut state, - "rocmd", - "info", - "daemon_stop", - "rocmd automation supervisor stopped", - None, - )?; - state.write(paths)?; - if let Some(task) = local_webhook_task { - task.abort(); - } - Ok(()) -} - -pub(crate) fn print_status(paths: &AppPaths) -> Result<()> { - let config = RocmCliConfig::load(paths).unwrap_or_default(); - println!("rocmd status"); - println!(" config dir: {}", paths.config_dir.display()); - println!(" data dir: {}", paths.data_dir.display()); - println!(" policy: on-demand by default, persistent only with background features"); - println!( - " automations desired: {}", - if config.automation_daemon_enabled() { - "enabled" - } else { - "disabled" - } - ); - match AutomationRuntimeState::load(paths)? { - Some(state) => { - println!( - " automations runtime: {} pid={} last_tick_unix_ms={}", - if state.running { "running" } else { "stopped" }, - state.daemon_pid, - state.last_tick_unix_ms - ); - println!( - " local_webhook_endpoint: {}", - state - .local_webhook_endpoint - .as_deref() - .unwrap_or("disabled") - ); - for watcher in state - .active_watchers - .into_iter() - .filter(|watcher| watcher.enabled) - { - println!( - " watcher {} mode={} last_event={}", - watcher.id, - watcher.mode.as_str(), - watcher.last_event.as_deref().unwrap_or("") - ); - } - } - None => println!(" automations runtime: inactive"), - } - println!( - " automation events: {}", - paths.automation_events_path().display() - ); - println!(" audit events: {}", paths.audit_events_path().display()); - - let records = crate::persistence::load_managed_services(paths)?; - if records.is_empty() { - println!(" services: none"); - return Ok(()); - } - - for record in records { - println!( - " service {} engine={} status={} endpoint={}", - record.service_id, record.engine, record.status, record.endpoint_url - ); - } - - Ok(()) -} - -fn build_runtime_state( - config: &RocmCliConfig, - automations_enabled: bool, -) -> AutomationRuntimeState { - let now = unix_time_millis(); - let active_watchers = builtin_watchers() - .iter() - .map(|watcher| WatcherRuntimeSnapshot { - id: watcher.id.to_owned(), - enabled: config.watcher_enabled(watcher), - mode: config.effective_watcher_mode(watcher), - summary: watcher.summary.to_owned(), - last_event: None, - last_event_unix_ms: None, - }) - .collect(); - AutomationRuntimeState { - running: automations_enabled, - automations_enabled, - daemon_pid: std::process::id(), - started_at_unix_ms: now, - last_tick_unix_ms: now, - local_webhook_endpoint: None, - active_watchers, - } -} - -#[cfg(unix)] -async fn shutdown_signal() { - let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) - .expect("failed to register SIGTERM handler"); - tokio::select! { - _ = tokio::signal::ctrl_c() => {} - _ = term.recv() => {} - } -} - -#[cfg(not(unix))] -async fn shutdown_signal() { - let _ = tokio::signal::ctrl_c().await; -} - #[cfg(test)] mod tests { use super::*; @@ -672,14 +390,6 @@ mod tests { #[cfg(unix)] use std::process::{Command as ProcessCommand, Stdio}; - #[tokio::test] - async fn local_webhook_requires_enabled_automation_loop() { - let (_root, paths) = temp_app_paths("local-webhook-requires-loop"); - let error = run_daemon(&paths, false, Some(0)).await.unwrap_err(); - - assert!(error.to_string().contains("requires --automations-enabled")); - } - #[test] fn stop_managed_service_removes_endpoint_key_file() -> Result<()> { let (root, paths) = temp_app_paths("stop-removes-endpoint-key"); diff --git a/docs/architecture.md b/docs/architecture.md index 6342300925..9da4b8fa96 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -85,16 +85,17 @@ Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): - [`cli.rs`](../apps/rocmd/src/cli.rs) — the `Cli`/`Command` clap definitions, the bridge-snapshot diagnostic printer, and the top-level dispatch in `run_cli`/`run_bin_cli`/`run_from_args`. - [`common.rs`](../apps/rocmd/src/common.rs) — helpers the other modules call into: GPU/amd-smi snapshotting, the bridge-snapshot diagnostic, the `rocm`-subprocess capture wrappers the sandbox and MCP clusters both call (they only choose the child's `ROCM_CLI_*_DIR` environment; `CommandCapture` and the spawn itself live in [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), shared with `rocm`'s chat tool bridge, and the bounded-execution half in [`process.rs`](../crates/rocm-core/src/process.rs)), and small arg/healthcheck/endpoint-key utilities. Not every helper here has more than one caller (`wait_for_port`, for example, is called only from [`watchers.rs`](../apps/rocmd/src/watchers.rs)). +- [`daemon.rs`](../apps/rocmd/src/daemon.rs) — the foreground `rocmd run` loop (`run_daemon`: evaluate the watchers each tick, dispatch local webhook events, write the runtime state, stop on a signal) and `rocmd status` (`print_status`), with the watcher tick interval. It uses [`watchers.rs`](../apps/rocmd/src/watchers.rs), [`webhook.rs`](../apps/rocmd/src/webhook.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs); `cli.rs` calls it. - [`mcp.rs`](../apps/rocmd/src/mcp.rs) — the MCP stdio server, tool schema table, tool dispatch, and the argv-building and result-shaping helpers behind the MCP tools (the `rocm` subprocess itself is spawned through [`common.rs`](../apps/rocmd/src/common.rs)). - [`persistence.rs`](../apps/rocmd/src/persistence.rs) — `record_event`/`load_managed_services`/`load_service_record`, the automation-event/audit-log and managed-service-registry I/O shared across the daemon's sandbox, MCP, service-lifecycle, and watcher code. - [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) — bubblewrap/native sandbox execution, atomic-write helpers, artifact prefetch policy gating, and the sandbox-tool result shaping for `check_updates`/`driver_plan`/`stop_server`. - [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs) — the sandbox-tool vocabulary: the `SandboxToolArg` value enum naming each sandbox tool, and `SandboxToolPolicy`, the artifact-download policy built from the `--allow-*` flags (including the Hugging Face token read from the environment when authenticated downloads are allowed). It uses no other module of the crate, so modules that need the vocabulary do not import `cli.rs`. -- [`service.rs`](../apps/rocmd/src/service.rs) — managed-service PID lifecycle/stop, and `run_daemon`'s foreground loop with `print_status`. +- [`service.rs`](../apps/rocmd/src/service.rs) — managed-service stop: terminating the recorded processes of a service after verifying their identity, and the stop report (`stop_managed_service`, `stop_report_confirmed`) that the sandbox and MCP tools read. - [`supervise.rs`](../apps/rocmd/src/supervise.rs) — launching and re-launching a managed service's supervisor: `supervise_service` (the `rocmd supervise` entry point: launch the engine child, record its identity, wait for readiness, record the exit) with the serve-log startup-phase polling and readiness wait it uses, and `restart_managed_service` (the recovery respawn: re-launch `rocmd supervise` detached for a service and record the new supervisor). Writes after a supervisor's first go through the private `update_supervised_record`, which re-reads the record and leaves a stop marker or another supervisor's record alone. Its production code uses only [`common.rs`](../apps/rocmd/src/common.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs). `cli.rs` calls `supervise_service`; `watchers.rs` (the server-recover handler) and `sandbox.rs` (the `restart_server` tool) call `restart_managed_service`. - [`watchers.rs`](../apps/rocmd/src/watchers.rs) — event collection/dispatch for all built-in watchers (TheRock update, GPU metrics/thermal-pressure, cache-warm, driver-upgrade, server-recover), plus managed-service recovery classification and automation-proposal queuing. - [`webhook.rs`](../apps/rocmd/src/webhook.rs) — the local webhook source: its axum routes, request validation, and watcher-kind allow-list. -[`lib.rs`](../apps/rocmd/src/lib.rs) itself is now top-level glue: module declarations, the three externally-consumed entry points re-exported from [`cli.rs`](../apps/rocmd/src/cli.rs) (`run_bin_cli`/`run_from_args`, plus `parse_args`, which `apps/rocm`'s test-only [`advised_commands.rs`](../apps/rocm/src/advised_commands.rs) uses to check advised `rocmd` commands), and the `WATCHER_TICK_INTERVAL` constant. +[`lib.rs`](../apps/rocmd/src/lib.rs) itself is now top-level glue: module declarations and the three externally-consumed entry points re-exported from [`cli.rs`](../apps/rocmd/src/cli.rs) (`run_bin_cli`/`run_from_args`, plus `parse_args`, which `apps/rocm`'s test-only [`advised_commands.rs`](../apps/rocm/src/advised_commands.rs) uses to check advised `rocmd` commands). ### `crates/rocm-core` — core library From 18e35ce06557341a85bb05221c48222a848d4aab Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 2 Nov 2026 11:07:39 +0000 Subject: [PATCH 07/13] refactor(rocmd): move payload_string into common.rs Move payload_string, the trimmed non-empty string read from a JSON payload, out of watchers.rs into common.rs. Both watchers.rs and webhook.rs call it, so it belongs in the module the other modules share; webhook.rs now reaches it there and no longer imports watchers.rs. Part of the rocmd module-cycle follow-up work. Besides the moved lines and import wiring, the only other change is the docs/architecture.md common.rs entry, which now names the helper. No tests move: none target this function directly. Signed-off-by: Jussi Elo --- apps/rocmd/src/common.rs | 9 +++++++++ apps/rocmd/src/watchers.rs | 10 +--------- apps/rocmd/src/webhook.rs | 4 ++-- docs/architecture.md | 2 +- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/apps/rocmd/src/common.rs b/apps/rocmd/src/common.rs index 5705d27c5f..ce0cb11e50 100644 --- a/apps/rocmd/src/common.rs +++ b/apps/rocmd/src/common.rs @@ -426,6 +426,15 @@ pub(crate) fn wait_for_port(host: &str, port: u16, timeout: Duration) -> bool { false } +pub(crate) fn payload_string(payload: &Value, key: &str) -> Option { + payload + .get(key) + .and_then(Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_owned) +} + #[cfg(test)] mod tests { use super::*; diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index 1478957089..5b68844b99 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -2,6 +2,7 @@ // // SPDX-License-Identifier: MIT +use crate::common::payload_string; use crate::persistence::load_service_record; use crate::supervise::restart_managed_service; use anyhow::{Context, Result, bail}; @@ -1122,15 +1123,6 @@ fn restricted_driver_plan_result(value: &Value) -> Result Option { - payload - .get(key) - .and_then(Value::as_str) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_owned) -} - #[cfg(test)] fn evaluate_server_recover( paths: &AppPaths, diff --git a/apps/rocmd/src/webhook.rs b/apps/rocmd/src/webhook.rs index 3c455b0430..e26c6f7f7e 100644 --- a/apps/rocmd/src/webhook.rs +++ b/apps/rocmd/src/webhook.rs @@ -162,12 +162,12 @@ pub(crate) fn local_webhook_event_from_request( bail!("server-recover webhook events require service_id"); } "cache-warm" - if crate::watchers::payload_string(&request.payload, "artifact_ref").is_none() => + if crate::common::payload_string(&request.payload, "artifact_ref").is_none() => { bail!("cache-warm webhook events require payload.artifact_ref"); } "driver-upgrade" - if crate::watchers::payload_string(&request.payload, "component").as_deref() + if crate::common::payload_string(&request.payload, "component").as_deref() != Some("driver") => { bail!("driver-upgrade webhook events require payload.component=driver"); diff --git a/docs/architecture.md b/docs/architecture.md index 9da4b8fa96..27899b52ac 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -84,7 +84,7 @@ Further `apps/rocm` modules, not part of the `main.rs` split: Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): - [`cli.rs`](../apps/rocmd/src/cli.rs) — the `Cli`/`Command` clap definitions, the bridge-snapshot diagnostic printer, and the top-level dispatch in `run_cli`/`run_bin_cli`/`run_from_args`. -- [`common.rs`](../apps/rocmd/src/common.rs) — helpers the other modules call into: GPU/amd-smi snapshotting, the bridge-snapshot diagnostic, the `rocm`-subprocess capture wrappers the sandbox and MCP clusters both call (they only choose the child's `ROCM_CLI_*_DIR` environment; `CommandCapture` and the spawn itself live in [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), shared with `rocm`'s chat tool bridge, and the bounded-execution half in [`process.rs`](../crates/rocm-core/src/process.rs)), and small arg/healthcheck/endpoint-key utilities. Not every helper here has more than one caller (`wait_for_port`, for example, is called only from [`watchers.rs`](../apps/rocmd/src/watchers.rs)). +- [`common.rs`](../apps/rocmd/src/common.rs) — helpers the other modules call into: GPU/amd-smi snapshotting, the bridge-snapshot diagnostic, the `rocm`-subprocess capture wrappers the sandbox and MCP clusters both call (they only choose the child's `ROCM_CLI_*_DIR` environment; `CommandCapture` and the spawn itself live in [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), shared with `rocm`'s chat tool bridge, and the bounded-execution half in [`process.rs`](../crates/rocm-core/src/process.rs)), and small arg/JSON-payload/healthcheck/endpoint-key utilities (`payload_string`, for example, is read by both [`watchers.rs`](../apps/rocmd/src/watchers.rs) and [`webhook.rs`](../apps/rocmd/src/webhook.rs)). Not every helper here has more than one caller (`wait_for_port`, for example, is called only from [`watchers.rs`](../apps/rocmd/src/watchers.rs)). - [`daemon.rs`](../apps/rocmd/src/daemon.rs) — the foreground `rocmd run` loop (`run_daemon`: evaluate the watchers each tick, dispatch local webhook events, write the runtime state, stop on a signal) and `rocmd status` (`print_status`), with the watcher tick interval. It uses [`watchers.rs`](../apps/rocmd/src/watchers.rs), [`webhook.rs`](../apps/rocmd/src/webhook.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs); `cli.rs` calls it. - [`mcp.rs`](../apps/rocmd/src/mcp.rs) — the MCP stdio server, tool schema table, tool dispatch, and the argv-building and result-shaping helpers behind the MCP tools (the `rocm` subprocess itself is spawned through [`common.rs`](../apps/rocmd/src/common.rs)). - [`persistence.rs`](../apps/rocmd/src/persistence.rs) — `record_event`/`load_managed_services`/`load_service_record`, the automation-event/audit-log and managed-service-registry I/O shared across the daemon's sandbox, MCP, service-lifecycle, and watcher code. From f4af40941b7f877c7095b6f74284a76563637151 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 2 Nov 2026 11:11:30 +0000 Subject: [PATCH 08/13] refactor(rocmd): return single-caller helpers to their callers Move run_rocm_capture out of common.rs into mcp.rs, its only caller, and wait_for_port into watchers.rs beside endpoint_service_recovery_reason, its only caller; gather_gpu_snapshot, which only common.rs itself calls, becomes private. The shared run_rocm_capture_for_paths stays in common.rs because both mcp.rs and sandbox.rs use it. Part of the rocmd module-cycle follow-up work. Besides the moved lines and import wiring, this commit changes: wait_for_port becomes private in its new home and gather_gpu_snapshot is narrowed from pub(crate) to private; the imports only the moved code needed move with it (std::net, std::thread and format_host_port into watchers.rs, std::time::Duration into mcp.rs); docs/architecture.md drops the common.rs sentence about wait_for_port having a single caller, now false, and names the capture wrapper that remains there. No tests move: none target these functions directly. Signed-off-by: Jussi Elo --- apps/rocmd/src/common.rs | 29 +++-------------------------- apps/rocmd/src/mcp.rs | 26 ++++++++++++++++---------- apps/rocmd/src/watchers.rs | 22 ++++++++++++++++++++-- docs/architecture.md | 4 ++-- 4 files changed, 41 insertions(+), 40 deletions(-) diff --git a/apps/rocmd/src/common.rs b/apps/rocmd/src/common.rs index ce0cb11e50..b111480ae4 100644 --- a/apps/rocmd/src/common.rs +++ b/apps/rocmd/src/common.rs @@ -6,8 +6,8 @@ use anyhow::{Context, Result, bail}; use rocm_core::{ AppPaths, AutomationRuntimeState, CodexBridgeEngine, CodexBridgeGpuSnapshot, CodexBridgeSnapshot, CommandCapture, ExamineSummary, RocmCliConfig, - default_engine_for_platform, format_host_port, load_recent_automation_events, - resolve_amd_smi_binary, unix_time_millis, + default_engine_for_platform, load_recent_automation_events, resolve_amd_smi_binary, + unix_time_millis, }; #[cfg(test)] use rocm_engine_protocol::EnginePluginDescriptor; @@ -19,11 +19,9 @@ use serde::Serialize; use serde::de::DeserializeOwned; use serde_json::Value; use std::io::Write; -use std::net::{SocketAddr, TcpStream}; #[cfg(test)] use std::path::PathBuf; use std::process::{Command as ProcessCommand, Stdio}; -use std::thread; use std::time::Duration; const AMD_SMI_PROBE_TIMEOUT: Duration = Duration::from_secs(2); @@ -56,7 +54,7 @@ pub(crate) fn update_check_message(status: &str) -> &'static str { } } -pub(crate) fn gather_gpu_snapshot() -> CodexBridgeGpuSnapshot { +fn gather_gpu_snapshot() -> CodexBridgeGpuSnapshot { let static_snapshot = match capture_amd_smi_json(&["static", "-a", "-g", "all", "--json"]) { Ok(value) => Some(value), Err(error) => { @@ -212,11 +210,6 @@ where .map(|plugin| plugin.executable_path)) } -pub(crate) fn run_rocm_capture(args: &[&str]) -> Result { - let paths = AppPaths::discover()?; - run_rocm_capture_for_paths(&paths, args, Duration::from_mins(2)) -} - /// Run `rocm` on `paths`' config/data/cache roots, via the shared /// [`rocm_core::run_rocm_capture_with_env`]. pub(crate) fn run_rocm_capture_for_paths( @@ -410,22 +403,6 @@ where serde_json::from_value(data).context("failed to deserialize engine response data") } -pub(crate) fn wait_for_port(host: &str, port: u16, timeout: Duration) -> bool { - let address: SocketAddr = match format_host_port(host, port).parse() { - Ok(value) => value, - Err(_) => return false, - }; - - let start = std::time::Instant::now(); - while start.elapsed() < timeout { - if TcpStream::connect_timeout(&address, Duration::from_millis(200)).is_ok() { - return true; - } - thread::sleep(Duration::from_millis(200)); - } - false -} - pub(crate) fn payload_string(payload: &Value, key: &str) -> Option { payload .get(key) diff --git a/apps/rocmd/src/mcp.rs b/apps/rocmd/src/mcp.rs index 4c11340b62..1fae7574a8 100644 --- a/apps/rocmd/src/mcp.rs +++ b/apps/rocmd/src/mcp.rs @@ -17,6 +17,7 @@ use serde_json::json; use std::collections::VecDeque; use std::fs; use std::io::{self, BufRead, Write}; +use std::time::Duration; pub(crate) fn run_mcp_server(paths: &AppPaths) -> Result<()> { let stdin = io::stdin(); @@ -567,7 +568,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result { let examine = ExamineSummary::gather()?; - let output = common::run_rocm_capture(&["examine"])?; + let output = run_rocm_capture(&["examine"])?; let text = command_capture_text(&output); if output.exit_status == 0 { Ok(tool_success(text, json!(examine))) @@ -670,7 +671,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result Result>(); - let output = common::run_rocm_capture(&refs)?; + let output = run_rocm_capture(&refs)?; Ok(tool_result_from_command( "Ran read-only `rocm` command.", output, @@ -689,7 +690,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result { - let output = common::run_rocm_capture(&["update"])?; + let output = run_rocm_capture(&["update"])?; Ok(tool_result_from_command( "Ran `rocm update`.", output, @@ -699,7 +700,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result { let argv = build_install_sdk_args(&arguments, true)?; let refs = argv.iter().map(String::as_str).collect::>(); - let output = common::run_rocm_capture(&refs)?; + let output = run_rocm_capture(&refs)?; Ok(tool_result_from_command( "Ran `rocm install sdk --dry-run`.", output, @@ -709,7 +710,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result { let argv = build_install_sdk_args(&arguments, false)?; let refs = argv.iter().map(String::as_str).collect::>(); - let output = common::run_rocm_capture(&refs)?; + let output = run_rocm_capture(&refs)?; Ok(tool_result_from_command( "Ran `rocm install sdk`.", output, @@ -719,7 +720,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result { let argv = build_install_engine_args(&arguments)?; let refs = argv.iter().map(String::as_str).collect::>(); - let output = common::run_rocm_capture(&refs)?; + let output = run_rocm_capture(&refs)?; Ok(tool_result_from_command( "Ran `rocm engines install`.", output, @@ -729,7 +730,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result { let argv = build_launch_server_args(&arguments)?; let refs = argv.iter().map(String::as_str).collect::>(); - let output = common::run_rocm_capture(&refs)?; + let output = run_rocm_capture(&refs)?; Ok(tool_result_from_command( "Ran `rocm serve --managed`.", output, @@ -747,7 +748,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result { let argv = build_watcher_enable_args(&arguments)?; let refs = argv.iter().map(String::as_str).collect::>(); - let output = common::run_rocm_capture(&refs)?; + let output = run_rocm_capture(&refs)?; Ok(tool_result_from_command( "Ran `rocm automations enable`.", output, @@ -765,7 +766,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result>(); - let output = common::run_rocm_capture(&refs)?; + let output = run_rocm_capture(&refs)?; Ok(tool_result_from_command( "Ran `rocm automations disable`.", output, @@ -779,6 +780,11 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result Result { + let paths = AppPaths::discover()?; + common::run_rocm_capture_for_paths(&paths, args, Duration::from_mins(2)) +} + fn tool_success(text: String, structured: Value) -> Value { json!({ "content": [ diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index 5b68844b99..1263c8a4e2 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -9,11 +9,13 @@ use anyhow::{Context, Result, bail}; use rocm_core::{ AppPaths, AutomationProposalRecord, AutomationRuntimeState, AutomationTriggerEvent, CodexBridgeGpuSnapshot, ManagedServiceRecord, RocmCliConfig, WatcherMode, - WatcherRuntimeSnapshot, append_automation_proposal, builtin_watchers, + WatcherRuntimeSnapshot, append_automation_proposal, builtin_watchers, format_host_port, resolve_model_recipe_artifact, unix_time_millis, }; use serde_json::Value; use serde_json::json; +use std::net::{SocketAddr, TcpStream}; +use std::thread; use std::time::Duration; const SERVER_RECOVER_BACKOFF_MS: u128 = 30_000; @@ -1446,10 +1448,26 @@ pub(crate) const fn stop_requested(record: &ManagedServiceRecord) -> bool { } fn endpoint_service_recovery_reason(record: &ManagedServiceRecord) -> Option { - (!crate::common::wait_for_port(&record.host, record.port, ENDPOINT_HEALTH_TIMEOUT)) + (!wait_for_port(&record.host, record.port, ENDPOINT_HEALTH_TIMEOUT)) .then(|| "endpoint_status_unreachable".to_owned()) } +fn wait_for_port(host: &str, port: u16, timeout: Duration) -> bool { + let address: SocketAddr = match format_host_port(host, port).parse() { + Ok(value) => value, + Err(_) => return false, + }; + + let start = std::time::Instant::now(); + while start.elapsed() < timeout { + if TcpStream::connect_timeout(&address, Duration::from_millis(200)).is_ok() { + return true; + } + thread::sleep(Duration::from_millis(200)); + } + false +} + fn manifest_service_recovery_reason( record: &ManagedServiceRecord, now_unix_ms: u128, diff --git a/docs/architecture.md b/docs/architecture.md index 27899b52ac..525d2ee206 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -84,9 +84,9 @@ Further `apps/rocm` modules, not part of the `main.rs` split: Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): - [`cli.rs`](../apps/rocmd/src/cli.rs) — the `Cli`/`Command` clap definitions, the bridge-snapshot diagnostic printer, and the top-level dispatch in `run_cli`/`run_bin_cli`/`run_from_args`. -- [`common.rs`](../apps/rocmd/src/common.rs) — helpers the other modules call into: GPU/amd-smi snapshotting, the bridge-snapshot diagnostic, the `rocm`-subprocess capture wrappers the sandbox and MCP clusters both call (they only choose the child's `ROCM_CLI_*_DIR` environment; `CommandCapture` and the spawn itself live in [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), shared with `rocm`'s chat tool bridge, and the bounded-execution half in [`process.rs`](../crates/rocm-core/src/process.rs)), and small arg/JSON-payload/healthcheck/endpoint-key utilities (`payload_string`, for example, is read by both [`watchers.rs`](../apps/rocmd/src/watchers.rs) and [`webhook.rs`](../apps/rocmd/src/webhook.rs)). Not every helper here has more than one caller (`wait_for_port`, for example, is called only from [`watchers.rs`](../apps/rocmd/src/watchers.rs)). +- [`common.rs`](../apps/rocmd/src/common.rs) — helpers the other modules call into: GPU/amd-smi snapshotting, the bridge-snapshot diagnostic, the `rocm`-subprocess capture wrapper (`run_rocm_capture_for_paths`) the sandbox and MCP clusters both call (it only chooses the child's `ROCM_CLI_*_DIR` environment; `CommandCapture` and the spawn itself live in [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), shared with `rocm`'s chat tool bridge, and the bounded-execution half in [`process.rs`](../crates/rocm-core/src/process.rs)), and small arg/JSON-payload/healthcheck/endpoint-key utilities (`payload_string`, for example, is read by both [`watchers.rs`](../apps/rocmd/src/watchers.rs) and [`webhook.rs`](../apps/rocmd/src/webhook.rs)). - [`daemon.rs`](../apps/rocmd/src/daemon.rs) — the foreground `rocmd run` loop (`run_daemon`: evaluate the watchers each tick, dispatch local webhook events, write the runtime state, stop on a signal) and `rocmd status` (`print_status`), with the watcher tick interval. It uses [`watchers.rs`](../apps/rocmd/src/watchers.rs), [`webhook.rs`](../apps/rocmd/src/webhook.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs); `cli.rs` calls it. -- [`mcp.rs`](../apps/rocmd/src/mcp.rs) — the MCP stdio server, tool schema table, tool dispatch, and the argv-building and result-shaping helpers behind the MCP tools (the `rocm` subprocess itself is spawned through [`common.rs`](../apps/rocmd/src/common.rs)). +- [`mcp.rs`](../apps/rocmd/src/mcp.rs) — the MCP stdio server, tool schema table, tool dispatch, and the argv-building and result-shaping helpers behind the MCP tools (the `rocm` subprocess itself is spawned through [`common.rs`](../apps/rocmd/src/common.rs), via `run_rocm_capture` here). - [`persistence.rs`](../apps/rocmd/src/persistence.rs) — `record_event`/`load_managed_services`/`load_service_record`, the automation-event/audit-log and managed-service-registry I/O shared across the daemon's sandbox, MCP, service-lifecycle, and watcher code. - [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) — bubblewrap/native sandbox execution, atomic-write helpers, artifact prefetch policy gating, and the sandbox-tool result shaping for `check_updates`/`driver_plan`/`stop_server`. - [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs) — the sandbox-tool vocabulary: the `SandboxToolArg` value enum naming each sandbox tool, and `SandboxToolPolicy`, the artifact-download policy built from the `--allow-*` flags (including the Hugging Face token read from the environment when authenticated downloads are allowed). It uses no other module of the crate, so modules that need the vocabulary do not import `cli.rs`. From 8399a110c9e29b6a3559b6df93ebdc8966b5e67a Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 2 Nov 2026 11:13:39 +0000 Subject: [PATCH 09/13] refactor(rocmd): move engine IPC and endpoint-key guards into engine_ipc.rs Move engine_request, engine_healthcheck_response, healthcheck_response_ready, healthcheck_response_recoverable, apply_endpoint_key_env and ensure_public_service_has_endpoint_key out of common.rs into a new engine_ipc.rs that uses no other module of the crate, together with the six tests that exercise only those symbols (recovery_refuses_a_service_that_lost_a_key_it_was_launched_with, recovery_respawn_fails_closed_for_a_public_service_without_a_key, apply_endpoint_key_env_sets_var_only_when_key_file_present, healthcheck_readiness_requires_ready_loaded_model, healthcheck_recoverability_tracks_failed_endpoint_state, healthcheck_readiness_withheld_while_the_model_only_lists). service.rs, supervise.rs and watchers.rs call them at the new path. Part of the rocmd module-cycle follow-up work. Besides the moved lines and import wiring, the only other change is docs/architecture.md: an entry for the new file, and the common.rs and supervise.rs entries adjusted to match. Signed-off-by: Jussi Elo --- apps/rocmd/src/common.rs | 334 +--------------------------------- apps/rocmd/src/engine_ipc.rs | 344 +++++++++++++++++++++++++++++++++++ apps/rocmd/src/lib.rs | 1 + apps/rocmd/src/service.rs | 2 +- apps/rocmd/src/supervise.rs | 10 +- apps/rocmd/src/watchers.rs | 18 +- docs/architecture.md | 5 +- 7 files changed, 365 insertions(+), 349 deletions(-) create mode 100644 apps/rocmd/src/engine_ipc.rs diff --git a/apps/rocmd/src/common.rs b/apps/rocmd/src/common.rs index b111480ae4..82b57aff41 100644 --- a/apps/rocmd/src/common.rs +++ b/apps/rocmd/src/common.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -use anyhow::{Context, Result, bail}; +use anyhow::{Context, Result}; use rocm_core::{ AppPaths, AutomationRuntimeState, CodexBridgeEngine, CodexBridgeGpuSnapshot, CodexBridgeSnapshot, CommandCapture, ExamineSummary, RocmCliConfig, @@ -11,17 +11,10 @@ use rocm_core::{ }; #[cfg(test)] use rocm_engine_protocol::EnginePluginDescriptor; -use rocm_engine_protocol::{ - EngineMethod, EngineRequestEnvelope, EngineResponseEnvelope, HealthcheckRequest, - HealthcheckResponse, -}; -use serde::Serialize; -use serde::de::DeserializeOwned; use serde_json::Value; -use std::io::Write; #[cfg(test)] use std::path::PathBuf; -use std::process::{Command as ProcessCommand, Stdio}; +use std::process::Command as ProcessCommand; use std::time::Duration; const AMD_SMI_PROBE_TIMEOUT: Duration = Duration::from_secs(2); @@ -235,174 +228,6 @@ pub(crate) fn optional_arg(flag: &str, value: Option<&str>) -> Vec { } } -pub(crate) fn engine_healthcheck_response( - paths: &AppPaths, - engine: &str, - service_id: &str, -) -> Result { - engine_request::<_, HealthcheckResponse>( - paths, - engine, - service_id, - EngineMethod::Healthcheck, - &HealthcheckRequest { - service_id: service_id.to_owned(), - }, - ) -} - -pub(crate) fn healthcheck_response_ready(response: &HealthcheckResponse) -> bool { - response.status == "ready" && response.model_loaded -} - -pub(crate) fn healthcheck_response_recoverable(response: &HealthcheckResponse) -> bool { - matches!( - response.status.as_str(), - "failed" | "unreachable" | "exited" - ) -} - -/// Re-thread the endpoint key file (public bind only) onto an engine child's -/// environment, mirroring the initial `rocm serve` spawn. A loopback service -/// with no stored key leaves the command's environment untouched, matching the -/// unauthenticated default. -/// -/// Returns whether a key was applied. Callers that spawn a *listener* must -/// check it against the service's host — a public bind with no key would come -/// up anonymous (see [`ensure_public_service_has_endpoint_key`]). Callers that -/// only make a stdio plugin call can ignore it. -/// -/// The file must hold a *usable* key, not merely exist: the engine adapters -/// resolve it with `endpoint_api_key_from_file` and enforce nothing when that -/// yields `None`, so treating an empty or malformed file as "protected" would -/// let the guard pass while the endpoint served anonymously. -#[must_use] -pub(crate) fn apply_endpoint_key_env( - command: &mut ProcessCommand, - paths: &AppPaths, - service_id: &str, -) -> bool { - if let Some(key_file) = rocm_engine_protocol::endpoint_key_file_if_present(paths, service_id) - .and_then(|path| rocm_engine_protocol::endpoint_api_key_file_if_valid(&path)) - { - command.env(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file); - return true; - } - false -} - -/// Refuse to respawn a recorded service on a public host without its endpoint -/// API key, so daemon recovery cannot reopen a protected endpoint anonymously. -/// -/// Mirrors the guard of the same name in `rocm`; the shared -/// [`rocm_engine_protocol::is_public_bind_host`] keeps the two classifications -/// identical for a given `ManagedServiceRecord::host`. -pub(crate) fn ensure_public_service_has_endpoint_key( - host: &str, - key_present: bool, - requires_api_key: bool, -) -> Result<()> { - // The bind address is not the whole story. A service bound to loopback is - // only private until something republishes the port, and a tailnet publish - // outlives both the process and this daemon. The requirement is recorded on - // the service precisely so recovery can honour it without re-deriving it - // from an address that no longer answers the question. - if requires_api_key && !key_present { - bail!( - "refusing to recover a service that was launched with an endpoint API key but no \ - longer has one: it would come back up without authentication, and something \ - outside this machine may still be publishing its port. Relaunch it with \ - `rocm serve --require-api-key` to issue a new key." - ); - } - if rocm_engine_protocol::is_public_bind_host(host) && !key_present { - bail!( - "refusing to respawn a service bound to the public host `{host}` without an endpoint \ - API key: it would come back up without authentication. Relaunch it with \ - `rocm serve --host {host} --allow-public-bind` to issue a new key." - ); - } - Ok(()) -} - -pub(crate) fn engine_request( - paths: &AppPaths, - engine: &str, - service_id: &str, - method: EngineMethod, - request: &T, -) -> Result -where - T: Serialize, - R: DeserializeOwned, -{ - let envelope = EngineRequestEnvelope { - method, - payload: serde_json::to_value(request).context("failed to serialize engine request")?, - }; - let engine_binary = - std::env::current_exe().context("failed to resolve current rocm executable path")?; - let mut command = ProcessCommand::new(&engine_binary); - command - .arg("__engine-stdio") - .arg(engine) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - // A protected public endpoint enforces its key on every request, including - // this stdio-transported healthcheck — without the carrier the engine - // adapter's HTTP probe is unauthenticated and the endpoint looks - // unreachable, which would misclassify a healthy protected service as - // recoverable. - // - // No host to check here: this is a stdio plugin call, not a listener, so a - // missing key only weakens this probe rather than opening a port. - let _ = apply_endpoint_key_env(&mut command, paths, service_id); - let mut child = command.spawn().with_context(|| { - format!( - "failed to spawn engine stdio process {}", - engine_binary.display() - ) - })?; - - { - let stdin = child - .stdin - .as_mut() - .context("engine stdio child did not expose stdin")?; - serde_json::to_writer(&mut *stdin, &envelope).context("failed to write engine request")?; - stdin.write_all(b"\n")?; - } - - let output = child - .wait_with_output() - .context("failed waiting for engine stdio response")?; - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); - if stderr.is_empty() { - bail!("engine stdio process exited with status {}", output.status); - } - bail!( - "engine stdio process exited with status {}: {}", - output.status, - stderr - ); - } - let envelope: EngineResponseEnvelope = - serde_json::from_slice(&output.stdout).context("failed to parse engine response")?; - if !envelope.ok { - let detail = envelope.error.map_or_else( - || "unknown engine error".to_owned(), - |error| format!("{}: {}", error.code, error.message), - ); - bail!(detail); - } - let data = envelope - .data - .context("engine response did not include data")?; - serde_json::from_value(data).context("failed to deserialize engine response data") -} - pub(crate) fn payload_string(payload: &Value, key: &str) -> Option { payload .get(key) @@ -419,161 +244,6 @@ mod tests { use rocm_core::engine_plugin_dirs; use std::fs; - #[test] - fn recovery_refuses_a_service_that_lost_a_key_it_was_launched_with() { - // The daemon keeps its own copy of this guard, and it only knew about - // public binds. A loopback service that something republishes — a - // tailnet publish outlives this daemon, let alone the process — would - // be recovered without authentication, and the rebuilt record would - // then disarm `rocm services restart` too. - for host in ["127.0.0.1", "localhost", "::1"] { - let error = ensure_public_service_has_endpoint_key(host, false, true) - .expect_err("a service launched with a key must not be recovered without one"); - assert!( - format!("{error:#}").contains("without authentication"), - "{error:#}" - ); - } - // With the key still present, recovery proceeds. - ensure_public_service_has_endpoint_key("127.0.0.1", true, true).unwrap(); - // And a service that never had one is untouched. - ensure_public_service_has_endpoint_key("127.0.0.1", false, false).unwrap(); - } - - #[test] - fn recovery_respawn_fails_closed_for_a_public_service_without_a_key() { - // Daemon recovery re-execs `rocmd supervise` for the recorded host. With - // the key gone the child would listen on that public host anonymously, - // so the spawn must be refused instead. - let error = ensure_public_service_has_endpoint_key("0.0.0.0", false, false).unwrap_err(); - assert!( - error.to_string().contains("without authentication"), - "{error:#}" - ); - - ensure_public_service_has_endpoint_key("0.0.0.0", true, false).unwrap(); - for host in ["127.0.0.1", "localhost", "::1"] { - ensure_public_service_has_endpoint_key(host, false, false) - .unwrap_or_else(|error| panic!("{host} must not require a key: {error:#}")); - } - } - - #[test] - fn apply_endpoint_key_env_sets_var_only_when_key_file_present() { - let (_root, paths) = temp_app_paths("apply-endpoint-key-env"); - let service_id = "svc-endpoint-key-env"; - - // Loopback service: no key file has ever been written, so the child's - // environment must be left untouched. - let mut command = ProcessCommand::new("true"); - assert!(!apply_endpoint_key_env(&mut command, &paths, service_id)); - assert!( - command - .get_envs() - .all(|(key, _)| key != rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV), - "loopback service must not receive the endpoint key env var" - ); - - // Public service: once a key file exists at the deterministic path, - // it must be threaded onto the child so the engine's HTTP probe - // authenticates. - let key_path = rocm_engine_protocol::endpoint_key_file_path(&paths, service_id); - fs::create_dir_all(paths.services_dir()).unwrap(); - fs::write(&key_path, "secret-key").unwrap(); - let mut command = ProcessCommand::new("true"); - assert!(apply_endpoint_key_env(&mut command, &paths, service_id)); - let env_value = command - .get_envs() - .find_map(|(key, value)| { - (key == rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV) - .then_some(value) - .flatten() - }) - .expect("endpoint key env var must be set once the key file exists"); - assert_eq!(env_value, key_path.as_os_str()); - - // An empty (or otherwise unusable) key file is not a key: the engine - // adapters would enforce nothing, so reporting `true` here would let the - // respawn guard pass and still open an anonymous public listener. - fs::write(&key_path, " \n").unwrap(); - let mut command = ProcessCommand::new("true"); - assert!(!apply_endpoint_key_env(&mut command, &paths, service_id)); - assert!( - command - .get_envs() - .all(|(key, _)| key != rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV), - "an unusable key file must not be threaded onto the child" - ); - } - - #[test] - fn healthcheck_readiness_requires_ready_loaded_model() { - let ready = HealthcheckResponse { - status: "ready".to_owned(), - model_loaded: true, - device: "cuda".to_owned(), - uptime_sec: 1, - queue_depth: 0, - last_error: None, - tokens_per_sec: None, - }; - assert!(healthcheck_response_ready(&ready)); - - let mut loading = ready.clone(); - loading.status = "loading_model".to_owned(); - assert!(!healthcheck_response_ready(&loading)); - - let mut unloaded = ready; - unloaded.model_loaded = false; - assert!(!healthcheck_response_ready(&unloaded)); - } - - #[test] - fn healthcheck_recoverability_tracks_failed_endpoint_state() { - let mut response = HealthcheckResponse { - status: "ready".to_owned(), - model_loaded: true, - device: "cuda".to_owned(), - uptime_sec: 1, - queue_depth: 0, - last_error: None, - tokens_per_sec: None, - }; - assert!(!healthcheck_response_recoverable(&response)); - - response.status = "unreachable".to_owned(); - assert!(healthcheck_response_recoverable(&response)); - - response.status = "failed".to_owned(); - assert!(healthcheck_response_recoverable(&response)); - - response.status = "loading_model".to_owned(); - assert!(!healthcheck_response_recoverable(&response)); - - // The status the engines report for a model that is listed but has not - // yet served an inference request. Restarting it would kill a model - // mid-load and start the wait over. - response.status = "loading".to_owned(); - assert!(!healthcheck_response_recoverable(&response)); - } - - #[test] - fn healthcheck_readiness_withheld_while_the_model_only_lists() { - // What an engine reports once `/v1/models` answers but inference has not: - // not ready, so `rocm serve` keeps waiting instead of handing the caller - // an endpoint that will hang on its first request. - let listing_only = HealthcheckResponse { - status: "loading".to_owned(), - model_loaded: false, - device: "unknown".to_owned(), - uptime_sec: 1, - queue_depth: 0, - last_error: None, - tokens_per_sec: None, - }; - assert!(!healthcheck_response_ready(&listing_only)); - } - #[test] fn gpu_snapshot_respects_disabled_telemetry_policy() { let mut config = RocmCliConfig::default(); diff --git a/apps/rocmd/src/engine_ipc.rs b/apps/rocmd/src/engine_ipc.rs new file mode 100644 index 0000000000..d61397ac7d --- /dev/null +++ b/apps/rocmd/src/engine_ipc.rs @@ -0,0 +1,344 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +use anyhow::{Context, Result, bail}; +use rocm_core::AppPaths; +use rocm_engine_protocol::{ + EngineMethod, EngineRequestEnvelope, EngineResponseEnvelope, HealthcheckRequest, + HealthcheckResponse, +}; +use serde::Serialize; +use serde::de::DeserializeOwned; +use std::io::Write; +use std::process::{Command as ProcessCommand, Stdio}; + +pub(crate) fn engine_healthcheck_response( + paths: &AppPaths, + engine: &str, + service_id: &str, +) -> Result { + engine_request::<_, HealthcheckResponse>( + paths, + engine, + service_id, + EngineMethod::Healthcheck, + &HealthcheckRequest { + service_id: service_id.to_owned(), + }, + ) +} + +pub(crate) fn healthcheck_response_ready(response: &HealthcheckResponse) -> bool { + response.status == "ready" && response.model_loaded +} + +pub(crate) fn healthcheck_response_recoverable(response: &HealthcheckResponse) -> bool { + matches!( + response.status.as_str(), + "failed" | "unreachable" | "exited" + ) +} + +/// Re-thread the endpoint key file (public bind only) onto an engine child's +/// environment, mirroring the initial `rocm serve` spawn. A loopback service +/// with no stored key leaves the command's environment untouched, matching the +/// unauthenticated default. +/// +/// Returns whether a key was applied. Callers that spawn a *listener* must +/// check it against the service's host — a public bind with no key would come +/// up anonymous (see [`ensure_public_service_has_endpoint_key`]). Callers that +/// only make a stdio plugin call can ignore it. +/// +/// The file must hold a *usable* key, not merely exist: the engine adapters +/// resolve it with `endpoint_api_key_from_file` and enforce nothing when that +/// yields `None`, so treating an empty or malformed file as "protected" would +/// let the guard pass while the endpoint served anonymously. +#[must_use] +pub(crate) fn apply_endpoint_key_env( + command: &mut ProcessCommand, + paths: &AppPaths, + service_id: &str, +) -> bool { + if let Some(key_file) = rocm_engine_protocol::endpoint_key_file_if_present(paths, service_id) + .and_then(|path| rocm_engine_protocol::endpoint_api_key_file_if_valid(&path)) + { + command.env(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file); + return true; + } + false +} + +/// Refuse to respawn a recorded service on a public host without its endpoint +/// API key, so daemon recovery cannot reopen a protected endpoint anonymously. +/// +/// Mirrors the guard of the same name in `rocm`; the shared +/// [`rocm_engine_protocol::is_public_bind_host`] keeps the two classifications +/// identical for a given `ManagedServiceRecord::host`. +pub(crate) fn ensure_public_service_has_endpoint_key( + host: &str, + key_present: bool, + requires_api_key: bool, +) -> Result<()> { + // The bind address is not the whole story. A service bound to loopback is + // only private until something republishes the port, and a tailnet publish + // outlives both the process and this daemon. The requirement is recorded on + // the service precisely so recovery can honour it without re-deriving it + // from an address that no longer answers the question. + if requires_api_key && !key_present { + bail!( + "refusing to recover a service that was launched with an endpoint API key but no \ + longer has one: it would come back up without authentication, and something \ + outside this machine may still be publishing its port. Relaunch it with \ + `rocm serve --require-api-key` to issue a new key." + ); + } + if rocm_engine_protocol::is_public_bind_host(host) && !key_present { + bail!( + "refusing to respawn a service bound to the public host `{host}` without an endpoint \ + API key: it would come back up without authentication. Relaunch it with \ + `rocm serve --host {host} --allow-public-bind` to issue a new key." + ); + } + Ok(()) +} + +pub(crate) fn engine_request( + paths: &AppPaths, + engine: &str, + service_id: &str, + method: EngineMethod, + request: &T, +) -> Result +where + T: Serialize, + R: DeserializeOwned, +{ + let envelope = EngineRequestEnvelope { + method, + payload: serde_json::to_value(request).context("failed to serialize engine request")?, + }; + let engine_binary = + std::env::current_exe().context("failed to resolve current rocm executable path")?; + let mut command = ProcessCommand::new(&engine_binary); + command + .arg("__engine-stdio") + .arg(engine) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + // A protected public endpoint enforces its key on every request, including + // this stdio-transported healthcheck — without the carrier the engine + // adapter's HTTP probe is unauthenticated and the endpoint looks + // unreachable, which would misclassify a healthy protected service as + // recoverable. + // + // No host to check here: this is a stdio plugin call, not a listener, so a + // missing key only weakens this probe rather than opening a port. + let _ = apply_endpoint_key_env(&mut command, paths, service_id); + let mut child = command.spawn().with_context(|| { + format!( + "failed to spawn engine stdio process {}", + engine_binary.display() + ) + })?; + + { + let stdin = child + .stdin + .as_mut() + .context("engine stdio child did not expose stdin")?; + serde_json::to_writer(&mut *stdin, &envelope).context("failed to write engine request")?; + stdin.write_all(b"\n")?; + } + + let output = child + .wait_with_output() + .context("failed waiting for engine stdio response")?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); + if stderr.is_empty() { + bail!("engine stdio process exited with status {}", output.status); + } + bail!( + "engine stdio process exited with status {}: {}", + output.status, + stderr + ); + } + let envelope: EngineResponseEnvelope = + serde_json::from_slice(&output.stdout).context("failed to parse engine response")?; + if !envelope.ok { + let detail = envelope.error.map_or_else( + || "unknown engine error".to_owned(), + |error| format!("{}: {}", error.code, error.message), + ); + bail!(detail); + } + let data = envelope + .data + .context("engine response did not include data")?; + serde_json::from_value(data).context("failed to deserialize engine response data") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::temp_app_paths; + use std::fs; + + #[test] + fn recovery_refuses_a_service_that_lost_a_key_it_was_launched_with() { + // The daemon keeps its own copy of this guard, and it only knew about + // public binds. A loopback service that something republishes — a + // tailnet publish outlives this daemon, let alone the process — would + // be recovered without authentication, and the rebuilt record would + // then disarm `rocm services restart` too. + for host in ["127.0.0.1", "localhost", "::1"] { + let error = ensure_public_service_has_endpoint_key(host, false, true) + .expect_err("a service launched with a key must not be recovered without one"); + assert!( + format!("{error:#}").contains("without authentication"), + "{error:#}" + ); + } + // With the key still present, recovery proceeds. + ensure_public_service_has_endpoint_key("127.0.0.1", true, true).unwrap(); + // And a service that never had one is untouched. + ensure_public_service_has_endpoint_key("127.0.0.1", false, false).unwrap(); + } + + #[test] + fn recovery_respawn_fails_closed_for_a_public_service_without_a_key() { + // Daemon recovery re-execs `rocmd supervise` for the recorded host. With + // the key gone the child would listen on that public host anonymously, + // so the spawn must be refused instead. + let error = ensure_public_service_has_endpoint_key("0.0.0.0", false, false).unwrap_err(); + assert!( + error.to_string().contains("without authentication"), + "{error:#}" + ); + + ensure_public_service_has_endpoint_key("0.0.0.0", true, false).unwrap(); + for host in ["127.0.0.1", "localhost", "::1"] { + ensure_public_service_has_endpoint_key(host, false, false) + .unwrap_or_else(|error| panic!("{host} must not require a key: {error:#}")); + } + } + + #[test] + fn apply_endpoint_key_env_sets_var_only_when_key_file_present() { + let (_root, paths) = temp_app_paths("apply-endpoint-key-env"); + let service_id = "svc-endpoint-key-env"; + + // Loopback service: no key file has ever been written, so the child's + // environment must be left untouched. + let mut command = ProcessCommand::new("true"); + assert!(!apply_endpoint_key_env(&mut command, &paths, service_id)); + assert!( + command + .get_envs() + .all(|(key, _)| key != rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV), + "loopback service must not receive the endpoint key env var" + ); + + // Public service: once a key file exists at the deterministic path, + // it must be threaded onto the child so the engine's HTTP probe + // authenticates. + let key_path = rocm_engine_protocol::endpoint_key_file_path(&paths, service_id); + fs::create_dir_all(paths.services_dir()).unwrap(); + fs::write(&key_path, "secret-key").unwrap(); + let mut command = ProcessCommand::new("true"); + assert!(apply_endpoint_key_env(&mut command, &paths, service_id)); + let env_value = command + .get_envs() + .find_map(|(key, value)| { + (key == rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV) + .then_some(value) + .flatten() + }) + .expect("endpoint key env var must be set once the key file exists"); + assert_eq!(env_value, key_path.as_os_str()); + + // An empty (or otherwise unusable) key file is not a key: the engine + // adapters would enforce nothing, so reporting `true` here would let the + // respawn guard pass and still open an anonymous public listener. + fs::write(&key_path, " \n").unwrap(); + let mut command = ProcessCommand::new("true"); + assert!(!apply_endpoint_key_env(&mut command, &paths, service_id)); + assert!( + command + .get_envs() + .all(|(key, _)| key != rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV), + "an unusable key file must not be threaded onto the child" + ); + } + + #[test] + fn healthcheck_readiness_requires_ready_loaded_model() { + let ready = HealthcheckResponse { + status: "ready".to_owned(), + model_loaded: true, + device: "cuda".to_owned(), + uptime_sec: 1, + queue_depth: 0, + last_error: None, + tokens_per_sec: None, + }; + assert!(healthcheck_response_ready(&ready)); + + let mut loading = ready.clone(); + loading.status = "loading_model".to_owned(); + assert!(!healthcheck_response_ready(&loading)); + + let mut unloaded = ready; + unloaded.model_loaded = false; + assert!(!healthcheck_response_ready(&unloaded)); + } + + #[test] + fn healthcheck_recoverability_tracks_failed_endpoint_state() { + let mut response = HealthcheckResponse { + status: "ready".to_owned(), + model_loaded: true, + device: "cuda".to_owned(), + uptime_sec: 1, + queue_depth: 0, + last_error: None, + tokens_per_sec: None, + }; + assert!(!healthcheck_response_recoverable(&response)); + + response.status = "unreachable".to_owned(); + assert!(healthcheck_response_recoverable(&response)); + + response.status = "failed".to_owned(); + assert!(healthcheck_response_recoverable(&response)); + + response.status = "loading_model".to_owned(); + assert!(!healthcheck_response_recoverable(&response)); + + // The status the engines report for a model that is listed but has not + // yet served an inference request. Restarting it would kill a model + // mid-load and start the wait over. + response.status = "loading".to_owned(); + assert!(!healthcheck_response_recoverable(&response)); + } + + #[test] + fn healthcheck_readiness_withheld_while_the_model_only_lists() { + // What an engine reports once `/v1/models` answers but inference has not: + // not ready, so `rocm serve` keeps waiting instead of handing the caller + // an endpoint that will hang on its first request. + let listing_only = HealthcheckResponse { + status: "loading".to_owned(), + model_loaded: false, + device: "unknown".to_owned(), + uptime_sec: 1, + queue_depth: 0, + last_error: None, + tokens_per_sec: None, + }; + assert!(!healthcheck_response_ready(&listing_only)); + } +} diff --git a/apps/rocmd/src/lib.rs b/apps/rocmd/src/lib.rs index bff91c32b0..a88933bc9b 100644 --- a/apps/rocmd/src/lib.rs +++ b/apps/rocmd/src/lib.rs @@ -7,6 +7,7 @@ mod cli; mod common; mod daemon; +mod engine_ipc; mod mcp; mod persistence; mod sandbox; diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index 316192263a..d65b971012 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -36,7 +36,7 @@ pub(crate) fn stop_managed_service_with( } fn request_engine_stop(paths: &AppPaths, record: &ManagedServiceRecord) -> Result { - crate::common::engine_request::<_, StopResponse>( + crate::engine_ipc::engine_request::<_, StopResponse>( paths, &record.engine, &record.service_id, diff --git a/apps/rocmd/src/supervise.rs b/apps/rocmd/src/supervise.rs index 7719de84c6..887744ea06 100644 --- a/apps/rocmd/src/supervise.rs +++ b/apps/rocmd/src/supervise.rs @@ -100,7 +100,7 @@ pub(crate) fn supervise_service( // attempt leaves the recorded restart_count and timestamps intact instead of // clobbering them with a record no live process will ever back. The spawn // site below re-checks against the key actually threaded onto the command. - crate::common::ensure_public_service_has_endpoint_key( + crate::engine_ipc::ensure_public_service_has_endpoint_key( &record.host, rocm_engine_protocol::endpoint_key_file_if_present(paths, &record.service_id) .and_then(|path| rocm_engine_protocol::endpoint_api_key_file_if_valid(&path)) @@ -144,8 +144,8 @@ pub(crate) fn supervise_service( // no auth, so fail closed instead — an unreachable service is recoverable, // an anonymous public one is not. let endpoint_key_applied = - crate::common::apply_endpoint_key_env(&mut command, paths, &record.service_id); - crate::common::ensure_public_service_has_endpoint_key( + crate::engine_ipc::apply_endpoint_key_env(&mut command, paths, &record.service_id); + crate::engine_ipc::ensure_public_service_has_endpoint_key( &record.host, endpoint_key_applied, record.requires_api_key, @@ -354,8 +354,8 @@ fn read_new_log_phase(log_path: &Path, pos: &mut u64) -> Option<&'static str> { } fn engine_healthcheck_ready(paths: &AppPaths, engine: &str, service_id: &str) -> Result { - Ok(crate::common::healthcheck_response_ready( - &crate::common::engine_healthcheck_response(paths, engine, service_id)?, + Ok(crate::engine_ipc::healthcheck_response_ready( + &crate::engine_ipc::engine_healthcheck_response(paths, engine, service_id)?, )) } diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index 1263c8a4e2..c4b9c4f6c2 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -1200,12 +1200,12 @@ fn service_record_matches_recovery_event( manifest_service_recovery_reason(record, unix_time_millis()).is_some() } "service.endpoint_recoverable" => endpoint_service_recovery_reason(record).is_some(), - "service.healthcheck_recoverable" => { - crate::common::engine_healthcheck_response(paths, &record.engine, &record.service_id) - .is_ok_and(|healthcheck| { - crate::common::healthcheck_response_recoverable(&healthcheck) - }) - } + "service.healthcheck_recoverable" => crate::engine_ipc::engine_healthcheck_response( + paths, + &record.engine, + &record.service_id, + ) + .is_ok_and(|healthcheck| crate::engine_ipc::healthcheck_response_recoverable(&healthcheck)), _ => false, } } @@ -1305,7 +1305,7 @@ fn handle_server_recover_event_with_record( // Report it and stop, rather than letting a permanent failure // propagate out of `evaluate_watchers` and take the whole daemon — // and every other watcher — down on each 30s recovery tick. - if let Err(error) = crate::common::ensure_public_service_has_endpoint_key( + if let Err(error) = crate::engine_ipc::ensure_public_service_has_endpoint_key( &record.host, rocm_engine_protocol::endpoint_key_file_if_present(paths, &record.service_id) .and_then(|path| rocm_engine_protocol::endpoint_api_key_file_if_valid(&path)) @@ -1409,7 +1409,7 @@ pub(crate) fn find_recoverable_service( return Ok(Some((record, reason))); } if matches!(record.status.as_str(), "ready" | "running") { - let Ok(healthcheck) = crate::common::engine_healthcheck_response( + let Ok(healthcheck) = crate::engine_ipc::engine_healthcheck_response( paths, &record.engine, &record.service_id, @@ -1419,7 +1419,7 @@ pub(crate) fn find_recoverable_service( } continue; }; - if crate::common::healthcheck_response_recoverable(&healthcheck) { + if crate::engine_ipc::healthcheck_response_recoverable(&healthcheck) { return Ok(Some(( record, format!("healthcheck_status_{}", healthcheck.status), diff --git a/docs/architecture.md b/docs/architecture.md index 525d2ee206..48062861e9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -84,14 +84,15 @@ Further `apps/rocm` modules, not part of the `main.rs` split: Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): - [`cli.rs`](../apps/rocmd/src/cli.rs) — the `Cli`/`Command` clap definitions, the bridge-snapshot diagnostic printer, and the top-level dispatch in `run_cli`/`run_bin_cli`/`run_from_args`. -- [`common.rs`](../apps/rocmd/src/common.rs) — helpers the other modules call into: GPU/amd-smi snapshotting, the bridge-snapshot diagnostic, the `rocm`-subprocess capture wrapper (`run_rocm_capture_for_paths`) the sandbox and MCP clusters both call (it only chooses the child's `ROCM_CLI_*_DIR` environment; `CommandCapture` and the spawn itself live in [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), shared with `rocm`'s chat tool bridge, and the bounded-execution half in [`process.rs`](../crates/rocm-core/src/process.rs)), and small arg/JSON-payload/healthcheck/endpoint-key utilities (`payload_string`, for example, is read by both [`watchers.rs`](../apps/rocmd/src/watchers.rs) and [`webhook.rs`](../apps/rocmd/src/webhook.rs)). +- [`common.rs`](../apps/rocmd/src/common.rs) — helpers the other modules call into: GPU/amd-smi snapshotting, the bridge-snapshot diagnostic, the `rocm`-subprocess capture wrapper (`run_rocm_capture_for_paths`) the sandbox and MCP clusters both call (it only chooses the child's `ROCM_CLI_*_DIR` environment; `CommandCapture` and the spawn itself live in [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), shared with `rocm`'s chat tool bridge, and the bounded-execution half in [`process.rs`](../crates/rocm-core/src/process.rs)), and small arg/JSON-payload utilities (`payload_string`, for example, is read by both [`watchers.rs`](../apps/rocmd/src/watchers.rs) and [`webhook.rs`](../apps/rocmd/src/webhook.rs)). - [`daemon.rs`](../apps/rocmd/src/daemon.rs) — the foreground `rocmd run` loop (`run_daemon`: evaluate the watchers each tick, dispatch local webhook events, write the runtime state, stop on a signal) and `rocmd status` (`print_status`), with the watcher tick interval. It uses [`watchers.rs`](../apps/rocmd/src/watchers.rs), [`webhook.rs`](../apps/rocmd/src/webhook.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs); `cli.rs` calls it. +- [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) — talking to an engine and guarding its endpoint key: `engine_request` (the stdio request to an engine process), the healthcheck call and its readiness and recoverability checks, and the endpoint-key helpers `apply_endpoint_key_env` and `ensure_public_service_has_endpoint_key`. It uses no other module of the crate; `service.rs`, `supervise.rs` and `watchers.rs` call it. - [`mcp.rs`](../apps/rocmd/src/mcp.rs) — the MCP stdio server, tool schema table, tool dispatch, and the argv-building and result-shaping helpers behind the MCP tools (the `rocm` subprocess itself is spawned through [`common.rs`](../apps/rocmd/src/common.rs), via `run_rocm_capture` here). - [`persistence.rs`](../apps/rocmd/src/persistence.rs) — `record_event`/`load_managed_services`/`load_service_record`, the automation-event/audit-log and managed-service-registry I/O shared across the daemon's sandbox, MCP, service-lifecycle, and watcher code. - [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) — bubblewrap/native sandbox execution, atomic-write helpers, artifact prefetch policy gating, and the sandbox-tool result shaping for `check_updates`/`driver_plan`/`stop_server`. - [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs) — the sandbox-tool vocabulary: the `SandboxToolArg` value enum naming each sandbox tool, and `SandboxToolPolicy`, the artifact-download policy built from the `--allow-*` flags (including the Hugging Face token read from the environment when authenticated downloads are allowed). It uses no other module of the crate, so modules that need the vocabulary do not import `cli.rs`. - [`service.rs`](../apps/rocmd/src/service.rs) — managed-service stop: terminating the recorded processes of a service after verifying their identity, and the stop report (`stop_managed_service`, `stop_report_confirmed`) that the sandbox and MCP tools read. -- [`supervise.rs`](../apps/rocmd/src/supervise.rs) — launching and re-launching a managed service's supervisor: `supervise_service` (the `rocmd supervise` entry point: launch the engine child, record its identity, wait for readiness, record the exit) with the serve-log startup-phase polling and readiness wait it uses, and `restart_managed_service` (the recovery respawn: re-launch `rocmd supervise` detached for a service and record the new supervisor). Writes after a supervisor's first go through the private `update_supervised_record`, which re-reads the record and leaves a stop marker or another supervisor's record alone. Its production code uses only [`common.rs`](../apps/rocmd/src/common.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs). `cli.rs` calls `supervise_service`; `watchers.rs` (the server-recover handler) and `sandbox.rs` (the `restart_server` tool) call `restart_managed_service`. +- [`supervise.rs`](../apps/rocmd/src/supervise.rs) — launching and re-launching a managed service's supervisor: `supervise_service` (the `rocmd supervise` entry point: launch the engine child, record its identity, wait for readiness, record the exit) with the serve-log startup-phase polling and readiness wait it uses, and `restart_managed_service` (the recovery respawn: re-launch `rocmd supervise` detached for a service and record the new supervisor). Writes after a supervisor's first go through the private `update_supervised_record`, which re-reads the record and leaves a stop marker or another supervisor's record alone. Its production code uses only [`common.rs`](../apps/rocmd/src/common.rs), [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs). `cli.rs` calls `supervise_service`; `watchers.rs` (the server-recover handler) and `sandbox.rs` (the `restart_server` tool) call `restart_managed_service`. - [`watchers.rs`](../apps/rocmd/src/watchers.rs) — event collection/dispatch for all built-in watchers (TheRock update, GPU metrics/thermal-pressure, cache-warm, driver-upgrade, server-recover), plus managed-service recovery classification and automation-proposal queuing. - [`webhook.rs`](../apps/rocmd/src/webhook.rs) — the local webhook source: its axum routes, request validation, and watcher-kind allow-list. From db7d996bb873930ac0ed65351ccc00bb507a8d98 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 2 Nov 2026 11:18:22 +0000 Subject: [PATCH 10/13] refactor(rocmd): move the host and bridge snapshot into host_snapshot.rs Move AMD_SMI_PROBE_TIMEOUT, build_bridge_snapshot, gather_gpu_snapshot, gather_gpu_snapshot_for_config, capture_amd_smi_json, bridge_engine_inventory and rocmd_engine_inventory out of common.rs into a new host_snapshot.rs, with gpu_snapshot_respects_disabled_telemetry_policy. The new module uses only persistence.rs; cli.rs, mcp.rs and watchers.rs call it at the new path. Part of the rocmd module-cycle follow-up work. Besides the moved lines and import wiring, this commit changes: two comments in rocm-core that named an rocmd path now name host_snapshot.rs. In process.rs the old comment said both of the test's callers were in apps/rocmd/src/common.rs, which had already stopped being true when the rocm capture moved into rocm-core, so it now names the amd-smi probe in host_snapshot.rs and the rocm capture that common.rs still makes through run_rocm_capture_with_env. In rocm_install.rs the comment pointed at apps/rocmd/src/lib.rs for the daemon's serialised snapshot; the snapshot containing the examine summary is built in host_snapshot.rs, so it points there (the old path never held it). docs/architecture.md gets an entry for the new file and the common.rs entry is reworded to what remains. Signed-off-by: Jussi Elo --- apps/rocmd/src/cli.rs | 2 +- apps/rocmd/src/common.rs | 187 +------------------------- apps/rocmd/src/host_snapshot.rs | 193 +++++++++++++++++++++++++++ apps/rocmd/src/lib.rs | 1 + apps/rocmd/src/mcp.rs | 7 +- apps/rocmd/src/watchers.rs | 2 +- crates/rocm-core/src/process.rs | 8 +- crates/rocm-core/src/rocm_install.rs | 2 +- docs/architecture.md | 3 +- 9 files changed, 212 insertions(+), 193 deletions(-) create mode 100644 apps/rocmd/src/host_snapshot.rs diff --git a/apps/rocmd/src/cli.rs b/apps/rocmd/src/cli.rs index 8e9e0a2139..0ebd4b6378 100644 --- a/apps/rocmd/src/cli.rs +++ b/apps/rocmd/src/cli.rs @@ -120,7 +120,7 @@ pub(crate) enum Command { } fn print_bridge_snapshot(paths: &AppPaths, pretty: bool) -> Result<()> { - let snapshot = crate::common::build_bridge_snapshot(paths)?; + let snapshot = crate::host_snapshot::build_bridge_snapshot(paths)?; if pretty { println!( diff --git a/apps/rocmd/src/common.rs b/apps/rocmd/src/common.rs index 82b57aff41..7b0ecb1608 100644 --- a/apps/rocmd/src/common.rs +++ b/apps/rocmd/src/common.rs @@ -2,38 +2,17 @@ // // SPDX-License-Identifier: MIT -use anyhow::{Context, Result}; -use rocm_core::{ - AppPaths, AutomationRuntimeState, CodexBridgeEngine, CodexBridgeGpuSnapshot, - CodexBridgeSnapshot, CommandCapture, ExamineSummary, RocmCliConfig, - default_engine_for_platform, load_recent_automation_events, resolve_amd_smi_binary, - unix_time_millis, -}; +#[cfg(test)] +use anyhow::Context; +use anyhow::Result; +use rocm_core::{AppPaths, CommandCapture}; #[cfg(test)] use rocm_engine_protocol::EnginePluginDescriptor; use serde_json::Value; #[cfg(test)] use std::path::PathBuf; -use std::process::Command as ProcessCommand; use std::time::Duration; -const AMD_SMI_PROBE_TIMEOUT: Duration = Duration::from_secs(2); - -pub(crate) fn build_bridge_snapshot(paths: &AppPaths) -> Result { - let config = RocmCliConfig::load(paths).unwrap_or_default(); - Ok(CodexBridgeSnapshot { - protocol: "rocmd-codex-bridge-v0".to_owned(), - generated_at_unix_ms: unix_time_millis(), - examine: ExamineSummary::gather()?, - gpu: gather_gpu_snapshot_for_config(&config), - config, - automation_runtime: AutomationRuntimeState::load(paths)?, - recent_automation_events: load_recent_automation_events(paths, 32)?, - engines: bridge_engine_inventory(), - services: crate::persistence::load_managed_services(paths)?, - }) -} - pub(crate) fn update_check_message(status: &str) -> &'static str { match status { "repair_available" => { @@ -47,146 +26,6 @@ pub(crate) fn update_check_message(status: &str) -> &'static str { } } -fn gather_gpu_snapshot() -> CodexBridgeGpuSnapshot { - let static_snapshot = match capture_amd_smi_json(&["static", "-a", "-g", "all", "--json"]) { - Ok(value) => Some(value), - Err(error) => { - return CodexBridgeGpuSnapshot { - // `{error:#}` rather than `to_string()`: the outermost context - // alone says "failed to launch amd-smi ...", which names the - // wrong thing when what actually happened was a timeout. Pinned - // by `examine-25`, which asserts that the rendered note names - // the timeout and not just the launch. - // - // Every failure `run_with_timeout` raises carries its cause the - // same way, since all of them arrive under that one context: - // the timeout, the failure to spawn, and the poll and reap - // failures of its wait loop. Spawn is the case on a host where - // none of the paths `resolve_amd_smi_binary` searches holds an - // `amd-smi`, down to the bare name on `PATH`; the cause is then - // the OS's own "not found" text, which is not the same string - // on Linux and Windows, and `rocmd` runs this probe on both -- - // so none of it is quoted here, and no test pins it. A host - // with no GPU but an amd-smi installed is a different case: it - // spawns and fails at a non-zero exit. (The non-zero-exit and - // parse failures below are their own outermost context and - // already read correctly.) - // - // The note is the only account of any of this that reaches the - // bridge snapshot and the automation watcher, so it carries the - // causes too -- bounded, because it is persisted; see - // `run_with_timeout`'s `MAX_QUOTED_OUTPUT_CHARS`. - amd_smi_available: false, - static_snapshot: None, - monitor_snapshot: None, - note: Some(format!("{error:#}")), - }; - } - }; - - let monitor_snapshot = match capture_amd_smi_json(&[ - "monitor", "-p", "-t", "-u", "-m", "-v", "-g", "all", "--json", - ]) { - Ok(value) => Some(value), - Err(error) => { - return CodexBridgeGpuSnapshot { - // Same reasoning as the `static` probe above, but a different - // test: examine-25's stalled `static` probe returns before this - // branch is ever reached, so reverting this `{error:#}` is - // caught by `examine-26` alone. Pinned by `examine-26`. - amd_smi_available: true, - static_snapshot, - monitor_snapshot: None, - note: Some(format!("{error:#}")), - }; - } - }; - - CodexBridgeGpuSnapshot { - amd_smi_available: true, - static_snapshot, - monitor_snapshot, - note: None, - } -} - -pub(crate) fn gather_gpu_snapshot_for_config(config: &RocmCliConfig) -> CodexBridgeGpuSnapshot { - if config.telemetry.local_inspection_enabled() { - gather_gpu_snapshot() - } else { - CodexBridgeGpuSnapshot { - amd_smi_available: false, - static_snapshot: None, - monitor_snapshot: None, - note: Some( - "gpu telemetry is disabled by rocm-cli config; no external reporting is implemented" - .to_owned(), - ), - } - } -} - -fn capture_amd_smi_json(args: &[&str]) -> Result { - let amd_smi_binary = resolve_amd_smi_binary(); - let mut command = ProcessCommand::new(&amd_smi_binary); - // No stdio here: `run_with_timeout` sets null stdin and piped stdout/stderr - // itself, since capturing both is its contract. - command.args(args); - let output = rocm_core::process::run_with_timeout(command, AMD_SMI_PROBE_TIMEOUT, "process") - .with_context(|| format!("failed to launch amd-smi {}", args.join(" ")))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); - let stdout = String::from_utf8_lossy(&output.stdout).trim().to_owned(); - anyhow::bail!( - "amd-smi {} failed: {}", - args.join(" "), - if !stderr.is_empty() { - stderr - } else if !stdout.is_empty() { - stdout - } else { - format!("exit status {}", output.status) - } - ); - } - - serde_json::from_slice(&output.stdout) - .with_context(|| format!("failed to parse amd-smi {} json", args.join(" "))) -} - -pub(crate) fn bridge_engine_inventory() -> Vec { - let default_engine = default_engine_for_platform(); - let current_exe = std::env::current_exe().ok(); - rocmd_engine_inventory() - .iter() - .map(|(id, summary)| CodexBridgeEngine { - id: (*id).to_owned(), - summary: (*summary).to_owned(), - default_for_platform: *id == default_engine, - installed_binary: true, - binary_path: current_exe.as_ref().map(|path| path.display().to_string()), - }) - .collect() -} - -const fn rocmd_engine_inventory() -> &'static [(&'static str, &'static str)] { - &[ - ( - "lemonade", - "embedded Lemonade server with ROCm llama.cpp backend", - ), - ( - "vllm", - "Linux/WSL ROCm GPU serving engine through external vLLM", - ), - ( - "aim", - "AMD Enterprise AI pre-built Docker serving containers", - ), - ] -} - #[cfg(test)] pub(crate) fn find_engine_plugin_binary( engine: &str, @@ -244,24 +83,6 @@ mod tests { use rocm_core::engine_plugin_dirs; use std::fs; - #[test] - fn gpu_snapshot_respects_disabled_telemetry_policy() { - let mut config = RocmCliConfig::default(); - config.telemetry.mode = rocm_core::TELEMETRY_MODE_OFF.to_owned(); - - let snapshot = gather_gpu_snapshot_for_config(&config); - - assert!(!snapshot.amd_smi_available); - assert!(snapshot.static_snapshot.is_none()); - assert!(snapshot.monitor_snapshot.is_none()); - assert!( - snapshot - .note - .as_deref() - .is_some_and(|note| note.contains("disabled by rocm-cli config")) - ); - } - #[test] fn engine_plugin_discovery_finds_runtime_binary() -> Result<()> { let (root, paths) = temp_app_paths("engine-plugin"); diff --git a/apps/rocmd/src/host_snapshot.rs b/apps/rocmd/src/host_snapshot.rs new file mode 100644 index 0000000000..93ce12edbc --- /dev/null +++ b/apps/rocmd/src/host_snapshot.rs @@ -0,0 +1,193 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +use anyhow::{Context, Result}; +use rocm_core::{ + AppPaths, AutomationRuntimeState, CodexBridgeEngine, CodexBridgeGpuSnapshot, + CodexBridgeSnapshot, ExamineSummary, RocmCliConfig, default_engine_for_platform, + load_recent_automation_events, resolve_amd_smi_binary, unix_time_millis, +}; +use serde_json::Value; +use std::process::Command as ProcessCommand; +use std::time::Duration; + +const AMD_SMI_PROBE_TIMEOUT: Duration = Duration::from_secs(2); + +pub(crate) fn build_bridge_snapshot(paths: &AppPaths) -> Result { + let config = RocmCliConfig::load(paths).unwrap_or_default(); + Ok(CodexBridgeSnapshot { + protocol: "rocmd-codex-bridge-v0".to_owned(), + generated_at_unix_ms: unix_time_millis(), + examine: ExamineSummary::gather()?, + gpu: gather_gpu_snapshot_for_config(&config), + config, + automation_runtime: AutomationRuntimeState::load(paths)?, + recent_automation_events: load_recent_automation_events(paths, 32)?, + engines: bridge_engine_inventory(), + services: crate::persistence::load_managed_services(paths)?, + }) +} + +fn gather_gpu_snapshot() -> CodexBridgeGpuSnapshot { + let static_snapshot = match capture_amd_smi_json(&["static", "-a", "-g", "all", "--json"]) { + Ok(value) => Some(value), + Err(error) => { + return CodexBridgeGpuSnapshot { + // `{error:#}` rather than `to_string()`: the outermost context + // alone says "failed to launch amd-smi ...", which names the + // wrong thing when what actually happened was a timeout. Pinned + // by `examine-25`, which asserts that the rendered note names + // the timeout and not just the launch. + // + // Every failure `run_with_timeout` raises carries its cause the + // same way, since all of them arrive under that one context: + // the timeout, the failure to spawn, and the poll and reap + // failures of its wait loop. Spawn is the case on a host where + // none of the paths `resolve_amd_smi_binary` searches holds an + // `amd-smi`, down to the bare name on `PATH`; the cause is then + // the OS's own "not found" text, which is not the same string + // on Linux and Windows, and `rocmd` runs this probe on both -- + // so none of it is quoted here, and no test pins it. A host + // with no GPU but an amd-smi installed is a different case: it + // spawns and fails at a non-zero exit. (The non-zero-exit and + // parse failures below are their own outermost context and + // already read correctly.) + // + // The note is the only account of any of this that reaches the + // bridge snapshot and the automation watcher, so it carries the + // causes too -- bounded, because it is persisted; see + // `run_with_timeout`'s `MAX_QUOTED_OUTPUT_CHARS`. + amd_smi_available: false, + static_snapshot: None, + monitor_snapshot: None, + note: Some(format!("{error:#}")), + }; + } + }; + + let monitor_snapshot = match capture_amd_smi_json(&[ + "monitor", "-p", "-t", "-u", "-m", "-v", "-g", "all", "--json", + ]) { + Ok(value) => Some(value), + Err(error) => { + return CodexBridgeGpuSnapshot { + // Same reasoning as the `static` probe above, but a different + // test: examine-25's stalled `static` probe returns before this + // branch is ever reached, so reverting this `{error:#}` is + // caught by `examine-26` alone. Pinned by `examine-26`. + amd_smi_available: true, + static_snapshot, + monitor_snapshot: None, + note: Some(format!("{error:#}")), + }; + } + }; + + CodexBridgeGpuSnapshot { + amd_smi_available: true, + static_snapshot, + monitor_snapshot, + note: None, + } +} + +pub(crate) fn gather_gpu_snapshot_for_config(config: &RocmCliConfig) -> CodexBridgeGpuSnapshot { + if config.telemetry.local_inspection_enabled() { + gather_gpu_snapshot() + } else { + CodexBridgeGpuSnapshot { + amd_smi_available: false, + static_snapshot: None, + monitor_snapshot: None, + note: Some( + "gpu telemetry is disabled by rocm-cli config; no external reporting is implemented" + .to_owned(), + ), + } + } +} + +fn capture_amd_smi_json(args: &[&str]) -> Result { + let amd_smi_binary = resolve_amd_smi_binary(); + let mut command = ProcessCommand::new(&amd_smi_binary); + // No stdio here: `run_with_timeout` sets null stdin and piped stdout/stderr + // itself, since capturing both is its contract. + command.args(args); + let output = rocm_core::process::run_with_timeout(command, AMD_SMI_PROBE_TIMEOUT, "process") + .with_context(|| format!("failed to launch amd-smi {}", args.join(" ")))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); + let stdout = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + anyhow::bail!( + "amd-smi {} failed: {}", + args.join(" "), + if !stderr.is_empty() { + stderr + } else if !stdout.is_empty() { + stdout + } else { + format!("exit status {}", output.status) + } + ); + } + + serde_json::from_slice(&output.stdout) + .with_context(|| format!("failed to parse amd-smi {} json", args.join(" "))) +} + +pub(crate) fn bridge_engine_inventory() -> Vec { + let default_engine = default_engine_for_platform(); + let current_exe = std::env::current_exe().ok(); + rocmd_engine_inventory() + .iter() + .map(|(id, summary)| CodexBridgeEngine { + id: (*id).to_owned(), + summary: (*summary).to_owned(), + default_for_platform: *id == default_engine, + installed_binary: true, + binary_path: current_exe.as_ref().map(|path| path.display().to_string()), + }) + .collect() +} + +const fn rocmd_engine_inventory() -> &'static [(&'static str, &'static str)] { + &[ + ( + "lemonade", + "embedded Lemonade server with ROCm llama.cpp backend", + ), + ( + "vllm", + "Linux/WSL ROCm GPU serving engine through external vLLM", + ), + ( + "aim", + "AMD Enterprise AI pre-built Docker serving containers", + ), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn gpu_snapshot_respects_disabled_telemetry_policy() { + let mut config = RocmCliConfig::default(); + config.telemetry.mode = rocm_core::TELEMETRY_MODE_OFF.to_owned(); + + let snapshot = gather_gpu_snapshot_for_config(&config); + + assert!(!snapshot.amd_smi_available); + assert!(snapshot.static_snapshot.is_none()); + assert!(snapshot.monitor_snapshot.is_none()); + assert!( + snapshot + .note + .as_deref() + .is_some_and(|note| note.contains("disabled by rocm-cli config")) + ); + } +} diff --git a/apps/rocmd/src/lib.rs b/apps/rocmd/src/lib.rs index a88933bc9b..b77042e705 100644 --- a/apps/rocmd/src/lib.rs +++ b/apps/rocmd/src/lib.rs @@ -8,6 +8,7 @@ mod cli; mod common; mod daemon; mod engine_ipc; +mod host_snapshot; mod mcp; mod persistence; mod sandbox; diff --git a/apps/rocmd/src/mcp.rs b/apps/rocmd/src/mcp.rs index 1fae7574a8..e46db62f2c 100644 --- a/apps/rocmd/src/mcp.rs +++ b/apps/rocmd/src/mcp.rs @@ -3,6 +3,7 @@ // SPDX-License-Identifier: MIT use crate::common; +use crate::host_snapshot; use crate::persistence; use anyhow::{Context, Result, bail}; #[cfg(test)] @@ -585,7 +586,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result { - let snapshot = common::build_bridge_snapshot(paths)?; + let snapshot = host_snapshot::build_bridge_snapshot(paths)?; Ok(tool_success( format!( "Captured bridge snapshot for {} / {} with default engine `{}`.", @@ -596,7 +597,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result { let config = RocmCliConfig::load(paths).unwrap_or_default(); - let gpu = common::gather_gpu_snapshot_for_config(&config); + let gpu = host_snapshot::gather_gpu_snapshot_for_config(&config); let status = if !config.telemetry.local_inspection_enabled() { "GPU telemetry is disabled by rocm-cli config." } else if gpu.amd_smi_available { @@ -607,7 +608,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result { - let engines = common::bridge_engine_inventory(); + let engines = host_snapshot::bridge_engine_inventory(); Ok(tool_success( format!("Found {} engine entries.", engines.len()), json!({ "engines": engines }), diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index c4b9c4f6c2..cd10cc0a41 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -65,7 +65,7 @@ fn collect_automation_events( state: &AutomationRuntimeState, ) -> Result> { collect_automation_events_with_gpu_snapshot(paths, state, || { - crate::common::gather_gpu_snapshot_for_config(config) + crate::host_snapshot::gather_gpu_snapshot_for_config(config) }) } diff --git a/crates/rocm-core/src/process.rs b/crates/rocm-core/src/process.rs index afbea28608..758d19762a 100644 --- a/crates/rocm-core/src/process.rs +++ b/crates/rocm-core/src/process.rs @@ -2010,9 +2010,11 @@ mod tests { // pre-setting it to the marker is cmd's equivalent of the unix // `read || echo`. A stdin left open blocks in `set /p` instead. // - // Both callers in `apps/rocmd/src/common.rs` run on Windows, and - // `run_with_timeout` has no `cfg(windows)` branch, so this covers the - // same promise with the same two-sidedness as the unix case. + // Both callers in `rocmd` run on Windows -- the amd-smi probe in + // `apps/rocmd/src/host_snapshot.rs` and the `rocm` capture that + // `apps/rocmd/src/common.rs` makes through `run_rocm_capture_with_env` + // -- and `run_with_timeout` has no `cfg(windows)` branch, so this covers + // the same promise with the same two-sidedness as the unix case. let fixture = WindowsFixture::script( "stdin", &format!( diff --git a/crates/rocm-core/src/rocm_install.rs b/crates/rocm-core/src/rocm_install.rs index 8ddab2ad5b..7830e4d4e6 100644 --- a/crates/rocm-core/src/rocm_install.rs +++ b/crates/rocm-core/src/rocm_install.rs @@ -28,7 +28,7 @@ pub struct LegacyRocmSummary { /// declares no version anywhere. /// /// Optional and defaulted so the daemon's serialised snapshot - /// (`apps/rocmd/src/lib.rs`) written before this field existed still loads. + /// (`apps/rocmd/src/host_snapshot.rs`) written before this field existed still loads. #[serde(default)] pub version: Option, } diff --git a/docs/architecture.md b/docs/architecture.md index 48062861e9..a1594abc26 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -84,9 +84,10 @@ Further `apps/rocm` modules, not part of the `main.rs` split: Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): - [`cli.rs`](../apps/rocmd/src/cli.rs) — the `Cli`/`Command` clap definitions, the bridge-snapshot diagnostic printer, and the top-level dispatch in `run_cli`/`run_bin_cli`/`run_from_args`. -- [`common.rs`](../apps/rocmd/src/common.rs) — helpers the other modules call into: GPU/amd-smi snapshotting, the bridge-snapshot diagnostic, the `rocm`-subprocess capture wrapper (`run_rocm_capture_for_paths`) the sandbox and MCP clusters both call (it only chooses the child's `ROCM_CLI_*_DIR` environment; `CommandCapture` and the spawn itself live in [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), shared with `rocm`'s chat tool bridge, and the bounded-execution half in [`process.rs`](../crates/rocm-core/src/process.rs)), and small arg/JSON-payload utilities (`payload_string`, for example, is read by both [`watchers.rs`](../apps/rocmd/src/watchers.rs) and [`webhook.rs`](../apps/rocmd/src/webhook.rs)). +- [`common.rs`](../apps/rocmd/src/common.rs) — helpers the other modules call into: the read-only `rocm update` message, the `rocm`-subprocess capture wrapper (`run_rocm_capture_for_paths`) the sandbox and MCP clusters both call (it only chooses the child's `ROCM_CLI_*_DIR` environment; `CommandCapture` and the spawn itself live in [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), shared with `rocm`'s chat tool bridge, and the bounded-execution half in [`process.rs`](../crates/rocm-core/src/process.rs)), and small arg/JSON-payload utilities (`payload_string`, for example, is read by both [`watchers.rs`](../apps/rocmd/src/watchers.rs) and [`webhook.rs`](../apps/rocmd/src/webhook.rs)). - [`daemon.rs`](../apps/rocmd/src/daemon.rs) — the foreground `rocmd run` loop (`run_daemon`: evaluate the watchers each tick, dispatch local webhook events, write the runtime state, stop on a signal) and `rocmd status` (`print_status`), with the watcher tick interval. It uses [`watchers.rs`](../apps/rocmd/src/watchers.rs), [`webhook.rs`](../apps/rocmd/src/webhook.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs); `cli.rs` calls it. - [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) — talking to an engine and guarding its endpoint key: `engine_request` (the stdio request to an engine process), the healthcheck call and its readiness and recoverability checks, and the endpoint-key helpers `apply_endpoint_key_env` and `ensure_public_service_has_endpoint_key`. It uses no other module of the crate; `service.rs`, `supervise.rs` and `watchers.rs` call it. +- [`host_snapshot.rs`](../apps/rocmd/src/host_snapshot.rs) — the host snapshot: `build_bridge_snapshot` (the bridge snapshot printed by `rocmd bridge-snapshot` and returned by the `bridge_snapshot` MCP tool, assembled from the examine summary, the GPU snapshot, the automation runtime state and recent events, the engine inventory and the managed services), the amd-smi GPU snapshot (`gather_gpu_snapshot_for_config`, which honours the telemetry setting, over time-bounded amd-smi probes) and `bridge_engine_inventory`. It uses only [`persistence.rs`](../apps/rocmd/src/persistence.rs); `cli.rs`, `mcp.rs` and `watchers.rs` call it. - [`mcp.rs`](../apps/rocmd/src/mcp.rs) — the MCP stdio server, tool schema table, tool dispatch, and the argv-building and result-shaping helpers behind the MCP tools (the `rocm` subprocess itself is spawned through [`common.rs`](../apps/rocmd/src/common.rs), via `run_rocm_capture` here). - [`persistence.rs`](../apps/rocmd/src/persistence.rs) — `record_event`/`load_managed_services`/`load_service_record`, the automation-event/audit-log and managed-service-registry I/O shared across the daemon's sandbox, MCP, service-lifecycle, and watcher code. - [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) — bubblewrap/native sandbox execution, atomic-write helpers, artifact prefetch policy gating, and the sandbox-tool result shaping for `check_updates`/`driver_plan`/`stop_server`. From 40e87375dd992aee55f43f850d407cb430c5a326 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 2 Nov 2026 11:21:22 +0000 Subject: [PATCH 11/13] refactor(rocmd): move print_json into cli.rs Move print_json out of mcp.rs into cli.rs, where all four of its callers are. It becomes private, and cli.rs no longer reaches into mcp.rs for a printer. Part of the rocmd module-cycle follow-up work. Besides the moved lines and import wiring, the only other changes are that print_json is narrowed from pub(crate) to private and that the docs/architecture.md cli.rs entry names it. No tests move: none target this function directly. Signed-off-by: Jussi Elo --- apps/rocmd/src/cli.rs | 17 +++++++++++++---- apps/rocmd/src/mcp.rs | 9 --------- docs/architecture.md | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/apps/rocmd/src/cli.rs b/apps/rocmd/src/cli.rs index 0ebd4b6378..d0f4aa9fea 100644 --- a/apps/rocmd/src/cli.rs +++ b/apps/rocmd/src/cli.rs @@ -6,6 +6,7 @@ use crate::sandbox_tool::{SandboxToolArg, SandboxToolPolicy}; use anyhow::{Context, Result, bail}; use clap::{Parser, Subcommand}; use rocm_core::{AppPaths, DEFAULT_LOCAL_HOST}; +use serde::Serialize; use serde_json::Value; use serde_json::json; use std::ffi::OsString; @@ -138,6 +139,14 @@ fn print_bridge_snapshot(paths: &AppPaths, pretty: bool) -> Result<()> { Ok(()) } +fn print_json(value: &T) -> Result<()> { + println!( + "{}", + serde_json::to_string_pretty(value).context("failed to serialize json output")? + ); + Ok(()) +} + #[tokio::main] pub async fn run_bin_cli() -> Result<()> { let cli = Cli::parse(); @@ -226,7 +235,7 @@ async fn run_cli(cli: Cli) -> Result<()> { allow_native_fallback, policy, )?; - crate::mcp::print_json(&value)?; + print_json(&value)?; } Command::SandboxTool { tool, @@ -250,13 +259,13 @@ async fn run_cli(cli: Cli) -> Result<()> { message, policy, )?; - crate::mcp::print_json(&value)?; + print_json(&value)?; } Command::McpServer => { crate::mcp::run_mcp_server(&paths)?; } Command::McpToolsJson => { - crate::mcp::print_json(&json!({ "tools": crate::mcp::rocm_mcp_tools() }))?; + print_json(&json!({ "tools": crate::mcp::rocm_mcp_tools() }))?; } Command::McpCall { name, @@ -277,7 +286,7 @@ async fn run_cli(cli: Cli) -> Result<()> { "arguments": arguments, }), )?; - crate::mcp::print_json(&result)?; + print_json(&result)?; } } diff --git a/apps/rocmd/src/mcp.rs b/apps/rocmd/src/mcp.rs index e46db62f2c..5cf6e335c9 100644 --- a/apps/rocmd/src/mcp.rs +++ b/apps/rocmd/src/mcp.rs @@ -12,7 +12,6 @@ use rocm_core::{ AppPaths, AutomationRuntimeState, CommandCapture, DEFAULT_LOCAL_HOST, ExamineSummary, RocmCliConfig, command_capture_text, load_recent_automation_events, }; -use serde::Serialize; use serde_json::Value; use serde_json::json; use std::collections::VecDeque; @@ -180,14 +179,6 @@ fn write_json_line(writer: &mut impl Write, value: &Value) -> Result<()> { Ok(()) } -pub(crate) fn print_json(value: &T) -> Result<()> { - println!( - "{}", - serde_json::to_string_pretty(value).context("failed to serialize json output")? - ); - Ok(()) -} - pub(crate) fn rocm_mcp_tools() -> Vec { vec![ rocm_mcp_tool( diff --git a/docs/architecture.md b/docs/architecture.md index a1594abc26..b1ece959d9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -83,7 +83,7 @@ Further `apps/rocm` modules, not part of the `main.rs` split: Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): -- [`cli.rs`](../apps/rocmd/src/cli.rs) — the `Cli`/`Command` clap definitions, the bridge-snapshot diagnostic printer, and the top-level dispatch in `run_cli`/`run_bin_cli`/`run_from_args`. +- [`cli.rs`](../apps/rocmd/src/cli.rs) — the `Cli`/`Command` clap definitions, the bridge-snapshot diagnostic printer, the pretty-JSON printer for the subcommands that print a result (`print_json`), and the top-level dispatch in `run_cli`/`run_bin_cli`/`run_from_args`. - [`common.rs`](../apps/rocmd/src/common.rs) — helpers the other modules call into: the read-only `rocm update` message, the `rocm`-subprocess capture wrapper (`run_rocm_capture_for_paths`) the sandbox and MCP clusters both call (it only chooses the child's `ROCM_CLI_*_DIR` environment; `CommandCapture` and the spawn itself live in [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), shared with `rocm`'s chat tool bridge, and the bounded-execution half in [`process.rs`](../crates/rocm-core/src/process.rs)), and small arg/JSON-payload utilities (`payload_string`, for example, is read by both [`watchers.rs`](../apps/rocmd/src/watchers.rs) and [`webhook.rs`](../apps/rocmd/src/webhook.rs)). - [`daemon.rs`](../apps/rocmd/src/daemon.rs) — the foreground `rocmd run` loop (`run_daemon`: evaluate the watchers each tick, dispatch local webhook events, write the runtime state, stop on a signal) and `rocmd status` (`print_status`), with the watcher tick interval. It uses [`watchers.rs`](../apps/rocmd/src/watchers.rs), [`webhook.rs`](../apps/rocmd/src/webhook.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs); `cli.rs` calls it. - [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) — talking to an engine and guarding its endpoint key: `engine_request` (the stdio request to an engine process), the healthcheck call and its readiness and recoverability checks, and the endpoint-key helpers `apply_endpoint_key_env` and `ensure_public_service_has_endpoint_key`. It uses no other module of the crate; `service.rs`, `supervise.rs` and `watchers.rs` call it. From ab57ea0b2baa5764decfa5a46acd212b33da8805 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 2 Nov 2026 14:04:50 +0000 Subject: [PATCH 12/13] refactor(rocmd): make mcp.rs's rocm capture wrapper private run_rocm_capture moved into mcp.rs with its only callers, so nothing outside the module needs crate-wide visibility any more. Signed-off-by: Jussi Elo --- apps/rocmd/src/mcp.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/rocmd/src/mcp.rs b/apps/rocmd/src/mcp.rs index 5cf6e335c9..cc38a042a6 100644 --- a/apps/rocmd/src/mcp.rs +++ b/apps/rocmd/src/mcp.rs @@ -772,7 +772,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result Result { +fn run_rocm_capture(args: &[&str]) -> Result { let paths = AppPaths::discover()?; common::run_rocm_capture_for_paths(&paths, args, Duration::from_mins(2)) } From 5ceb77ff39085a6c1b626bf8affad29fb2007904 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 2 Nov 2026 14:06:49 +0000 Subject: [PATCH 13/13] docs(architecture): link the callers named in the new rocmd entries The five new module entries cited sibling files as bare backtick paths. Link each one relative to the doc so the docs-links check follows it when a file moves. Signed-off-by: Jussi Elo --- docs/architecture.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index b1ece959d9..75bdaa2106 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -85,15 +85,15 @@ Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): - [`cli.rs`](../apps/rocmd/src/cli.rs) — the `Cli`/`Command` clap definitions, the bridge-snapshot diagnostic printer, the pretty-JSON printer for the subcommands that print a result (`print_json`), and the top-level dispatch in `run_cli`/`run_bin_cli`/`run_from_args`. - [`common.rs`](../apps/rocmd/src/common.rs) — helpers the other modules call into: the read-only `rocm update` message, the `rocm`-subprocess capture wrapper (`run_rocm_capture_for_paths`) the sandbox and MCP clusters both call (it only chooses the child's `ROCM_CLI_*_DIR` environment; `CommandCapture` and the spawn itself live in [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), shared with `rocm`'s chat tool bridge, and the bounded-execution half in [`process.rs`](../crates/rocm-core/src/process.rs)), and small arg/JSON-payload utilities (`payload_string`, for example, is read by both [`watchers.rs`](../apps/rocmd/src/watchers.rs) and [`webhook.rs`](../apps/rocmd/src/webhook.rs)). -- [`daemon.rs`](../apps/rocmd/src/daemon.rs) — the foreground `rocmd run` loop (`run_daemon`: evaluate the watchers each tick, dispatch local webhook events, write the runtime state, stop on a signal) and `rocmd status` (`print_status`), with the watcher tick interval. It uses [`watchers.rs`](../apps/rocmd/src/watchers.rs), [`webhook.rs`](../apps/rocmd/src/webhook.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs); `cli.rs` calls it. -- [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) — talking to an engine and guarding its endpoint key: `engine_request` (the stdio request to an engine process), the healthcheck call and its readiness and recoverability checks, and the endpoint-key helpers `apply_endpoint_key_env` and `ensure_public_service_has_endpoint_key`. It uses no other module of the crate; `service.rs`, `supervise.rs` and `watchers.rs` call it. -- [`host_snapshot.rs`](../apps/rocmd/src/host_snapshot.rs) — the host snapshot: `build_bridge_snapshot` (the bridge snapshot printed by `rocmd bridge-snapshot` and returned by the `bridge_snapshot` MCP tool, assembled from the examine summary, the GPU snapshot, the automation runtime state and recent events, the engine inventory and the managed services), the amd-smi GPU snapshot (`gather_gpu_snapshot_for_config`, which honours the telemetry setting, over time-bounded amd-smi probes) and `bridge_engine_inventory`. It uses only [`persistence.rs`](../apps/rocmd/src/persistence.rs); `cli.rs`, `mcp.rs` and `watchers.rs` call it. +- [`daemon.rs`](../apps/rocmd/src/daemon.rs) — the foreground `rocmd run` loop (`run_daemon`: evaluate the watchers each tick, dispatch local webhook events, write the runtime state, stop on a signal) and `rocmd status` (`print_status`), with the watcher tick interval. It uses [`watchers.rs`](../apps/rocmd/src/watchers.rs), [`webhook.rs`](../apps/rocmd/src/webhook.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs); [`cli.rs`](../apps/rocmd/src/cli.rs) calls it. +- [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) — talking to an engine and guarding its endpoint key: `engine_request` (the stdio request to an engine process), the healthcheck call and its readiness and recoverability checks, and the endpoint-key helpers `apply_endpoint_key_env` and `ensure_public_service_has_endpoint_key`. It uses no other module of the crate; [`service.rs`](../apps/rocmd/src/service.rs), [`supervise.rs`](../apps/rocmd/src/supervise.rs) and [`watchers.rs`](../apps/rocmd/src/watchers.rs) call it. +- [`host_snapshot.rs`](../apps/rocmd/src/host_snapshot.rs) — the host snapshot: `build_bridge_snapshot` (the bridge snapshot printed by `rocmd bridge-snapshot` and returned by the `bridge_snapshot` MCP tool, assembled from the examine summary, the GPU snapshot, the automation runtime state and recent events, the engine inventory and the managed services), the amd-smi GPU snapshot (`gather_gpu_snapshot_for_config`, which honours the telemetry setting, over time-bounded amd-smi probes) and `bridge_engine_inventory`. It uses only [`persistence.rs`](../apps/rocmd/src/persistence.rs); [`cli.rs`](../apps/rocmd/src/cli.rs), [`mcp.rs`](../apps/rocmd/src/mcp.rs) and [`watchers.rs`](../apps/rocmd/src/watchers.rs) call it. - [`mcp.rs`](../apps/rocmd/src/mcp.rs) — the MCP stdio server, tool schema table, tool dispatch, and the argv-building and result-shaping helpers behind the MCP tools (the `rocm` subprocess itself is spawned through [`common.rs`](../apps/rocmd/src/common.rs), via `run_rocm_capture` here). - [`persistence.rs`](../apps/rocmd/src/persistence.rs) — `record_event`/`load_managed_services`/`load_service_record`, the automation-event/audit-log and managed-service-registry I/O shared across the daemon's sandbox, MCP, service-lifecycle, and watcher code. - [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) — bubblewrap/native sandbox execution, atomic-write helpers, artifact prefetch policy gating, and the sandbox-tool result shaping for `check_updates`/`driver_plan`/`stop_server`. -- [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs) — the sandbox-tool vocabulary: the `SandboxToolArg` value enum naming each sandbox tool, and `SandboxToolPolicy`, the artifact-download policy built from the `--allow-*` flags (including the Hugging Face token read from the environment when authenticated downloads are allowed). It uses no other module of the crate, so modules that need the vocabulary do not import `cli.rs`. +- [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs) — the sandbox-tool vocabulary: the `SandboxToolArg` value enum naming each sandbox tool, and `SandboxToolPolicy`, the artifact-download policy built from the `--allow-*` flags (including the Hugging Face token read from the environment when authenticated downloads are allowed). It uses no other module of the crate, so modules that need the vocabulary do not import [`cli.rs`](../apps/rocmd/src/cli.rs). - [`service.rs`](../apps/rocmd/src/service.rs) — managed-service stop: terminating the recorded processes of a service after verifying their identity, and the stop report (`stop_managed_service`, `stop_report_confirmed`) that the sandbox and MCP tools read. -- [`supervise.rs`](../apps/rocmd/src/supervise.rs) — launching and re-launching a managed service's supervisor: `supervise_service` (the `rocmd supervise` entry point: launch the engine child, record its identity, wait for readiness, record the exit) with the serve-log startup-phase polling and readiness wait it uses, and `restart_managed_service` (the recovery respawn: re-launch `rocmd supervise` detached for a service and record the new supervisor). Writes after a supervisor's first go through the private `update_supervised_record`, which re-reads the record and leaves a stop marker or another supervisor's record alone. Its production code uses only [`common.rs`](../apps/rocmd/src/common.rs), [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs). `cli.rs` calls `supervise_service`; `watchers.rs` (the server-recover handler) and `sandbox.rs` (the `restart_server` tool) call `restart_managed_service`. +- [`supervise.rs`](../apps/rocmd/src/supervise.rs) — launching and re-launching a managed service's supervisor: `supervise_service` (the `rocmd supervise` entry point: launch the engine child, record its identity, wait for readiness, record the exit) with the serve-log startup-phase polling and readiness wait it uses, and `restart_managed_service` (the recovery respawn: re-launch `rocmd supervise` detached for a service and record the new supervisor). Writes after a supervisor's first go through the private `update_supervised_record`, which re-reads the record and leaves a stop marker or another supervisor's record alone. Its production code uses only [`common.rs`](../apps/rocmd/src/common.rs), [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs). [`cli.rs`](../apps/rocmd/src/cli.rs) calls `supervise_service`; [`watchers.rs`](../apps/rocmd/src/watchers.rs) (the server-recover handler) and [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) (the `restart_server` tool) call `restart_managed_service`. - [`watchers.rs`](../apps/rocmd/src/watchers.rs) — event collection/dispatch for all built-in watchers (TheRock update, GPU metrics/thermal-pressure, cache-warm, driver-upgrade, server-recover), plus managed-service recovery classification and automation-proposal queuing. - [`webhook.rs`](../apps/rocmd/src/webhook.rs) — the local webhook source: its axum routes, request validation, and watcher-kind allow-list.