diff --git a/apps/rocm/src/advised_commands.rs b/apps/rocm/src/advised_commands.rs index e7c84a3420..ddf04c18e5 100644 --- a/apps/rocm/src/advised_commands.rs +++ b/apps/rocm/src/advised_commands.rs @@ -1201,12 +1201,12 @@ const NOT_INVOCATIONS: &[(&str, &str, &str)] = &[ "error message", ), ( - "apps/rocmd/src/service.rs", + "apps/rocmd/src/daemon.rs", "rocmd automation supervisor started", "log line", ), ( - "apps/rocmd/src/service.rs", + "apps/rocmd/src/daemon.rs", "rocmd automation supervisor stopped", "log line", ), diff --git a/apps/rocmd/src/cli.rs b/apps/rocmd/src/cli.rs index 10fa91d81c..d0f4aa9fea 100644 --- a/apps/rocmd/src/cli.rs +++ b/apps/rocmd/src/cli.rs @@ -2,9 +2,11 @@ // // SPDX-License-Identifier: MIT +use crate::sandbox_tool::{SandboxToolArg, SandboxToolPolicy}; use anyhow::{Context, Result, bail}; -use clap::{Parser, Subcommand, ValueEnum}; +use clap::{Parser, Subcommand}; use rocm_core::{AppPaths, DEFAULT_LOCAL_HOST}; +use serde::Serialize; use serde_json::Value; use serde_json::json; use std::ffi::OsString; @@ -118,74 +120,8 @@ pub(crate) enum Command { }, } -#[derive(Debug, Clone, Copy, Eq, PartialEq, ValueEnum)] -#[value(rename_all = "snake_case")] -pub(crate) enum SandboxToolArg { - CheckUpdates, - DriverPlan, - ExamineSnapshot, - ListServers, - RestartServer, - StopServer, - PrefetchArtifact, - NotifyUser, -} - -impl SandboxToolArg { - pub(crate) const fn as_cli_value(self) -> &'static str { - match self { - Self::CheckUpdates => "check_updates", - Self::DriverPlan => "driver_plan", - Self::ExamineSnapshot => "examine_snapshot", - Self::ListServers => "list_servers", - Self::RestartServer => "restart_server", - Self::StopServer => "stop_server", - Self::PrefetchArtifact => "prefetch_artifact", - Self::NotifyUser => "notify_user", - } - } - - #[cfg(target_os = "linux")] - pub(crate) const fn writes_data(self) -> bool { - matches!( - self, - Self::RestartServer | Self::StopServer | Self::NotifyUser - ) - } - - #[cfg(target_os = "linux")] - pub(crate) const fn writes_cache(self) -> bool { - matches!(self, Self::CheckUpdates | Self::PrefetchArtifact) - } -} - -#[derive(Debug, Clone, Default)] -pub(crate) struct SandboxToolPolicy { - pub(crate) allow_artifact_download: bool, - pub(crate) artifact_max_bytes: Option, - pub(crate) allow_huggingface_download: bool, - pub(crate) huggingface_token: Option, -} - -impl SandboxToolPolicy { - pub(crate) fn from_cli( - allow_artifact_download: bool, - artifact_max_bytes: Option, - allow_huggingface_download: bool, - ) -> Self { - Self { - allow_artifact_download, - artifact_max_bytes, - allow_huggingface_download, - huggingface_token: allow_huggingface_download - .then(resolve_huggingface_token) - .flatten(), - } - } -} - fn print_bridge_snapshot(paths: &AppPaths, pretty: bool) -> Result<()> { - let snapshot = crate::common::build_bridge_snapshot(paths)?; + let snapshot = crate::host_snapshot::build_bridge_snapshot(paths)?; if pretty { println!( @@ -203,19 +139,12 @@ fn print_bridge_snapshot(paths: &AppPaths, pretty: bool) -> Result<()> { Ok(()) } -fn resolve_huggingface_token() -> Option { - [ - "ROCM_CLI_HUGGINGFACE_TOKEN", - "HF_TOKEN", - "HUGGING_FACE_HUB_TOKEN", - ] - .iter() - .find_map(|name| { - std::env::var(name) - .ok() - .map(|value| value.trim().to_owned()) - .filter(|value| !value.is_empty()) - }) +fn print_json(value: &T) -> Result<()> { + println!( + "{}", + serde_json::to_string_pretty(value).context("failed to serialize json output")? + ); + Ok(()) } #[tokio::main] @@ -249,7 +178,7 @@ async fn run_cli(cli: Cli) -> Result<()> { Command::Run { automations_enabled, local_webhook_port, - } => crate::service::run_daemon(&paths, automations_enabled, local_webhook_port).await?, + } => crate::daemon::run_daemon(&paths, automations_enabled, local_webhook_port).await?, Command::Supervise { service_id, engine, @@ -262,7 +191,7 @@ async fn run_cli(cli: Cli) -> Result<()> { device_policy, gpu, engine_recipe_json, - } => crate::service::supervise_service( + } => crate::supervise::supervise_service( &paths, service_id, engine, @@ -277,7 +206,7 @@ async fn run_cli(cli: Cli) -> Result<()> { engine_recipe_json, )?, Command::Status => { - crate::service::print_status(&paths)?; + crate::daemon::print_status(&paths)?; } Command::BridgeSnapshot { pretty } => { print_bridge_snapshot(&paths, pretty)?; @@ -306,7 +235,7 @@ async fn run_cli(cli: Cli) -> Result<()> { allow_native_fallback, policy, )?; - crate::mcp::print_json(&value)?; + print_json(&value)?; } Command::SandboxTool { tool, @@ -330,13 +259,13 @@ async fn run_cli(cli: Cli) -> Result<()> { message, policy, )?; - crate::mcp::print_json(&value)?; + print_json(&value)?; } Command::McpServer => { crate::mcp::run_mcp_server(&paths)?; } Command::McpToolsJson => { - crate::mcp::print_json(&json!({ "tools": crate::mcp::rocm_mcp_tools() }))?; + print_json(&json!({ "tools": crate::mcp::rocm_mcp_tools() }))?; } Command::McpCall { name, @@ -357,7 +286,7 @@ async fn run_cli(cli: Cli) -> Result<()> { "arguments": arguments, }), )?; - crate::mcp::print_json(&result)?; + print_json(&result)?; } } diff --git a/apps/rocmd/src/common.rs b/apps/rocmd/src/common.rs index 5705d27c5f..7b0ecb1608 100644 --- a/apps/rocmd/src/common.rs +++ b/apps/rocmd/src/common.rs @@ -2,47 +2,17 @@ // // SPDX-License-Identifier: MIT -use anyhow::{Context, Result, bail}; -use rocm_core::{ - AppPaths, AutomationRuntimeState, CodexBridgeEngine, CodexBridgeGpuSnapshot, - CodexBridgeSnapshot, CommandCapture, ExamineSummary, RocmCliConfig, - default_engine_for_platform, format_host_port, load_recent_automation_events, - resolve_amd_smi_binary, unix_time_millis, -}; +#[cfg(test)] +use anyhow::Context; +use anyhow::Result; +use rocm_core::{AppPaths, CommandCapture}; #[cfg(test)] use rocm_engine_protocol::EnginePluginDescriptor; -use rocm_engine_protocol::{ - EngineMethod, EngineRequestEnvelope, EngineResponseEnvelope, HealthcheckRequest, - HealthcheckResponse, -}; -use serde::Serialize; -use serde::de::DeserializeOwned; use serde_json::Value; -use std::io::Write; -use std::net::{SocketAddr, TcpStream}; #[cfg(test)] use std::path::PathBuf; -use std::process::{Command as ProcessCommand, Stdio}; -use std::thread; use std::time::Duration; -const AMD_SMI_PROBE_TIMEOUT: Duration = Duration::from_secs(2); - -pub(crate) fn build_bridge_snapshot(paths: &AppPaths) -> Result { - let config = RocmCliConfig::load(paths).unwrap_or_default(); - Ok(CodexBridgeSnapshot { - protocol: "rocmd-codex-bridge-v0".to_owned(), - generated_at_unix_ms: unix_time_millis(), - examine: ExamineSummary::gather()?, - gpu: gather_gpu_snapshot_for_config(&config), - config, - automation_runtime: AutomationRuntimeState::load(paths)?, - recent_automation_events: load_recent_automation_events(paths, 32)?, - engines: bridge_engine_inventory(), - services: crate::persistence::load_managed_services(paths)?, - }) -} - pub(crate) fn update_check_message(status: &str) -> &'static str { match status { "repair_available" => { @@ -56,146 +26,6 @@ pub(crate) fn update_check_message(status: &str) -> &'static str { } } -pub(crate) fn gather_gpu_snapshot() -> CodexBridgeGpuSnapshot { - let static_snapshot = match capture_amd_smi_json(&["static", "-a", "-g", "all", "--json"]) { - Ok(value) => Some(value), - Err(error) => { - return CodexBridgeGpuSnapshot { - // `{error:#}` rather than `to_string()`: the outermost context - // alone says "failed to launch amd-smi ...", which names the - // wrong thing when what actually happened was a timeout. Pinned - // by `examine-25`, which asserts that the rendered note names - // the timeout and not just the launch. - // - // Every failure `run_with_timeout` raises carries its cause the - // same way, since all of them arrive under that one context: - // the timeout, the failure to spawn, and the poll and reap - // failures of its wait loop. Spawn is the case on a host where - // none of the paths `resolve_amd_smi_binary` searches holds an - // `amd-smi`, down to the bare name on `PATH`; the cause is then - // the OS's own "not found" text, which is not the same string - // on Linux and Windows, and `rocmd` runs this probe on both -- - // so none of it is quoted here, and no test pins it. A host - // with no GPU but an amd-smi installed is a different case: it - // spawns and fails at a non-zero exit. (The non-zero-exit and - // parse failures below are their own outermost context and - // already read correctly.) - // - // The note is the only account of any of this that reaches the - // bridge snapshot and the automation watcher, so it carries the - // causes too -- bounded, because it is persisted; see - // `run_with_timeout`'s `MAX_QUOTED_OUTPUT_CHARS`. - amd_smi_available: false, - static_snapshot: None, - monitor_snapshot: None, - note: Some(format!("{error:#}")), - }; - } - }; - - let monitor_snapshot = match capture_amd_smi_json(&[ - "monitor", "-p", "-t", "-u", "-m", "-v", "-g", "all", "--json", - ]) { - Ok(value) => Some(value), - Err(error) => { - return CodexBridgeGpuSnapshot { - // Same reasoning as the `static` probe above, but a different - // test: examine-25's stalled `static` probe returns before this - // branch is ever reached, so reverting this `{error:#}` is - // caught by `examine-26` alone. Pinned by `examine-26`. - amd_smi_available: true, - static_snapshot, - monitor_snapshot: None, - note: Some(format!("{error:#}")), - }; - } - }; - - CodexBridgeGpuSnapshot { - amd_smi_available: true, - static_snapshot, - monitor_snapshot, - note: None, - } -} - -pub(crate) fn gather_gpu_snapshot_for_config(config: &RocmCliConfig) -> CodexBridgeGpuSnapshot { - if config.telemetry.local_inspection_enabled() { - gather_gpu_snapshot() - } else { - CodexBridgeGpuSnapshot { - amd_smi_available: false, - static_snapshot: None, - monitor_snapshot: None, - note: Some( - "gpu telemetry is disabled by rocm-cli config; no external reporting is implemented" - .to_owned(), - ), - } - } -} - -fn capture_amd_smi_json(args: &[&str]) -> Result { - let amd_smi_binary = resolve_amd_smi_binary(); - let mut command = ProcessCommand::new(&amd_smi_binary); - // No stdio here: `run_with_timeout` sets null stdin and piped stdout/stderr - // itself, since capturing both is its contract. - command.args(args); - let output = rocm_core::process::run_with_timeout(command, AMD_SMI_PROBE_TIMEOUT, "process") - .with_context(|| format!("failed to launch amd-smi {}", args.join(" ")))?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); - let stdout = String::from_utf8_lossy(&output.stdout).trim().to_owned(); - anyhow::bail!( - "amd-smi {} failed: {}", - args.join(" "), - if !stderr.is_empty() { - stderr - } else if !stdout.is_empty() { - stdout - } else { - format!("exit status {}", output.status) - } - ); - } - - serde_json::from_slice(&output.stdout) - .with_context(|| format!("failed to parse amd-smi {} json", args.join(" "))) -} - -pub(crate) fn bridge_engine_inventory() -> Vec { - let default_engine = default_engine_for_platform(); - let current_exe = std::env::current_exe().ok(); - rocmd_engine_inventory() - .iter() - .map(|(id, summary)| CodexBridgeEngine { - id: (*id).to_owned(), - summary: (*summary).to_owned(), - default_for_platform: *id == default_engine, - installed_binary: true, - binary_path: current_exe.as_ref().map(|path| path.display().to_string()), - }) - .collect() -} - -const fn rocmd_engine_inventory() -> &'static [(&'static str, &'static str)] { - &[ - ( - "lemonade", - "embedded Lemonade server with ROCm llama.cpp backend", - ), - ( - "vllm", - "Linux/WSL ROCm GPU serving engine through external vLLM", - ), - ( - "aim", - "AMD Enterprise AI pre-built Docker serving containers", - ), - ] -} - #[cfg(test)] pub(crate) fn find_engine_plugin_binary( engine: &str, @@ -212,11 +42,6 @@ where .map(|plugin| plugin.executable_path)) } -pub(crate) fn run_rocm_capture(args: &[&str]) -> Result { - let paths = AppPaths::discover()?; - run_rocm_capture_for_paths(&paths, args, Duration::from_mins(2)) -} - /// Run `rocm` on `paths`' config/data/cache roots, via the shared /// [`rocm_core::run_rocm_capture_with_env`]. pub(crate) fn run_rocm_capture_for_paths( @@ -242,188 +67,13 @@ pub(crate) fn optional_arg(flag: &str, value: Option<&str>) -> Vec { } } -pub(crate) fn engine_healthcheck_response( - paths: &AppPaths, - engine: &str, - service_id: &str, -) -> Result { - engine_request::<_, HealthcheckResponse>( - paths, - engine, - service_id, - EngineMethod::Healthcheck, - &HealthcheckRequest { - service_id: service_id.to_owned(), - }, - ) -} - -pub(crate) fn healthcheck_response_ready(response: &HealthcheckResponse) -> bool { - response.status == "ready" && response.model_loaded -} - -pub(crate) fn healthcheck_response_recoverable(response: &HealthcheckResponse) -> bool { - matches!( - response.status.as_str(), - "failed" | "unreachable" | "exited" - ) -} - -/// Re-thread the endpoint key file (public bind only) onto an engine child's -/// environment, mirroring the initial `rocm serve` spawn. A loopback service -/// with no stored key leaves the command's environment untouched, matching the -/// unauthenticated default. -/// -/// Returns whether a key was applied. Callers that spawn a *listener* must -/// check it against the service's host — a public bind with no key would come -/// up anonymous (see [`ensure_public_service_has_endpoint_key`]). Callers that -/// only make a stdio plugin call can ignore it. -/// -/// The file must hold a *usable* key, not merely exist: the engine adapters -/// resolve it with `endpoint_api_key_from_file` and enforce nothing when that -/// yields `None`, so treating an empty or malformed file as "protected" would -/// let the guard pass while the endpoint served anonymously. -#[must_use] -pub(crate) fn apply_endpoint_key_env( - command: &mut ProcessCommand, - paths: &AppPaths, - service_id: &str, -) -> bool { - if let Some(key_file) = rocm_engine_protocol::endpoint_key_file_if_present(paths, service_id) - .and_then(|path| rocm_engine_protocol::endpoint_api_key_file_if_valid(&path)) - { - command.env(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file); - return true; - } - false -} - -/// Refuse to respawn a recorded service on a public host without its endpoint -/// API key, so daemon recovery cannot reopen a protected endpoint anonymously. -/// -/// Mirrors the guard of the same name in `rocm`; the shared -/// [`rocm_engine_protocol::is_public_bind_host`] keeps the two classifications -/// identical for a given `ManagedServiceRecord::host`. -pub(crate) fn ensure_public_service_has_endpoint_key( - host: &str, - key_present: bool, - requires_api_key: bool, -) -> Result<()> { - // The bind address is not the whole story. A service bound to loopback is - // only private until something republishes the port, and a tailnet publish - // outlives both the process and this daemon. The requirement is recorded on - // the service precisely so recovery can honour it without re-deriving it - // from an address that no longer answers the question. - if requires_api_key && !key_present { - bail!( - "refusing to recover a service that was launched with an endpoint API key but no \ - longer has one: it would come back up without authentication, and something \ - outside this machine may still be publishing its port. Relaunch it with \ - `rocm serve --require-api-key` to issue a new key." - ); - } - if rocm_engine_protocol::is_public_bind_host(host) && !key_present { - bail!( - "refusing to respawn a service bound to the public host `{host}` without an endpoint \ - API key: it would come back up without authentication. Relaunch it with \ - `rocm serve --host {host} --allow-public-bind` to issue a new key." - ); - } - Ok(()) -} - -pub(crate) fn engine_request( - paths: &AppPaths, - engine: &str, - service_id: &str, - method: EngineMethod, - request: &T, -) -> Result -where - T: Serialize, - R: DeserializeOwned, -{ - let envelope = EngineRequestEnvelope { - method, - payload: serde_json::to_value(request).context("failed to serialize engine request")?, - }; - let engine_binary = - std::env::current_exe().context("failed to resolve current rocm executable path")?; - let mut command = ProcessCommand::new(&engine_binary); - command - .arg("__engine-stdio") - .arg(engine) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - // A protected public endpoint enforces its key on every request, including - // this stdio-transported healthcheck — without the carrier the engine - // adapter's HTTP probe is unauthenticated and the endpoint looks - // unreachable, which would misclassify a healthy protected service as - // recoverable. - // - // No host to check here: this is a stdio plugin call, not a listener, so a - // missing key only weakens this probe rather than opening a port. - let _ = apply_endpoint_key_env(&mut command, paths, service_id); - let mut child = command.spawn().with_context(|| { - format!( - "failed to spawn engine stdio process {}", - engine_binary.display() - ) - })?; - - { - let stdin = child - .stdin - .as_mut() - .context("engine stdio child did not expose stdin")?; - serde_json::to_writer(&mut *stdin, &envelope).context("failed to write engine request")?; - stdin.write_all(b"\n")?; - } - - let output = child - .wait_with_output() - .context("failed waiting for engine stdio response")?; - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); - if stderr.is_empty() { - bail!("engine stdio process exited with status {}", output.status); - } - bail!( - "engine stdio process exited with status {}: {}", - output.status, - stderr - ); - } - let envelope: EngineResponseEnvelope = - serde_json::from_slice(&output.stdout).context("failed to parse engine response")?; - if !envelope.ok { - let detail = envelope.error.map_or_else( - || "unknown engine error".to_owned(), - |error| format!("{}: {}", error.code, error.message), - ); - bail!(detail); - } - let data = envelope - .data - .context("engine response did not include data")?; - serde_json::from_value(data).context("failed to deserialize engine response data") -} - -pub(crate) fn wait_for_port(host: &str, port: u16, timeout: Duration) -> bool { - let address: SocketAddr = match format_host_port(host, port).parse() { - Ok(value) => value, - Err(_) => return false, - }; - - let start = std::time::Instant::now(); - while start.elapsed() < timeout { - if TcpStream::connect_timeout(&address, Duration::from_millis(200)).is_ok() { - return true; - } - thread::sleep(Duration::from_millis(200)); - } - false +pub(crate) fn payload_string(payload: &Value, key: &str) -> Option { + payload + .get(key) + .and_then(Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_owned) } #[cfg(test)] @@ -433,179 +83,6 @@ mod tests { use rocm_core::engine_plugin_dirs; use std::fs; - #[test] - fn recovery_refuses_a_service_that_lost_a_key_it_was_launched_with() { - // The daemon keeps its own copy of this guard, and it only knew about - // public binds. A loopback service that something republishes — a - // tailnet publish outlives this daemon, let alone the process — would - // be recovered without authentication, and the rebuilt record would - // then disarm `rocm services restart` too. - for host in ["127.0.0.1", "localhost", "::1"] { - let error = ensure_public_service_has_endpoint_key(host, false, true) - .expect_err("a service launched with a key must not be recovered without one"); - assert!( - format!("{error:#}").contains("without authentication"), - "{error:#}" - ); - } - // With the key still present, recovery proceeds. - ensure_public_service_has_endpoint_key("127.0.0.1", true, true).unwrap(); - // And a service that never had one is untouched. - ensure_public_service_has_endpoint_key("127.0.0.1", false, false).unwrap(); - } - - #[test] - fn recovery_respawn_fails_closed_for_a_public_service_without_a_key() { - // Daemon recovery re-execs `rocmd supervise` for the recorded host. With - // the key gone the child would listen on that public host anonymously, - // so the spawn must be refused instead. - let error = ensure_public_service_has_endpoint_key("0.0.0.0", false, false).unwrap_err(); - assert!( - error.to_string().contains("without authentication"), - "{error:#}" - ); - - ensure_public_service_has_endpoint_key("0.0.0.0", true, false).unwrap(); - for host in ["127.0.0.1", "localhost", "::1"] { - ensure_public_service_has_endpoint_key(host, false, false) - .unwrap_or_else(|error| panic!("{host} must not require a key: {error:#}")); - } - } - - #[test] - fn apply_endpoint_key_env_sets_var_only_when_key_file_present() { - let (_root, paths) = temp_app_paths("apply-endpoint-key-env"); - let service_id = "svc-endpoint-key-env"; - - // Loopback service: no key file has ever been written, so the child's - // environment must be left untouched. - let mut command = ProcessCommand::new("true"); - assert!(!apply_endpoint_key_env(&mut command, &paths, service_id)); - assert!( - command - .get_envs() - .all(|(key, _)| key != rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV), - "loopback service must not receive the endpoint key env var" - ); - - // Public service: once a key file exists at the deterministic path, - // it must be threaded onto the child so the engine's HTTP probe - // authenticates. - let key_path = rocm_engine_protocol::endpoint_key_file_path(&paths, service_id); - fs::create_dir_all(paths.services_dir()).unwrap(); - fs::write(&key_path, "secret-key").unwrap(); - let mut command = ProcessCommand::new("true"); - assert!(apply_endpoint_key_env(&mut command, &paths, service_id)); - let env_value = command - .get_envs() - .find_map(|(key, value)| { - (key == rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV) - .then_some(value) - .flatten() - }) - .expect("endpoint key env var must be set once the key file exists"); - assert_eq!(env_value, key_path.as_os_str()); - - // An empty (or otherwise unusable) key file is not a key: the engine - // adapters would enforce nothing, so reporting `true` here would let the - // respawn guard pass and still open an anonymous public listener. - fs::write(&key_path, " \n").unwrap(); - let mut command = ProcessCommand::new("true"); - assert!(!apply_endpoint_key_env(&mut command, &paths, service_id)); - assert!( - command - .get_envs() - .all(|(key, _)| key != rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV), - "an unusable key file must not be threaded onto the child" - ); - } - - #[test] - fn healthcheck_readiness_requires_ready_loaded_model() { - let ready = HealthcheckResponse { - status: "ready".to_owned(), - model_loaded: true, - device: "cuda".to_owned(), - uptime_sec: 1, - queue_depth: 0, - last_error: None, - tokens_per_sec: None, - }; - assert!(healthcheck_response_ready(&ready)); - - let mut loading = ready.clone(); - loading.status = "loading_model".to_owned(); - assert!(!healthcheck_response_ready(&loading)); - - let mut unloaded = ready; - unloaded.model_loaded = false; - assert!(!healthcheck_response_ready(&unloaded)); - } - - #[test] - fn healthcheck_recoverability_tracks_failed_endpoint_state() { - let mut response = HealthcheckResponse { - status: "ready".to_owned(), - model_loaded: true, - device: "cuda".to_owned(), - uptime_sec: 1, - queue_depth: 0, - last_error: None, - tokens_per_sec: None, - }; - assert!(!healthcheck_response_recoverable(&response)); - - response.status = "unreachable".to_owned(); - assert!(healthcheck_response_recoverable(&response)); - - response.status = "failed".to_owned(); - assert!(healthcheck_response_recoverable(&response)); - - response.status = "loading_model".to_owned(); - assert!(!healthcheck_response_recoverable(&response)); - - // The status the engines report for a model that is listed but has not - // yet served an inference request. Restarting it would kill a model - // mid-load and start the wait over. - response.status = "loading".to_owned(); - assert!(!healthcheck_response_recoverable(&response)); - } - - #[test] - fn healthcheck_readiness_withheld_while_the_model_only_lists() { - // What an engine reports once `/v1/models` answers but inference has not: - // not ready, so `rocm serve` keeps waiting instead of handing the caller - // an endpoint that will hang on its first request. - let listing_only = HealthcheckResponse { - status: "loading".to_owned(), - model_loaded: false, - device: "unknown".to_owned(), - uptime_sec: 1, - queue_depth: 0, - last_error: None, - tokens_per_sec: None, - }; - assert!(!healthcheck_response_ready(&listing_only)); - } - - #[test] - fn gpu_snapshot_respects_disabled_telemetry_policy() { - let mut config = RocmCliConfig::default(); - config.telemetry.mode = rocm_core::TELEMETRY_MODE_OFF.to_owned(); - - let snapshot = gather_gpu_snapshot_for_config(&config); - - assert!(!snapshot.amd_smi_available); - assert!(snapshot.static_snapshot.is_none()); - assert!(snapshot.monitor_snapshot.is_none()); - assert!( - snapshot - .note - .as_deref() - .is_some_and(|note| note.contains("disabled by rocm-cli config")) - ); - } - #[test] fn engine_plugin_discovery_finds_runtime_binary() -> Result<()> { let (root, paths) = temp_app_paths("engine-plugin"); diff --git a/apps/rocmd/src/daemon.rs b/apps/rocmd/src/daemon.rs new file mode 100644 index 0000000000..9369b8f487 --- /dev/null +++ b/apps/rocmd/src/daemon.rs @@ -0,0 +1,305 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +use anyhow::{Context, Result, bail}; +use rocm_core::{ + AppPaths, AutomationRuntimeState, RocmCliConfig, WatcherRuntimeSnapshot, builtin_watchers, + unix_time_millis, +}; +use std::io::{self, Write}; +use std::time::Duration; +use tokio::time::{self, MissedTickBehavior}; + +const WATCHER_TICK_INTERVAL: Duration = Duration::from_secs(5); + +pub(crate) async fn run_daemon( + paths: &AppPaths, + automations_enabled: bool, + local_webhook_port: Option, +) -> Result<()> { + if local_webhook_port.is_some() && !automations_enabled { + bail!("local webhook source requires --automations-enabled"); + } + + let config = RocmCliConfig::load(paths)?; + let mut state = build_runtime_state(&config, automations_enabled); + let local_webhook = if let Some(port) = local_webhook_port { + Some(crate::webhook::start_local_webhook_source(port).await?) + } else { + None + }; + let (local_webhook_endpoint, mut local_webhook_receiver, local_webhook_task) = + match local_webhook { + Some(source) => ( + Some(source.endpoint), + Some(source.receiver), + Some(source.task), + ), + None => (None, None, None), + }; + state.local_webhook_endpoint = local_webhook_endpoint.clone(); + + println!("rocmd run"); + println!(" automations enabled: {automations_enabled}"); + println!( + " lifecycle: {}", + if automations_enabled { + "persistent" + } else { + "on-demand" + } + ); + println!(" config: {}", paths.config_path().display()); + println!(" state: {}", paths.automation_state_path().display()); + println!( + " local_webhook_endpoint: {}", + local_webhook_endpoint.as_deref().unwrap_or("disabled") + ); + let enabled_count = state + .active_watchers + .iter() + .filter(|watcher| watcher.enabled) + .count(); + println!(" enabled watchers: {enabled_count}"); + // This banner is the foreground-loop readiness contract used by callers and + // integration tests. Flush it before any persistent work so piped stdout on + // Windows cannot retain the line in a userspace buffer indefinitely. + io::stdout() + .flush() + .context("failed to flush rocmd run banner")?; + + if !automations_enabled { + println!( + " note: rerun with --automations-enabled to keep rocmd alive for watcher execution" + ); + return Ok(()); + } + + paths.ensure()?; + state.write(paths)?; + crate::persistence::record_event( + paths, + &mut state, + "rocmd", + "info", + "daemon_start", + "rocmd automation supervisor started", + None, + )?; + state.write(paths)?; + + crate::watchers::evaluate_watchers(paths, &config, &mut state)?; + state.last_tick_unix_ms = unix_time_millis(); + state.write(paths)?; + + let shutdown = shutdown_signal(); + tokio::pin!(shutdown); + + let mut ticker = time::interval(WATCHER_TICK_INTERVAL); + ticker.set_missed_tick_behavior(MissedTickBehavior::Delay); + + loop { + tokio::select! { + _ = ticker.tick() => { + let config = RocmCliConfig::load(paths)?; + crate::watchers::reconcile_watcher_snapshots(&config, &mut state); + crate::watchers::evaluate_watchers(paths, &config, &mut state)?; + state.last_tick_unix_ms = unix_time_millis(); + state.write(paths)?; + } + event = crate::webhook::receive_local_webhook_event(&mut local_webhook_receiver) => { + if let Some(event) = event { + let config = RocmCliConfig::load(paths)?; + crate::watchers::reconcile_watcher_snapshots(&config, &mut state); + crate::persistence::record_event( + paths, + &mut state, + "rocmd", + "info", + "local_webhook_event", + &format!( + "received local webhook event kind={} watcher_hint={}; dispatching through existing watcher policy; webhook payload grants no new action", + event.kind, + event.watcher_hint.as_deref().unwrap_or("") + ), + event.service_id.clone(), + )?; + if let Err(error) = + crate::watchers::evaluate_watchers_for_events(paths, &config, &mut state, &[event]) + { + crate::persistence::record_event( + paths, + &mut state, + "rocmd", + "error", + "local_webhook_dispatch_failed", + &format!( + "local webhook event could not be dispatched through watcher policy: {error}" + ), + None, + )?; + } + state.last_tick_unix_ms = unix_time_millis(); + state.write(paths)?; + } else { + local_webhook_receiver = None; + state.local_webhook_endpoint = None; + crate::persistence::record_event( + paths, + &mut state, + "rocmd", + "warn", + "local_webhook_stopped", + "local webhook source stopped; automation daemon continues without webhook ingestion", + None, + )?; + state.write(paths)?; + } + } + () = &mut shutdown => { + break; + } + } + } + + state.running = false; + state.last_tick_unix_ms = unix_time_millis(); + state.local_webhook_endpoint = None; + crate::persistence::record_event( + paths, + &mut state, + "rocmd", + "info", + "daemon_stop", + "rocmd automation supervisor stopped", + None, + )?; + state.write(paths)?; + if let Some(task) = local_webhook_task { + task.abort(); + } + Ok(()) +} + +pub(crate) fn print_status(paths: &AppPaths) -> Result<()> { + let config = RocmCliConfig::load(paths).unwrap_or_default(); + println!("rocmd status"); + println!(" config dir: {}", paths.config_dir.display()); + println!(" data dir: {}", paths.data_dir.display()); + println!(" policy: on-demand by default, persistent only with background features"); + println!( + " automations desired: {}", + if config.automation_daemon_enabled() { + "enabled" + } else { + "disabled" + } + ); + match AutomationRuntimeState::load(paths)? { + Some(state) => { + println!( + " automations runtime: {} pid={} last_tick_unix_ms={}", + if state.running { "running" } else { "stopped" }, + state.daemon_pid, + state.last_tick_unix_ms + ); + println!( + " local_webhook_endpoint: {}", + state + .local_webhook_endpoint + .as_deref() + .unwrap_or("disabled") + ); + for watcher in state + .active_watchers + .into_iter() + .filter(|watcher| watcher.enabled) + { + println!( + " watcher {} mode={} last_event={}", + watcher.id, + watcher.mode.as_str(), + watcher.last_event.as_deref().unwrap_or("") + ); + } + } + None => println!(" automations runtime: inactive"), + } + println!( + " automation events: {}", + paths.automation_events_path().display() + ); + println!(" audit events: {}", paths.audit_events_path().display()); + + let records = crate::persistence::load_managed_services(paths)?; + if records.is_empty() { + println!(" services: none"); + return Ok(()); + } + + for record in records { + println!( + " service {} engine={} status={} endpoint={}", + record.service_id, record.engine, record.status, record.endpoint_url + ); + } + + Ok(()) +} + +fn build_runtime_state( + config: &RocmCliConfig, + automations_enabled: bool, +) -> AutomationRuntimeState { + let now = unix_time_millis(); + let active_watchers = builtin_watchers() + .iter() + .map(|watcher| WatcherRuntimeSnapshot { + id: watcher.id.to_owned(), + enabled: config.watcher_enabled(watcher), + mode: config.effective_watcher_mode(watcher), + summary: watcher.summary.to_owned(), + last_event: None, + last_event_unix_ms: None, + }) + .collect(); + AutomationRuntimeState { + running: automations_enabled, + automations_enabled, + daemon_pid: std::process::id(), + started_at_unix_ms: now, + last_tick_unix_ms: now, + local_webhook_endpoint: None, + active_watchers, + } +} + +#[cfg(unix)] +async fn shutdown_signal() { + let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) + .expect("failed to register SIGTERM handler"); + tokio::select! { + _ = tokio::signal::ctrl_c() => {} + _ = term.recv() => {} + } +} + +#[cfg(not(unix))] +async fn shutdown_signal() { + let _ = tokio::signal::ctrl_c().await; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::temp_app_paths; + + #[tokio::test] + async fn local_webhook_requires_enabled_automation_loop() { + let (_root, paths) = temp_app_paths("local-webhook-requires-loop"); + let error = run_daemon(&paths, false, Some(0)).await.unwrap_err(); + + assert!(error.to_string().contains("requires --automations-enabled")); + } +} diff --git a/apps/rocmd/src/engine_ipc.rs b/apps/rocmd/src/engine_ipc.rs new file mode 100644 index 0000000000..d61397ac7d --- /dev/null +++ b/apps/rocmd/src/engine_ipc.rs @@ -0,0 +1,344 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +use anyhow::{Context, Result, bail}; +use rocm_core::AppPaths; +use rocm_engine_protocol::{ + EngineMethod, EngineRequestEnvelope, EngineResponseEnvelope, HealthcheckRequest, + HealthcheckResponse, +}; +use serde::Serialize; +use serde::de::DeserializeOwned; +use std::io::Write; +use std::process::{Command as ProcessCommand, Stdio}; + +pub(crate) fn engine_healthcheck_response( + paths: &AppPaths, + engine: &str, + service_id: &str, +) -> Result { + engine_request::<_, HealthcheckResponse>( + paths, + engine, + service_id, + EngineMethod::Healthcheck, + &HealthcheckRequest { + service_id: service_id.to_owned(), + }, + ) +} + +pub(crate) fn healthcheck_response_ready(response: &HealthcheckResponse) -> bool { + response.status == "ready" && response.model_loaded +} + +pub(crate) fn healthcheck_response_recoverable(response: &HealthcheckResponse) -> bool { + matches!( + response.status.as_str(), + "failed" | "unreachable" | "exited" + ) +} + +/// Re-thread the endpoint key file (public bind only) onto an engine child's +/// environment, mirroring the initial `rocm serve` spawn. A loopback service +/// with no stored key leaves the command's environment untouched, matching the +/// unauthenticated default. +/// +/// Returns whether a key was applied. Callers that spawn a *listener* must +/// check it against the service's host — a public bind with no key would come +/// up anonymous (see [`ensure_public_service_has_endpoint_key`]). Callers that +/// only make a stdio plugin call can ignore it. +/// +/// The file must hold a *usable* key, not merely exist: the engine adapters +/// resolve it with `endpoint_api_key_from_file` and enforce nothing when that +/// yields `None`, so treating an empty or malformed file as "protected" would +/// let the guard pass while the endpoint served anonymously. +#[must_use] +pub(crate) fn apply_endpoint_key_env( + command: &mut ProcessCommand, + paths: &AppPaths, + service_id: &str, +) -> bool { + if let Some(key_file) = rocm_engine_protocol::endpoint_key_file_if_present(paths, service_id) + .and_then(|path| rocm_engine_protocol::endpoint_api_key_file_if_valid(&path)) + { + command.env(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file); + return true; + } + false +} + +/// Refuse to respawn a recorded service on a public host without its endpoint +/// API key, so daemon recovery cannot reopen a protected endpoint anonymously. +/// +/// Mirrors the guard of the same name in `rocm`; the shared +/// [`rocm_engine_protocol::is_public_bind_host`] keeps the two classifications +/// identical for a given `ManagedServiceRecord::host`. +pub(crate) fn ensure_public_service_has_endpoint_key( + host: &str, + key_present: bool, + requires_api_key: bool, +) -> Result<()> { + // The bind address is not the whole story. A service bound to loopback is + // only private until something republishes the port, and a tailnet publish + // outlives both the process and this daemon. The requirement is recorded on + // the service precisely so recovery can honour it without re-deriving it + // from an address that no longer answers the question. + if requires_api_key && !key_present { + bail!( + "refusing to recover a service that was launched with an endpoint API key but no \ + longer has one: it would come back up without authentication, and something \ + outside this machine may still be publishing its port. Relaunch it with \ + `rocm serve --require-api-key` to issue a new key." + ); + } + if rocm_engine_protocol::is_public_bind_host(host) && !key_present { + bail!( + "refusing to respawn a service bound to the public host `{host}` without an endpoint \ + API key: it would come back up without authentication. Relaunch it with \ + `rocm serve --host {host} --allow-public-bind` to issue a new key." + ); + } + Ok(()) +} + +pub(crate) fn engine_request( + paths: &AppPaths, + engine: &str, + service_id: &str, + method: EngineMethod, + request: &T, +) -> Result +where + T: Serialize, + R: DeserializeOwned, +{ + let envelope = EngineRequestEnvelope { + method, + payload: serde_json::to_value(request).context("failed to serialize engine request")?, + }; + let engine_binary = + std::env::current_exe().context("failed to resolve current rocm executable path")?; + let mut command = ProcessCommand::new(&engine_binary); + command + .arg("__engine-stdio") + .arg(engine) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + // A protected public endpoint enforces its key on every request, including + // this stdio-transported healthcheck — without the carrier the engine + // adapter's HTTP probe is unauthenticated and the endpoint looks + // unreachable, which would misclassify a healthy protected service as + // recoverable. + // + // No host to check here: this is a stdio plugin call, not a listener, so a + // missing key only weakens this probe rather than opening a port. + let _ = apply_endpoint_key_env(&mut command, paths, service_id); + let mut child = command.spawn().with_context(|| { + format!( + "failed to spawn engine stdio process {}", + engine_binary.display() + ) + })?; + + { + let stdin = child + .stdin + .as_mut() + .context("engine stdio child did not expose stdin")?; + serde_json::to_writer(&mut *stdin, &envelope).context("failed to write engine request")?; + stdin.write_all(b"\n")?; + } + + let output = child + .wait_with_output() + .context("failed waiting for engine stdio response")?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); + if stderr.is_empty() { + bail!("engine stdio process exited with status {}", output.status); + } + bail!( + "engine stdio process exited with status {}: {}", + output.status, + stderr + ); + } + let envelope: EngineResponseEnvelope = + serde_json::from_slice(&output.stdout).context("failed to parse engine response")?; + if !envelope.ok { + let detail = envelope.error.map_or_else( + || "unknown engine error".to_owned(), + |error| format!("{}: {}", error.code, error.message), + ); + bail!(detail); + } + let data = envelope + .data + .context("engine response did not include data")?; + serde_json::from_value(data).context("failed to deserialize engine response data") +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::temp_app_paths; + use std::fs; + + #[test] + fn recovery_refuses_a_service_that_lost_a_key_it_was_launched_with() { + // The daemon keeps its own copy of this guard, and it only knew about + // public binds. A loopback service that something republishes — a + // tailnet publish outlives this daemon, let alone the process — would + // be recovered without authentication, and the rebuilt record would + // then disarm `rocm services restart` too. + for host in ["127.0.0.1", "localhost", "::1"] { + let error = ensure_public_service_has_endpoint_key(host, false, true) + .expect_err("a service launched with a key must not be recovered without one"); + assert!( + format!("{error:#}").contains("without authentication"), + "{error:#}" + ); + } + // With the key still present, recovery proceeds. + ensure_public_service_has_endpoint_key("127.0.0.1", true, true).unwrap(); + // And a service that never had one is untouched. + ensure_public_service_has_endpoint_key("127.0.0.1", false, false).unwrap(); + } + + #[test] + fn recovery_respawn_fails_closed_for_a_public_service_without_a_key() { + // Daemon recovery re-execs `rocmd supervise` for the recorded host. With + // the key gone the child would listen on that public host anonymously, + // so the spawn must be refused instead. + let error = ensure_public_service_has_endpoint_key("0.0.0.0", false, false).unwrap_err(); + assert!( + error.to_string().contains("without authentication"), + "{error:#}" + ); + + ensure_public_service_has_endpoint_key("0.0.0.0", true, false).unwrap(); + for host in ["127.0.0.1", "localhost", "::1"] { + ensure_public_service_has_endpoint_key(host, false, false) + .unwrap_or_else(|error| panic!("{host} must not require a key: {error:#}")); + } + } + + #[test] + fn apply_endpoint_key_env_sets_var_only_when_key_file_present() { + let (_root, paths) = temp_app_paths("apply-endpoint-key-env"); + let service_id = "svc-endpoint-key-env"; + + // Loopback service: no key file has ever been written, so the child's + // environment must be left untouched. + let mut command = ProcessCommand::new("true"); + assert!(!apply_endpoint_key_env(&mut command, &paths, service_id)); + assert!( + command + .get_envs() + .all(|(key, _)| key != rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV), + "loopback service must not receive the endpoint key env var" + ); + + // Public service: once a key file exists at the deterministic path, + // it must be threaded onto the child so the engine's HTTP probe + // authenticates. + let key_path = rocm_engine_protocol::endpoint_key_file_path(&paths, service_id); + fs::create_dir_all(paths.services_dir()).unwrap(); + fs::write(&key_path, "secret-key").unwrap(); + let mut command = ProcessCommand::new("true"); + assert!(apply_endpoint_key_env(&mut command, &paths, service_id)); + let env_value = command + .get_envs() + .find_map(|(key, value)| { + (key == rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV) + .then_some(value) + .flatten() + }) + .expect("endpoint key env var must be set once the key file exists"); + assert_eq!(env_value, key_path.as_os_str()); + + // An empty (or otherwise unusable) key file is not a key: the engine + // adapters would enforce nothing, so reporting `true` here would let the + // respawn guard pass and still open an anonymous public listener. + fs::write(&key_path, " \n").unwrap(); + let mut command = ProcessCommand::new("true"); + assert!(!apply_endpoint_key_env(&mut command, &paths, service_id)); + assert!( + command + .get_envs() + .all(|(key, _)| key != rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV), + "an unusable key file must not be threaded onto the child" + ); + } + + #[test] + fn healthcheck_readiness_requires_ready_loaded_model() { + let ready = HealthcheckResponse { + status: "ready".to_owned(), + model_loaded: true, + device: "cuda".to_owned(), + uptime_sec: 1, + queue_depth: 0, + last_error: None, + tokens_per_sec: None, + }; + assert!(healthcheck_response_ready(&ready)); + + let mut loading = ready.clone(); + loading.status = "loading_model".to_owned(); + assert!(!healthcheck_response_ready(&loading)); + + let mut unloaded = ready; + unloaded.model_loaded = false; + assert!(!healthcheck_response_ready(&unloaded)); + } + + #[test] + fn healthcheck_recoverability_tracks_failed_endpoint_state() { + let mut response = HealthcheckResponse { + status: "ready".to_owned(), + model_loaded: true, + device: "cuda".to_owned(), + uptime_sec: 1, + queue_depth: 0, + last_error: None, + tokens_per_sec: None, + }; + assert!(!healthcheck_response_recoverable(&response)); + + response.status = "unreachable".to_owned(); + assert!(healthcheck_response_recoverable(&response)); + + response.status = "failed".to_owned(); + assert!(healthcheck_response_recoverable(&response)); + + response.status = "loading_model".to_owned(); + assert!(!healthcheck_response_recoverable(&response)); + + // The status the engines report for a model that is listed but has not + // yet served an inference request. Restarting it would kill a model + // mid-load and start the wait over. + response.status = "loading".to_owned(); + assert!(!healthcheck_response_recoverable(&response)); + } + + #[test] + fn healthcheck_readiness_withheld_while_the_model_only_lists() { + // What an engine reports once `/v1/models` answers but inference has not: + // not ready, so `rocm serve` keeps waiting instead of handing the caller + // an endpoint that will hang on its first request. + let listing_only = HealthcheckResponse { + status: "loading".to_owned(), + model_loaded: false, + device: "unknown".to_owned(), + uptime_sec: 1, + queue_depth: 0, + last_error: None, + tokens_per_sec: None, + }; + assert!(!healthcheck_response_ready(&listing_only)); + } +} diff --git a/apps/rocmd/src/host_snapshot.rs b/apps/rocmd/src/host_snapshot.rs new file mode 100644 index 0000000000..93ce12edbc --- /dev/null +++ b/apps/rocmd/src/host_snapshot.rs @@ -0,0 +1,193 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +use anyhow::{Context, Result}; +use rocm_core::{ + AppPaths, AutomationRuntimeState, CodexBridgeEngine, CodexBridgeGpuSnapshot, + CodexBridgeSnapshot, ExamineSummary, RocmCliConfig, default_engine_for_platform, + load_recent_automation_events, resolve_amd_smi_binary, unix_time_millis, +}; +use serde_json::Value; +use std::process::Command as ProcessCommand; +use std::time::Duration; + +const AMD_SMI_PROBE_TIMEOUT: Duration = Duration::from_secs(2); + +pub(crate) fn build_bridge_snapshot(paths: &AppPaths) -> Result { + let config = RocmCliConfig::load(paths).unwrap_or_default(); + Ok(CodexBridgeSnapshot { + protocol: "rocmd-codex-bridge-v0".to_owned(), + generated_at_unix_ms: unix_time_millis(), + examine: ExamineSummary::gather()?, + gpu: gather_gpu_snapshot_for_config(&config), + config, + automation_runtime: AutomationRuntimeState::load(paths)?, + recent_automation_events: load_recent_automation_events(paths, 32)?, + engines: bridge_engine_inventory(), + services: crate::persistence::load_managed_services(paths)?, + }) +} + +fn gather_gpu_snapshot() -> CodexBridgeGpuSnapshot { + let static_snapshot = match capture_amd_smi_json(&["static", "-a", "-g", "all", "--json"]) { + Ok(value) => Some(value), + Err(error) => { + return CodexBridgeGpuSnapshot { + // `{error:#}` rather than `to_string()`: the outermost context + // alone says "failed to launch amd-smi ...", which names the + // wrong thing when what actually happened was a timeout. Pinned + // by `examine-25`, which asserts that the rendered note names + // the timeout and not just the launch. + // + // Every failure `run_with_timeout` raises carries its cause the + // same way, since all of them arrive under that one context: + // the timeout, the failure to spawn, and the poll and reap + // failures of its wait loop. Spawn is the case on a host where + // none of the paths `resolve_amd_smi_binary` searches holds an + // `amd-smi`, down to the bare name on `PATH`; the cause is then + // the OS's own "not found" text, which is not the same string + // on Linux and Windows, and `rocmd` runs this probe on both -- + // so none of it is quoted here, and no test pins it. A host + // with no GPU but an amd-smi installed is a different case: it + // spawns and fails at a non-zero exit. (The non-zero-exit and + // parse failures below are their own outermost context and + // already read correctly.) + // + // The note is the only account of any of this that reaches the + // bridge snapshot and the automation watcher, so it carries the + // causes too -- bounded, because it is persisted; see + // `run_with_timeout`'s `MAX_QUOTED_OUTPUT_CHARS`. + amd_smi_available: false, + static_snapshot: None, + monitor_snapshot: None, + note: Some(format!("{error:#}")), + }; + } + }; + + let monitor_snapshot = match capture_amd_smi_json(&[ + "monitor", "-p", "-t", "-u", "-m", "-v", "-g", "all", "--json", + ]) { + Ok(value) => Some(value), + Err(error) => { + return CodexBridgeGpuSnapshot { + // Same reasoning as the `static` probe above, but a different + // test: examine-25's stalled `static` probe returns before this + // branch is ever reached, so reverting this `{error:#}` is + // caught by `examine-26` alone. Pinned by `examine-26`. + amd_smi_available: true, + static_snapshot, + monitor_snapshot: None, + note: Some(format!("{error:#}")), + }; + } + }; + + CodexBridgeGpuSnapshot { + amd_smi_available: true, + static_snapshot, + monitor_snapshot, + note: None, + } +} + +pub(crate) fn gather_gpu_snapshot_for_config(config: &RocmCliConfig) -> CodexBridgeGpuSnapshot { + if config.telemetry.local_inspection_enabled() { + gather_gpu_snapshot() + } else { + CodexBridgeGpuSnapshot { + amd_smi_available: false, + static_snapshot: None, + monitor_snapshot: None, + note: Some( + "gpu telemetry is disabled by rocm-cli config; no external reporting is implemented" + .to_owned(), + ), + } + } +} + +fn capture_amd_smi_json(args: &[&str]) -> Result { + let amd_smi_binary = resolve_amd_smi_binary(); + let mut command = ProcessCommand::new(&amd_smi_binary); + // No stdio here: `run_with_timeout` sets null stdin and piped stdout/stderr + // itself, since capturing both is its contract. + command.args(args); + let output = rocm_core::process::run_with_timeout(command, AMD_SMI_PROBE_TIMEOUT, "process") + .with_context(|| format!("failed to launch amd-smi {}", args.join(" ")))?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr).trim().to_owned(); + let stdout = String::from_utf8_lossy(&output.stdout).trim().to_owned(); + anyhow::bail!( + "amd-smi {} failed: {}", + args.join(" "), + if !stderr.is_empty() { + stderr + } else if !stdout.is_empty() { + stdout + } else { + format!("exit status {}", output.status) + } + ); + } + + serde_json::from_slice(&output.stdout) + .with_context(|| format!("failed to parse amd-smi {} json", args.join(" "))) +} + +pub(crate) fn bridge_engine_inventory() -> Vec { + let default_engine = default_engine_for_platform(); + let current_exe = std::env::current_exe().ok(); + rocmd_engine_inventory() + .iter() + .map(|(id, summary)| CodexBridgeEngine { + id: (*id).to_owned(), + summary: (*summary).to_owned(), + default_for_platform: *id == default_engine, + installed_binary: true, + binary_path: current_exe.as_ref().map(|path| path.display().to_string()), + }) + .collect() +} + +const fn rocmd_engine_inventory() -> &'static [(&'static str, &'static str)] { + &[ + ( + "lemonade", + "embedded Lemonade server with ROCm llama.cpp backend", + ), + ( + "vllm", + "Linux/WSL ROCm GPU serving engine through external vLLM", + ), + ( + "aim", + "AMD Enterprise AI pre-built Docker serving containers", + ), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn gpu_snapshot_respects_disabled_telemetry_policy() { + let mut config = RocmCliConfig::default(); + config.telemetry.mode = rocm_core::TELEMETRY_MODE_OFF.to_owned(); + + let snapshot = gather_gpu_snapshot_for_config(&config); + + assert!(!snapshot.amd_smi_available); + assert!(snapshot.static_snapshot.is_none()); + assert!(snapshot.monitor_snapshot.is_none()); + assert!( + snapshot + .note + .as_deref() + .is_some_and(|note| note.contains("disabled by rocm-cli config")) + ); + } +} diff --git a/apps/rocmd/src/lib.rs b/apps/rocmd/src/lib.rs index 3333461903..b77042e705 100644 --- a/apps/rocmd/src/lib.rs +++ b/apps/rocmd/src/lib.rs @@ -6,118 +6,18 @@ mod cli; mod common; +mod daemon; +mod engine_ipc; +mod host_snapshot; mod mcp; mod persistence; mod sandbox; +mod sandbox_tool; mod service; +mod supervise; #[cfg(test)] mod test_support; mod watchers; mod webhook; pub use cli::{parse_args, run_bin_cli, run_from_args}; - -use std::time::Duration; - -const WATCHER_TICK_INTERVAL: Duration = Duration::from_secs(5); -const ARTIFACT_PREFETCH_TIMEOUT: Duration = Duration::from_mins(10); - -#[cfg(test)] -mod tests { - use super::*; - use crate::service::sandbox_stop_server_value; - use crate::test_support::{ - assert_unconfirmed_stop_kept_the_service, seed_keyed_service, stop_with_outcome, - temp_app_paths, - }; - use crate::watchers::load_service_record; - use anyhow::Result; - use rocm_core::ManagedServiceRecord; - use serde_json::Value; - use std::fs; - - #[test] - fn sandbox_tool_stop_server_updates_manifest_and_skips_current_pid() -> Result<()> { - let (root, paths) = temp_app_paths("sandbox-stop-current-pid"); - paths.ensure()?; - let current_pid = std::process::id(); - let mut record = ManagedServiceRecord::new( - &paths, - "svc-current", - "vllm", - "qwen", - "Qwen/Qwen3.5", - "127.0.0.1", - 11435, - "managed", - current_pid, - None, - None, - None, - ); - record.engine_pid = Some(current_pid); - record.status = "ready".to_owned(); - record.write()?; - - let value = sandbox::run_sandbox_tool( - &paths, - cli::SandboxToolArg::StopServer, - Some("svc-current".to_owned()), - None, - None, - cli::SandboxToolPolicy::default(), - )?; - let reloaded = watchers::load_service_record(&paths, "svc-current")?; - fs::remove_dir_all(root).ok(); - - assert_eq!(value.get("status").and_then(Value::as_str), Some("stopped")); - assert_eq!(value.get("mutating").and_then(Value::as_bool), Some(true)); - assert_eq!(reloaded.status, "stopped"); - assert!( - value - .get("result") - .and_then(|result| result.get("skipped_pids")) - .and_then(Value::as_array) - .is_some_and(|pids| pids - .iter() - .any(|pid| pid.as_u64() == Some(u64::from(current_pid)))) - ); - Ok(()) - } - - /// The sandbox tool used to report `status: stopped` unconditionally. - #[test] - fn sandbox_tool_stop_server_reports_an_unconfirmed_stop() -> Result<()> { - let (root, paths) = temp_app_paths("sandbox-stop-unconfirmed"); - paths.ensure()?; - let service_id = "svc-sandbox-stop-unconfirmed"; - let key_path = seed_keyed_service(&paths, service_id, 11448)?; - - // The tool is `stop_managed_service` feeding `sandbox_stop_server_value`; - // `sandbox_tool_stop_server_updates_manifest_and_skips_current_pid` - // drives that whole tool for real. - let result = stop_with_outcome( - &paths, - service_id, - rocm_core::TerminationOutcome::Unverified, - ) - .map(sandbox_stop_server_value); - let key_kept = key_path.exists(); - let reloaded = load_service_record(&paths, service_id); - fs::remove_dir_all(root).ok(); - - let value = result?; - assert_eq!( - value.get("status").and_then(Value::as_str), - Some("stop_unconfirmed"), - "{value}" - ); - assert_eq!( - value.pointer("/result/stopped").and_then(Value::as_bool), - Some(false), - "{value}" - ); - assert_unconfirmed_stop_kept_the_service(&reloaded?, key_kept); - Ok(()) - } -} diff --git a/apps/rocmd/src/mcp.rs b/apps/rocmd/src/mcp.rs index bffe3fae47..cc38a042a6 100644 --- a/apps/rocmd/src/mcp.rs +++ b/apps/rocmd/src/mcp.rs @@ -3,6 +3,7 @@ // SPDX-License-Identifier: MIT use crate::common; +use crate::host_snapshot; use crate::persistence; use anyhow::{Context, Result, bail}; #[cfg(test)] @@ -11,12 +12,12 @@ use rocm_core::{ AppPaths, AutomationRuntimeState, CommandCapture, DEFAULT_LOCAL_HOST, ExamineSummary, RocmCliConfig, command_capture_text, load_recent_automation_events, }; -use serde::Serialize; use serde_json::Value; use serde_json::json; use std::collections::VecDeque; use std::fs; use std::io::{self, BufRead, Write}; +use std::time::Duration; pub(crate) fn run_mcp_server(paths: &AppPaths) -> Result<()> { let stdin = io::stdin(); @@ -178,14 +179,6 @@ fn write_json_line(writer: &mut impl Write, value: &Value) -> Result<()> { Ok(()) } -pub(crate) fn print_json(value: &T) -> Result<()> { - println!( - "{}", - serde_json::to_string_pretty(value).context("failed to serialize json output")? - ); - Ok(()) -} - pub(crate) fn rocm_mcp_tools() -> Vec { vec![ rocm_mcp_tool( @@ -567,7 +560,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result { let examine = ExamineSummary::gather()?; - let output = common::run_rocm_capture(&["examine"])?; + let output = run_rocm_capture(&["examine"])?; let text = command_capture_text(&output); if output.exit_status == 0 { Ok(tool_success(text, json!(examine))) @@ -584,7 +577,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result { - let snapshot = common::build_bridge_snapshot(paths)?; + let snapshot = host_snapshot::build_bridge_snapshot(paths)?; Ok(tool_success( format!( "Captured bridge snapshot for {} / {} with default engine `{}`.", @@ -595,7 +588,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result { let config = RocmCliConfig::load(paths).unwrap_or_default(); - let gpu = common::gather_gpu_snapshot_for_config(&config); + let gpu = host_snapshot::gather_gpu_snapshot_for_config(&config); let status = if !config.telemetry.local_inspection_enabled() { "GPU telemetry is disabled by rocm-cli config." } else if gpu.amd_smi_available { @@ -606,7 +599,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result { - let engines = common::bridge_engine_inventory(); + let engines = host_snapshot::bridge_engine_inventory(); Ok(tool_success( format!("Found {} engine entries.", engines.len()), json!({ "engines": engines }), @@ -670,7 +663,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result Result>(); - let output = common::run_rocm_capture(&refs)?; + let output = run_rocm_capture(&refs)?; Ok(tool_result_from_command( "Ran read-only `rocm` command.", output, @@ -689,7 +682,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result { - let output = common::run_rocm_capture(&["update"])?; + let output = run_rocm_capture(&["update"])?; Ok(tool_result_from_command( "Ran `rocm update`.", output, @@ -699,7 +692,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result { let argv = build_install_sdk_args(&arguments, true)?; let refs = argv.iter().map(String::as_str).collect::>(); - let output = common::run_rocm_capture(&refs)?; + let output = run_rocm_capture(&refs)?; Ok(tool_result_from_command( "Ran `rocm install sdk --dry-run`.", output, @@ -709,7 +702,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result { let argv = build_install_sdk_args(&arguments, false)?; let refs = argv.iter().map(String::as_str).collect::>(); - let output = common::run_rocm_capture(&refs)?; + let output = run_rocm_capture(&refs)?; Ok(tool_result_from_command( "Ran `rocm install sdk`.", output, @@ -719,7 +712,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result { let argv = build_install_engine_args(&arguments)?; let refs = argv.iter().map(String::as_str).collect::>(); - let output = common::run_rocm_capture(&refs)?; + let output = run_rocm_capture(&refs)?; Ok(tool_result_from_command( "Ran `rocm engines install`.", output, @@ -729,7 +722,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result { let argv = build_launch_server_args(&arguments)?; let refs = argv.iter().map(String::as_str).collect::>(); - let output = common::run_rocm_capture(&refs)?; + let output = run_rocm_capture(&refs)?; Ok(tool_result_from_command( "Ran `rocm serve --managed`.", output, @@ -747,7 +740,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result { let argv = build_watcher_enable_args(&arguments)?; let refs = argv.iter().map(String::as_str).collect::>(); - let output = common::run_rocm_capture(&refs)?; + let output = run_rocm_capture(&refs)?; Ok(tool_result_from_command( "Ran `rocm automations enable`.", output, @@ -765,7 +758,7 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result>(); - let output = common::run_rocm_capture(&refs)?; + let output = run_rocm_capture(&refs)?; Ok(tool_result_from_command( "Ran `rocm automations disable`.", output, @@ -779,6 +772,11 @@ pub(crate) fn handle_mcp_tool_call(paths: &AppPaths, params: &Value) -> Result Result { + let paths = AppPaths::discover()?; + common::run_rocm_capture_for_paths(&paths, args, Duration::from_mins(2)) +} + fn tool_success(text: String, structured: Value) -> Value { json!({ "content": [ @@ -1142,11 +1140,11 @@ fn system_prefix_requires_ack(prefix: &std::path::Path) -> bool { #[cfg(test)] mod tests { use super::*; + use crate::persistence::load_service_record; use crate::test_support::{ assert_unconfirmed_stop_kept_the_service, seed_keyed_service, stop_with_outcome, temp_app_paths, workspace_test_artifact_dir, }; - use crate::watchers::load_service_record; use std::fs; use std::path::PathBuf; diff --git a/apps/rocmd/src/persistence.rs b/apps/rocmd/src/persistence.rs index 847c92b7aa..5d1d3f1214 100644 --- a/apps/rocmd/src/persistence.rs +++ b/apps/rocmd/src/persistence.rs @@ -2,7 +2,7 @@ // // SPDX-License-Identifier: MIT -use anyhow::{Context, Result}; +use anyhow::{Context, Result, bail}; use rocm_core::{ AppPaths, AuditEventRecord, AutomationEventRecord, AutomationRuntimeState, ManagedServiceRecord, append_audit_event, append_automation_event, unix_time_millis, @@ -78,6 +78,31 @@ pub(crate) fn load_managed_services(paths: &AppPaths) -> Result Result { + rocm_core::ServiceId::new(service_id) + .with_context(|| format!("invalid managed service id `{service_id}`"))?; + let manifest_path = paths.service_manifest_path(service_id); + let bytes = fs::read(&manifest_path).with_context(|| { + format!( + "managed service `{service_id}` not found at {}", + manifest_path.display() + ) + })?; + let record = serde_json::from_slice::(&bytes) + .with_context(|| format!("failed to parse {}", manifest_path.display()))?; + if record.service_id != service_id { + bail!( + "managed service manifest {} contains service_id `{}`, expected `{service_id}`", + manifest_path.display(), + record.service_id + ); + } + Ok(record) +} + #[cfg(test)] mod tests { use super::*; diff --git a/apps/rocmd/src/sandbox.rs b/apps/rocmd/src/sandbox.rs index d7b0e0dc93..01b6d0ab71 100644 --- a/apps/rocmd/src/sandbox.rs +++ b/apps/rocmd/src/sandbox.rs @@ -2,12 +2,11 @@ // // SPDX-License-Identifier: MIT -use crate::ARTIFACT_PREFETCH_TIMEOUT; -use crate::cli::{SandboxToolArg, SandboxToolPolicy}; use crate::common; use crate::persistence::load_managed_services; -use crate::service::{sandbox_stop_server_value, stop_managed_service}; -use crate::watchers::restart_managed_service; +use crate::sandbox_tool::{SandboxToolArg, SandboxToolPolicy}; +use crate::service::{stop_managed_service, stop_report_confirmed}; +use crate::supervise::restart_managed_service; use anyhow::{Context, Result, bail}; use rocm_core::{ AppPaths, AuditEventRecord, CommandCapture, ExamineSummary, ModelRecipeArtifactRecord, @@ -30,6 +29,8 @@ use std::process::{Command as ProcessCommand, Stdio}; use std::thread; use std::time::Duration; +const ARTIFACT_PREFETCH_TIMEOUT: Duration = Duration::from_mins(10); + pub(crate) fn run_sandbox_runner( paths: &AppPaths, tool: SandboxToolArg, @@ -256,6 +257,23 @@ pub(crate) fn run_sandbox_tool( } } +/// The `stop_server` sandbox tool's answer for a [`stop_managed_service`] +/// report. The status is taken from the same verdict that decided what the +/// stop wrote to the manifest, so the two cannot disagree. +fn sandbox_stop_server_value(stopped: Value) -> Value { + let status = if stop_report_confirmed(&stopped) { + "stopped" + } else { + "stop_unconfirmed" + }; + json!({ + "tool": SandboxToolArg::StopServer.as_cli_value(), + "status": status, + "mutating": true, + "result": stopped, + }) +} + pub(crate) fn record_notification_audit( paths: &AppPaths, actor: &str, @@ -973,7 +991,12 @@ fn command_available(name: &str) -> bool { #[cfg(test)] mod tests { use super::*; - use crate::test_support::temp_app_paths; + use crate::persistence::load_service_record; + use crate::test_support::{ + assert_unconfirmed_stop_kept_the_service, seed_keyed_service, stop_with_outcome, + temp_app_paths, + }; + use rocm_core::ManagedServiceRecord; use std::io::Read; fn serve_one_http_response(bytes: Vec) -> Result { @@ -2248,4 +2271,89 @@ mod tests { "https://huggingface.co@evil.example/Qwen/Test-1B" )); } + + #[test] + fn sandbox_tool_stop_server_updates_manifest_and_skips_current_pid() -> Result<()> { + let (root, paths) = temp_app_paths("sandbox-stop-current-pid"); + paths.ensure()?; + let current_pid = std::process::id(); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-current", + "vllm", + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + 11435, + "managed", + current_pid, + None, + None, + None, + ); + record.engine_pid = Some(current_pid); + record.status = "ready".to_owned(); + record.write()?; + + let value = run_sandbox_tool( + &paths, + SandboxToolArg::StopServer, + Some("svc-current".to_owned()), + None, + None, + SandboxToolPolicy::default(), + )?; + let reloaded = load_service_record(&paths, "svc-current")?; + fs::remove_dir_all(root).ok(); + + assert_eq!(value.get("status").and_then(Value::as_str), Some("stopped")); + assert_eq!(value.get("mutating").and_then(Value::as_bool), Some(true)); + assert_eq!(reloaded.status, "stopped"); + assert!( + value + .get("result") + .and_then(|result| result.get("skipped_pids")) + .and_then(Value::as_array) + .is_some_and(|pids| pids + .iter() + .any(|pid| pid.as_u64() == Some(u64::from(current_pid)))) + ); + Ok(()) + } + + /// The sandbox tool used to report `status: stopped` unconditionally. + #[test] + fn sandbox_tool_stop_server_reports_an_unconfirmed_stop() -> Result<()> { + let (root, paths) = temp_app_paths("sandbox-stop-unconfirmed"); + paths.ensure()?; + let service_id = "svc-sandbox-stop-unconfirmed"; + let key_path = seed_keyed_service(&paths, service_id, 11448)?; + + // The tool is `stop_managed_service` feeding `sandbox_stop_server_value`; + // `sandbox_tool_stop_server_updates_manifest_and_skips_current_pid` + // drives that whole tool for real. + let result = stop_with_outcome( + &paths, + service_id, + rocm_core::TerminationOutcome::Unverified, + ) + .map(sandbox_stop_server_value); + let key_kept = key_path.exists(); + let reloaded = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let value = result?; + assert_eq!( + value.get("status").and_then(Value::as_str), + Some("stop_unconfirmed"), + "{value}" + ); + assert_eq!( + value.pointer("/result/stopped").and_then(Value::as_bool), + Some(false), + "{value}" + ); + assert_unconfirmed_stop_kept_the_service(&reloaded?, key_kept); + Ok(()) + } } diff --git a/apps/rocmd/src/sandbox_tool.rs b/apps/rocmd/src/sandbox_tool.rs new file mode 100644 index 0000000000..107b66f155 --- /dev/null +++ b/apps/rocmd/src/sandbox_tool.rs @@ -0,0 +1,86 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +use clap::ValueEnum; + +#[derive(Debug, Clone, Copy, Eq, PartialEq, ValueEnum)] +#[value(rename_all = "snake_case")] +pub(crate) enum SandboxToolArg { + CheckUpdates, + DriverPlan, + ExamineSnapshot, + ListServers, + RestartServer, + StopServer, + PrefetchArtifact, + NotifyUser, +} + +impl SandboxToolArg { + pub(crate) const fn as_cli_value(self) -> &'static str { + match self { + Self::CheckUpdates => "check_updates", + Self::DriverPlan => "driver_plan", + Self::ExamineSnapshot => "examine_snapshot", + Self::ListServers => "list_servers", + Self::RestartServer => "restart_server", + Self::StopServer => "stop_server", + Self::PrefetchArtifact => "prefetch_artifact", + Self::NotifyUser => "notify_user", + } + } + + #[cfg(target_os = "linux")] + pub(crate) const fn writes_data(self) -> bool { + matches!( + self, + Self::RestartServer | Self::StopServer | Self::NotifyUser + ) + } + + #[cfg(target_os = "linux")] + pub(crate) const fn writes_cache(self) -> bool { + matches!(self, Self::CheckUpdates | Self::PrefetchArtifact) + } +} + +#[derive(Debug, Clone, Default)] +pub(crate) struct SandboxToolPolicy { + pub(crate) allow_artifact_download: bool, + pub(crate) artifact_max_bytes: Option, + pub(crate) allow_huggingface_download: bool, + pub(crate) huggingface_token: Option, +} + +impl SandboxToolPolicy { + pub(crate) fn from_cli( + allow_artifact_download: bool, + artifact_max_bytes: Option, + allow_huggingface_download: bool, + ) -> Self { + Self { + allow_artifact_download, + artifact_max_bytes, + allow_huggingface_download, + huggingface_token: allow_huggingface_download + .then(resolve_huggingface_token) + .flatten(), + } + } +} + +fn resolve_huggingface_token() -> Option { + [ + "ROCM_CLI_HUGGINGFACE_TOKEN", + "HF_TOKEN", + "HUGGING_FACE_HUB_TOKEN", + ] + .iter() + .find_map(|name| { + std::env::var(name) + .ok() + .map(|value| value.trim().to_owned()) + .filter(|value| !value.is_empty()) + }) +} diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index 79793d1f8a..d65b971012 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -2,21 +2,14 @@ // // SPDX-License-Identifier: MIT -use anyhow::{Context, Result, bail}; -use rocm_core::{ - AppPaths, AutomationRuntimeState, ManagedServiceRecord, RocmCliConfig, WatcherRuntimeSnapshot, - builtin_watchers, daemon_binary_path, unix_time_millis, -}; +use anyhow::{Context, Result}; +use rocm_core::{AppPaths, ManagedServiceRecord}; use rocm_engine_protocol::{EngineMethod, StopRequest, StopResponse}; use serde_json::Value; use serde_json::json; -use std::fs; -use std::io::{self, Write}; -use std::path::Path; +#[cfg(windows)] use std::process::{Command as ProcessCommand, Stdio}; -use std::thread; use std::time::Duration; -use tokio::time::{self, MissedTickBehavior}; pub(crate) fn stop_managed_service(paths: &AppPaths, service_id: &str) -> Result { stop_managed_service_with(paths, service_id, terminate_recorded_pid) @@ -43,7 +36,7 @@ pub(crate) fn stop_managed_service_with( } fn request_engine_stop(paths: &AppPaths, record: &ManagedServiceRecord) -> Result { - crate::common::engine_request::<_, StopResponse>( + crate::engine_ipc::engine_request::<_, StopResponse>( paths, &record.engine, &record.service_id, @@ -214,23 +207,6 @@ pub(crate) fn stop_report_confirmed(report: &Value) -> bool { report.get("stopped").and_then(Value::as_bool) == Some(true) } -/// The `stop_server` sandbox tool's answer for a [`stop_managed_service`] -/// report. The status is taken from the same verdict that decided what the -/// stop wrote to the manifest, so the two cannot disagree. -pub(crate) fn sandbox_stop_server_value(stopped: Value) -> Value { - let status = if stop_report_confirmed(&stopped) { - "stopped" - } else { - "stop_unconfirmed" - }; - json!({ - "tool": crate::cli::SandboxToolArg::StopServer.as_cli_value(), - "status": status, - "mutating": true, - "result": stopped, - }) -} - /// How long a stop waits for a recorded process tree to exit on its own before /// escalating to a forced kill. Matches `rocm services stop`, which terminates /// the same records. @@ -404,1041 +380,15 @@ fn terminate_recorded_pid(identity: &rocm_core::ProcessIdentity) -> rocm_core::T ) } -pub(crate) async fn run_daemon( - paths: &AppPaths, - automations_enabled: bool, - local_webhook_port: Option, -) -> Result<()> { - if local_webhook_port.is_some() && !automations_enabled { - bail!("local webhook source requires --automations-enabled"); - } - - let config = RocmCliConfig::load(paths)?; - let mut state = build_runtime_state(&config, automations_enabled); - let local_webhook = if let Some(port) = local_webhook_port { - Some(crate::webhook::start_local_webhook_source(port).await?) - } else { - None - }; - let (local_webhook_endpoint, mut local_webhook_receiver, local_webhook_task) = - match local_webhook { - Some(source) => ( - Some(source.endpoint), - Some(source.receiver), - Some(source.task), - ), - None => (None, None, None), - }; - state.local_webhook_endpoint = local_webhook_endpoint.clone(); - - println!("rocmd run"); - println!(" automations enabled: {automations_enabled}"); - println!( - " lifecycle: {}", - if automations_enabled { - "persistent" - } else { - "on-demand" - } - ); - println!(" config: {}", paths.config_path().display()); - println!(" state: {}", paths.automation_state_path().display()); - println!( - " local_webhook_endpoint: {}", - local_webhook_endpoint.as_deref().unwrap_or("disabled") - ); - let enabled_count = state - .active_watchers - .iter() - .filter(|watcher| watcher.enabled) - .count(); - println!(" enabled watchers: {enabled_count}"); - // This banner is the foreground-loop readiness contract used by callers and - // integration tests. Flush it before any persistent work so piped stdout on - // Windows cannot retain the line in a userspace buffer indefinitely. - io::stdout() - .flush() - .context("failed to flush rocmd run banner")?; - - if !automations_enabled { - println!( - " note: rerun with --automations-enabled to keep rocmd alive for watcher execution" - ); - return Ok(()); - } - - paths.ensure()?; - state.write(paths)?; - crate::persistence::record_event( - paths, - &mut state, - "rocmd", - "info", - "daemon_start", - "rocmd automation supervisor started", - None, - )?; - state.write(paths)?; - - crate::watchers::evaluate_watchers(paths, &config, &mut state)?; - state.last_tick_unix_ms = unix_time_millis(); - state.write(paths)?; - - let shutdown = shutdown_signal(); - tokio::pin!(shutdown); - - let mut ticker = time::interval(crate::WATCHER_TICK_INTERVAL); - ticker.set_missed_tick_behavior(MissedTickBehavior::Delay); - - loop { - tokio::select! { - _ = ticker.tick() => { - let config = RocmCliConfig::load(paths)?; - crate::watchers::reconcile_watcher_snapshots(&config, &mut state); - crate::watchers::evaluate_watchers(paths, &config, &mut state)?; - state.last_tick_unix_ms = unix_time_millis(); - state.write(paths)?; - } - event = crate::webhook::receive_local_webhook_event(&mut local_webhook_receiver) => { - if let Some(event) = event { - let config = RocmCliConfig::load(paths)?; - crate::watchers::reconcile_watcher_snapshots(&config, &mut state); - crate::persistence::record_event( - paths, - &mut state, - "rocmd", - "info", - "local_webhook_event", - &format!( - "received local webhook event kind={} watcher_hint={}; dispatching through existing watcher policy; webhook payload grants no new action", - event.kind, - event.watcher_hint.as_deref().unwrap_or("") - ), - event.service_id.clone(), - )?; - if let Err(error) = - crate::watchers::evaluate_watchers_for_events(paths, &config, &mut state, &[event]) - { - crate::persistence::record_event( - paths, - &mut state, - "rocmd", - "error", - "local_webhook_dispatch_failed", - &format!( - "local webhook event could not be dispatched through watcher policy: {error}" - ), - None, - )?; - } - state.last_tick_unix_ms = unix_time_millis(); - state.write(paths)?; - } else { - local_webhook_receiver = None; - state.local_webhook_endpoint = None; - crate::persistence::record_event( - paths, - &mut state, - "rocmd", - "warn", - "local_webhook_stopped", - "local webhook source stopped; automation daemon continues without webhook ingestion", - None, - )?; - state.write(paths)?; - } - } - () = &mut shutdown => { - break; - } - } - } - - state.running = false; - state.last_tick_unix_ms = unix_time_millis(); - state.local_webhook_endpoint = None; - crate::persistence::record_event( - paths, - &mut state, - "rocmd", - "info", - "daemon_stop", - "rocmd automation supervisor stopped", - None, - )?; - state.write(paths)?; - if let Some(task) = local_webhook_task { - task.abort(); - } - Ok(()) -} - -pub(crate) fn print_status(paths: &AppPaths) -> Result<()> { - let config = RocmCliConfig::load(paths).unwrap_or_default(); - println!("rocmd status"); - println!(" config dir: {}", paths.config_dir.display()); - println!(" data dir: {}", paths.data_dir.display()); - println!(" policy: on-demand by default, persistent only with background features"); - println!( - " automations desired: {}", - if config.automation_daemon_enabled() { - "enabled" - } else { - "disabled" - } - ); - match AutomationRuntimeState::load(paths)? { - Some(state) => { - println!( - " automations runtime: {} pid={} last_tick_unix_ms={}", - if state.running { "running" } else { "stopped" }, - state.daemon_pid, - state.last_tick_unix_ms - ); - println!( - " local_webhook_endpoint: {}", - state - .local_webhook_endpoint - .as_deref() - .unwrap_or("disabled") - ); - for watcher in state - .active_watchers - .into_iter() - .filter(|watcher| watcher.enabled) - { - println!( - " watcher {} mode={} last_event={}", - watcher.id, - watcher.mode.as_str(), - watcher.last_event.as_deref().unwrap_or("") - ); - } - } - None => println!(" automations runtime: inactive"), - } - println!( - " automation events: {}", - paths.automation_events_path().display() - ); - println!(" audit events: {}", paths.audit_events_path().display()); - - let records = crate::persistence::load_managed_services(paths)?; - if records.is_empty() { - println!(" services: none"); - return Ok(()); - } - - for record in records { - println!( - " service {} engine={} status={} endpoint={}", - record.service_id, record.engine, record.status, record.endpoint_url - ); - } - - Ok(()) -} - -fn parse_gpu_indices_arg(value: Option<&str>) -> Result> { - let Some(raw) = value else { - return Ok(Vec::new()); - }; - match rocm_engine_protocol::GpuSelection::parse_cli_value(raw).map_err(anyhow::Error::msg)? { - rocm_engine_protocol::GpuSelection::Auto => Ok(Vec::new()), - rocm_engine_protocol::GpuSelection::Index(index) => Ok(vec![index]), - } -} - -#[allow(clippy::too_many_arguments)] -pub(crate) fn supervise_service( - paths: &AppPaths, - service_id: String, - engine: String, - model_ref: String, - canonical_model_id: String, - runtime_id: Option, - env_id: Option, - host: String, - port: u16, - device_policy: String, - gpu: Option, - engine_recipe_json: Option, -) -> Result<()> { - paths.ensure()?; - fs::create_dir_all(paths.engine_logs_dir(&engine))?; - fs::create_dir_all(paths.engine_state_dir(&engine))?; - fs::create_dir_all(paths.services_dir())?; - - let gpu_indices = parse_gpu_indices_arg(gpu.as_deref())?; - let _ = daemon_binary_path(); - - let mut record = ManagedServiceRecord::new( - paths, - service_id, - engine.clone(), - model_ref, - canonical_model_id.clone(), - host, - port, - "managed", - std::process::id(), - runtime_id.clone(), - env_id.clone(), - Some(device_policy.clone()), - ); - // `new` recorded this PID; this pairs it with its start-time token. - record.record_supervisor_identity(std::process::id()); - record.gpu_indices = gpu_indices; - record.engine_recipe_json = engine_recipe_json.clone(); - // Carried over from whatever is on disk. `ManagedServiceRecord::new` starts - // this false, so rebuilding a record here without restoring it would not - // just skip the check now — it would write the weakened record back and - // disarm every later `rocm services restart` as well. - // - // Propagated, not defaulted. This read arms the guard below, so it is not - // best-effort the way an identical-looking call feeding a printed warning - // would be. `load_managed_services` already *skips* unparseable records, so - // an `Err` here is a real I/O failure — and a missing directory is `Ok` - // anyway. Swallowing it would say "no service ever required a key", the - // key-file fallback is false precisely when a service has been stopped, and - // the weakened record would then be written back at the bottom of this - // function. That is the outcome the comment above says must not happen. - let existing = crate::persistence::load_managed_services(paths) - .context( - "could not read the service registry to check whether this service requires an \ - endpoint API key; refusing to recover it rather than assume it does not", - )? - .into_iter() - .find(|existing| existing.service_id == record.service_id); - let previously_required = existing - .as_ref() - .is_some_and(|existing| existing.requires_api_key); - // Only what the registry recorded. The `|| key-file-is-present` clause that - // used to be here re-derived the flag the same way `spawn_managed_engine_child` - // did, and was wrong for the same reason: a public bind always has a key file - // whether or not auth was ever demanded, so recovery re-armed this on services - // that never asked for it and refused them with the wrong remediation. - record.requires_api_key = previously_required; - // A standing stop request is carried over too. The restart that launched - // this supervisor clears the marker before it does so, so one found here was - // set by a stop issued since — and `ManagedServiceRecord::new` starting it - // at `None` would quietly withdraw that stop and hand the service straight - // back to the daemon's recovery. - record.stop_requested_unix_ms = existing.and_then(|existing| existing.stop_requested_unix_ms); - // Refuse a keyless public respawn before the manifest write, so a refused - // attempt leaves the recorded restart_count and timestamps intact instead of - // clobbering them with a record no live process will ever back. The spawn - // site below re-checks against the key actually threaded onto the command. - crate::common::ensure_public_service_has_endpoint_key( - &record.host, - rocm_engine_protocol::endpoint_key_file_if_present(paths, &record.service_id) - .and_then(|path| rocm_engine_protocol::endpoint_api_key_file_if_valid(&path)) - .is_some(), - record.requires_api_key, - )?; - record.write()?; - - let log_file = fs::File::create(&record.log_path) - .with_context(|| format!("failed to create {}", record.log_path.display()))?; - let log_file_err = log_file - .try_clone() - .context("failed to clone service log file handle")?; - - let rocm_binary = - std::env::current_exe().context("failed to resolve current rocm executable path")?; - let mut command = ProcessCommand::new(rocm_binary); - command - .args(engine_serve_http_args( - &engine, - &record.service_id, - &canonical_model_id, - &record.host, - record.port, - &device_policy, - &record.gpu_indices, - runtime_id.as_deref(), - env_id.as_deref(), - engine_recipe_json.as_deref(), - &record.engine_state_path, - )) - .stdin(Stdio::null()) - .stdout(Stdio::from(log_file)) - .stderr(Stdio::from(log_file_err)); - // Re-thread the endpoint key file (public bind only) onto the engine child, - // same as the initial `rocm serve` spawn. This path also runs on daemon - // recovery (`restart_managed_service` re-execs `rocmd supervise`), so - // without this a previously-authenticated public service would come back - // up anonymous after a crash/recover cycle. - // If the key is gone the child would listen on the recorded public host with - // no auth, so fail closed instead — an unreachable service is recoverable, - // an anonymous public one is not. - let endpoint_key_applied = - crate::common::apply_endpoint_key_env(&mut command, paths, &record.service_id); - crate::common::ensure_public_service_has_endpoint_key( - &record.host, - endpoint_key_applied, - record.requires_api_key, - )?; - let mut child = command - .spawn() - .with_context(|| format!("failed to spawn engine supervisor child for {engine}"))?; - - // Every write from here on goes through `update_supervised_record`: it - // applies only this supervisor's own fields to the record as it is on disk - // now, so a stop marker, or anything else another writer persisted meanwhile, - // survives it. - // - // Captured while the child is known alive, so a later stop verifies this - // exact process instead of whatever has since inherited its PID. - let engine_pid = child.id(); - update_supervised_record(paths, &record, |current| { - current.record_engine_identity(engine_pid); - current.status = "running".to_owned(); - })?; - - let became_ready = wait_for_service_ready( - paths, - &record.engine, - &record.service_id, - &record.log_path, - Duration::from_mins(3), - |phase| { - let _ = update_supervised_record(paths, &record, |current| { - current.startup_phase = Some(phase.to_owned()); - }); - }, - ); - if became_ready { - update_supervised_record(paths, &record, |current| { - current.status = "ready".to_owned(); - // The phase only describes the coming-up window; clear it once ready. - current.startup_phase = None; - })?; - } - - let exit_status = child.wait().context("failed waiting for engine child")?; - record_supervised_exit(paths, &record, exit_status.success())?; - - if exit_status.success() { - Ok(()) - } else { - std::process::exit(exit_status.code().unwrap_or(1)); - } -} - -/// Apply one of [`supervise_service`]'s own changes to its service's record as -/// it is on disk now, rather than writing back the copy the supervisor built at -/// startup. -/// -/// `write` replaces the whole file, and the supervisor's copy is stale the -/// moment anyone else writes: a stop records `stop_requested_unix_ms` before it -/// signals anything, and `rocm`'s liveness refresh and the daemon's restart -/// write here too. Writing the startup snapshot back would erase that stop -/// marker — and a record reading `failed` with no marker is exactly what the -/// daemon's recovery restarts, undoing the stop. So `update` touches only the -/// fields the supervisor owns, and never the marker. -/// -/// Nothing is written when the record no longer belongs to this supervisor: -/// -/// - it is gone — `rocm services remove` deleted it, and writing would -/// resurrect a service the operator removed; -/// - it names a different supervisor process — a confirmed stop clears the -/// recorded PIDs, and a restart records its own. Either way the record now -/// describes something this supervisor is not, and its view of the service -/// (`running`, `failed`, …) would overwrite a truer one. -pub(crate) fn update_supervised_record( - paths: &AppPaths, - supervisor: &ManagedServiceRecord, - update: impl FnOnce(&mut ManagedServiceRecord), -) -> Result<()> { - if !paths.service_manifest_path(&supervisor.service_id).exists() { - return Ok(()); - } - let mut current = crate::watchers::load_service_record(paths, &supervisor.service_id)?; - if current.supervisor_pid != supervisor.supervisor_pid - || current.supervisor_start_ticks != supervisor.supervisor_start_ticks - { - return Ok(()); - } - update(&mut current); - current.write() -} - -/// [`supervise_service`]'s last write: the engine child has exited, so record -/// how. Separate so a test can drive exactly this write. -fn record_supervised_exit( - paths: &AppPaths, - supervisor: &ManagedServiceRecord, - success: bool, -) -> Result<()> { - update_supervised_record(paths, supervisor, |current| { - current.status = if success { "stopped" } else { "failed" }.to_owned(); - }) -} - -#[allow(clippy::too_many_arguments)] -fn engine_serve_http_args( - engine: &str, - service_id: &str, - canonical_model_id: &str, - host: &str, - port: u16, - device_policy: &str, - gpu_indices: &[u32], - runtime_id: Option<&str>, - env_id: Option<&str>, - engine_recipe_json: Option<&str>, - state_path: &Path, -) -> Vec { - let mut args = vec![ - "__engine-serve-http".to_owned(), - engine.to_owned(), - service_id.to_owned(), - canonical_model_id.to_owned(), - "--host".to_owned(), - host.to_owned(), - "--port".to_owned(), - port.to_string(), - "--device-policy".to_owned(), - device_policy.to_owned(), - ]; - if let Some(csv) = rocm_engine_protocol::gpu_indices_to_csv(gpu_indices) { - args.extend(["--gpu".to_owned(), csv]); - } - args.extend(crate::common::optional_arg("--runtime-id", runtime_id)); - args.extend(crate::common::optional_arg("--env-id", env_id)); - args.extend(crate::common::optional_arg( - "--engine-recipe-json", - engine_recipe_json, - )); - args.extend(["--state-path".to_owned(), state_path.display().to_string()]); - args -} - -fn build_runtime_state( - config: &RocmCliConfig, - automations_enabled: bool, -) -> AutomationRuntimeState { - let now = unix_time_millis(); - let active_watchers = builtin_watchers() - .iter() - .map(|watcher| WatcherRuntimeSnapshot { - id: watcher.id.to_owned(), - enabled: config.watcher_enabled(watcher), - mode: config.effective_watcher_mode(watcher), - summary: watcher.summary.to_owned(), - last_event: None, - last_event_unix_ms: None, - }) - .collect(); - AutomationRuntimeState { - running: automations_enabled, - automations_enabled, - daemon_pid: std::process::id(), - started_at_unix_ms: now, - last_tick_unix_ms: now, - local_webhook_endpoint: None, - active_watchers, - } -} - -#[cfg(unix)] -async fn shutdown_signal() { - let mut term = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()) - .expect("failed to register SIGTERM handler"); - tokio::select! { - _ = tokio::signal::ctrl_c() => {} - _ = term.recv() => {} - } -} - -#[cfg(not(unix))] -async fn shutdown_signal() { - let _ = tokio::signal::ctrl_c().await; -} - -/// Keep only the final visible segment of a `\r`-redrawn progress line. -/// -/// Progress tools (pip, tqdm, Hugging Face) redraw a line in place with a bare -/// carriage return and no newline, so the segment after the last `\r` is its -/// final visible state. Lines without `\r` pass through unchanged. (Same -/// collapse rule the dashboard job console applies to streamed job output.) -fn last_cr_segment(line: &str) -> &str { - line.rsplit('\r').next().unwrap_or(line) -} - -/// Classify a single serve-log line into a coarse startup phase token -/// (`downloading`/`loading`/`warmup`), or `None` when the line carries no phase -/// signal. Case-insensitive substring match over the common vLLM / llama.cpp / -/// Hugging Face startup vocabulary. Checked warmup → loading → downloading so -/// the latest lifecycle stage a line mentions wins. -fn classify_startup_phase(line: &str) -> Option<&'static str> { - let lower = line.to_ascii_lowercase(); - if lower.contains("capturing cuda graph") - || lower.contains("capturing the model") - || lower.contains("warming up") - || lower.contains("warmup") - { - Some("warmup") - } else if lower.contains("loading weights") - || lower.contains("loading model") - || lower.contains("load_tensors") - || lower.contains("llama_model_loader") - || lower.contains("model loading took") - { - Some("loading") - } else if lower.contains("downloading") || lower.contains("fetching") { - Some("downloading") - } else { - None - } -} - -/// Read log bytes appended since `*pos`, advance `*pos`, and return the most -/// recent recognizable startup phase in that new output (later lines win, so a -/// download → load → warmup progression advances naturally). -/// -/// Best-effort: any I/O error (file not created yet, transient read) yields -/// `None`. A shrunk file (rotation/truncation) resets the cursor to the top. -fn read_new_log_phase(log_path: &Path, pos: &mut u64) -> Option<&'static str> { - use std::io::{Read, Seek, SeekFrom}; - let mut file = fs::File::open(log_path).ok()?; - let len = file.metadata().ok()?.len(); - if len < *pos { - *pos = 0; - } - if len == *pos { - return None; - } - file.seek(SeekFrom::Start(*pos)).ok()?; - let mut bytes = Vec::new(); - let read = file.read_to_end(&mut bytes).ok()?; - *pos += read as u64; - let text = String::from_utf8_lossy(&bytes); - let mut phase = None; - for line in text.lines() { - if let Some(found) = classify_startup_phase(last_cr_segment(line)) { - phase = Some(found); - } - } - phase -} - -fn engine_healthcheck_ready(paths: &AppPaths, engine: &str, service_id: &str) -> Result { - Ok(crate::common::healthcheck_response_ready( - &crate::common::engine_healthcheck_response(paths, engine, service_id)?, - )) -} - -/// Poll a freshly-spawned service until its healthcheck reports ready (or the -/// timeout elapses), tailing its log file meanwhile and reporting each coarse -/// startup phase transition via `on_phase`. -fn wait_for_service_ready( - paths: &AppPaths, - engine: &str, - service_id: &str, - log_path: &Path, - timeout: Duration, - mut on_phase: impl FnMut(&str), -) -> bool { - let start = std::time::Instant::now(); - let mut log_pos: u64 = 0; - let mut last_phase: Option<&'static str> = None; - while start.elapsed() < timeout { - if let Some(phase) = read_new_log_phase(log_path, &mut log_pos) - && last_phase != Some(phase) - { - last_phase = Some(phase); - on_phase(phase); - } - if engine_healthcheck_ready(paths, engine, service_id).unwrap_or(false) { - return true; - } - thread::sleep(Duration::from_millis(200)); - } - false -} - #[cfg(test)] mod tests { use super::*; - use crate::test_support::identity_probe_record; - use crate::test_support::{ - UNCONFIRMED_STOP_PID, seed_keyed_service, temp_app_paths, unique_test_root, - }; - use crate::watchers::load_service_record; - use crate::watchers::{find_recoverable_service, stop_requested}; - - /// Drive `supervise_service` far enough to reach the key guard, and return - /// what it did. - /// - /// The guard sits before the manifest write and well before any spawn, so a - /// refusal returns without starting a process — which is what makes the real - /// call site testable at all. The arguments below are the shape a recovery - /// re-exec passes: a loopback bind, no GPU, no recipe. - /// - /// This exists because testing `ensure_public_service_has_endpoint_key` - /// directly with literal arguments cannot catch the defect that actually - /// happened twice in this crate's history — the guard being *wired up* with - /// the wrong value at its call site. - /// - /// Bounded, and the bound is the assertion. A guard that fails to refuse - /// does not return an error — it falls through to the engine spawn and - /// supervises a child that never exits, so an unbounded call would hang the - /// suite instead of failing it. Both callers below are regression tests for - /// a fail-*open*, which is exactly the shape that turns into a hang. - fn supervise_at_the_guard(paths: &AppPaths, service_id: &str) -> Result<()> { - let paths = paths.clone(); - let service_id = service_id.to_owned(); - let (sender, receiver) = std::sync::mpsc::channel(); - std::thread::spawn(move || { - let outcome = supervise_service( - &paths, - service_id, - "llamacpp".to_owned(), - "a-model".to_owned(), - "a-model".to_owned(), - None, - None, - "127.0.0.1".to_owned(), - 11434, - "gpu_required".to_owned(), - None, - None, - ); - let _ = sender.send(outcome.map_err(|error| format!("{error:#}"))); - }); - receiver - .recv_timeout(std::time::Duration::from_secs(30)) - .unwrap_or_else(|_| { - panic!( - "supervise_service did not return within 30s: the key guard let the call \ - through and it reached the engine spawn, which is the fail-open this test \ - exists to catch" - ) - }) - .map_err(anyhow::Error::msg) - } - - /// Write a service record into the registry the way a live service would - /// have left it behind. - fn seed_registry(paths: &AppPaths, service_id: &str, requires_api_key: bool) { - fs::create_dir_all(paths.services_dir()).unwrap(); - let mut record = ManagedServiceRecord::new( - paths, - service_id.to_owned(), - "llamacpp".to_owned(), - "a-model".to_owned(), - "a-model".to_owned(), - "127.0.0.1".to_owned(), - 11434, - "managed", - std::process::id(), - None, - None, - Some("gpu_required".to_owned()), - ); - record.requires_api_key = requires_api_key; - record.write().unwrap(); - } - - /// Drive `supervise_service` and report whether the key guard let it past. - /// - /// Decided on what the call *returns*, not on any file. The obvious - /// observable — the manifest appearing — is useless here, because - /// `seed_registry` has already written one, so polling for it passes - /// whatever the guard does. That mistake was made first and caught by - /// mutating the code the test claims to protect. - /// - /// A call the guard admits does not return: it carries on to the engine - /// spawn. So the guard's refusal is the only thing that comes back quickly, - /// and it is identified by its message rather than by the mere fact of an - /// error — a later, unrelated failure must not read as a refusal. - fn guard_admits(paths: &AppPaths, service_id: &str) -> bool { - let owned_paths = paths.clone(); - let owned_id = service_id.to_owned(); - let (sender, receiver) = std::sync::mpsc::channel(); - std::thread::spawn(move || { - let outcome = supervise_service( - &owned_paths, - owned_id, - "llamacpp".to_owned(), - "a-model".to_owned(), - "a-model".to_owned(), - None, - None, - "127.0.0.1".to_owned(), - 11434, - "gpu_required".to_owned(), - None, - None, - ); - let _ = sender.send(outcome.map_err(|error| format!("{error:#}"))); - }); - - match receiver.recv_timeout(std::time::Duration::from_secs(10)) { - // The key guard refused, by its own words. - Ok(Err(rendered)) if rendered.contains("without authentication") => false, - // Anything else means it got past the guard: it either finished, or - // failed later for a reason that is not this guard, or is still - // running because it reached the spawn. - _ => true, - } - } - - #[test] - fn a_service_that_never_required_a_key_is_not_refused_for_lacking_one() { - // The other direction of the guard, and the one no test covered. - // Hardcoding `record.requires_api_key = true` at the restore site passes - // every other test in this crate, because they all seed a service that - // *does* require a key. This is the case that catches it. - // - // Two records are seeded, not one: with a single record the - // `existing.service_id == record.service_id` half of the lookup does - // nothing, so dropping that comparison would go unnoticed and one - // service's requirement would leak onto another's. - let (root, paths) = temp_app_paths("supervise-no-key-needed"); - seed_registry(&paths, "svc-needs-key", true); - seed_registry(&paths, "svc-plain", false); - - assert!( - guard_admits(&paths, "svc-plain"), - "a loopback service that never asked for a key must not be refused for lacking one" - ); - - let _ = fs::remove_dir_all(&root); - } - - #[test] - fn supervising_a_service_that_required_a_key_refuses_when_the_key_is_gone() { - // The real call site, not the guard in isolation. `supervise_service` - // rebuilds the record with `ManagedServiceRecord::new`, which starts - // `requires_api_key` false, and restores it from the registry. Passing - // the wrong value here — a literal, or the freshly-built field before it - // is restored — is exactly the miswiring that shipped twice in this - // crate and that a literal-argument unit test cannot see. - let (root, paths) = temp_app_paths("supervise-requires-key"); - seed_registry(&paths, "svc-needs-key", true); - - let error = supervise_at_the_guard(&paths, "svc-needs-key") - .expect_err("a service that required a key must not be recovered without one"); - assert!( - format!("{error:#}").contains("without authentication"), - "{error:#}" - ); - - // And the refusal must not have weakened what is on disk. The guard runs - // before `record.write()` precisely so a refused attempt leaves the - // recorded requirement armed for the next attempt. - let stored = crate::persistence::load_managed_services(&paths).unwrap(); - let stored = stored - .iter() - .find(|candidate| candidate.service_id == "svc-needs-key") - .expect("the seeded record must survive a refused recovery"); - assert!( - stored.requires_api_key, - "a refused recovery must not disarm the requirement" - ); - - let _ = fs::remove_dir_all(&root); - } - - #[test] - fn a_registry_that_cannot_be_read_refuses_recovery_rather_than_assuming_no_key() { - // `load_managed_services` already skips records it cannot parse, so an - // `Err` from it is a real I/O failure — and a missing directory is `Ok`. - // Defaulting it away therefore says "no service ever required a key", - // which is fail-open on an auth gate and, worse, gets written back. - // - // The failure is provoked portably: a directory named like a record - // makes the `fs::read` inside the loop fail rather than the read_dir. - let (root, paths) = temp_app_paths("supervise-unreadable-registry"); - fs::create_dir_all(paths.services_dir().join("not-a-record.json")).unwrap(); - - let error = supervise_at_the_guard(&paths, "svc-unknown") - .expect_err("an unreadable registry must refuse, not assume no key was required"); - let rendered = format!("{error:#}"); - assert!( - rendered.contains("could not read the service registry"), - "{rendered}" - ); - - // Nothing was written: a registry we could not read is not a registry we - // may add a weakened record to. - assert!( - !paths.service_manifest_path("svc-unknown").exists(), - "a refused recovery must not persist a record" - ); - - let _ = fs::remove_dir_all(&root); - } - - #[test] - fn last_cr_segment_keeps_final_progress_redraw() { - // A tqdm/HF-style in-place redraw collapses to its last segment. - assert_eq!( - last_cr_segment("Downloading: 10%\rDownloading: 55%\rDownloading: 100%"), - "Downloading: 100%" - ); - // A plain line is unchanged. - assert_eq!( - last_cr_segment("Loading model weights"), - "Loading model weights" - ); - } - - #[test] - fn classify_startup_phase_maps_engine_vocabulary() { - assert_eq!( - classify_startup_phase("Downloading shards: 100%"), - Some("downloading") - ); - assert_eq!( - classify_startup_phase("Fetching 12 files"), - Some("downloading") - ); - assert_eq!( - classify_startup_phase("INFO: Loading model weights took 4.2s"), - Some("loading") - ); - assert_eq!( - classify_startup_phase("llama_model_loader: loaded meta data"), - Some("loading") - ); - assert_eq!( - classify_startup_phase("Capturing CUDA graph shapes"), - Some("warmup") - ); - assert_eq!( - classify_startup_phase("Warming up the engine"), - Some("warmup") - ); - // Ordinary chatter carries no phase signal. - assert_eq!( - classify_startup_phase("Uvicorn running on http://..."), - None - ); - } - - #[test] - fn classify_startup_phase_emits_only_dashboard_known_tokens() { - // These tokens are the wire contract with the dashboard's - // `StartupPhase::from_token` (rocm-dash-core); emitting anything else - // would be silently dropped there. rocmd can't link that crate, so the - // contract is pinned here by literal. - for line in [ - "Downloading shards", - "Loading model weights", - "Capturing CUDA graph", - ] { - let token = classify_startup_phase(line).expect("line is a phase signal"); - assert!( - matches!(token, "downloading" | "loading" | "warmup"), - "token {token:?} must be one the dashboard understands" - ); - } - } - - #[test] - fn read_new_log_phase_advances_and_tracks_latest() { - use std::io::Write as _; - // Workspace-local test root (rooted at CARGO_MANIFEST_DIR, not the - // ambient temp dir) — same helper the other rocmd tests use. - let dir = unique_test_root(&format!("rocmd-phase-{}", std::process::id())); - let log = dir.join("svc.log"); - std::fs::write(&log, "boot\nDownloading shards: 100%\n").unwrap(); - - let mut pos = 0_u64; - assert_eq!(read_new_log_phase(&log, &mut pos), Some("downloading")); - // No new bytes → no phase, cursor unchanged. - let after_first = pos; - assert_eq!(read_new_log_phase(&log, &mut pos), None); - assert_eq!(pos, after_first); - - // Appending a later stage advances the phase. - let mut f = std::fs::OpenOptions::new().append(true).open(&log).unwrap(); - writeln!(f, "Loading model weights took 3s").unwrap(); - assert_eq!(read_new_log_phase(&log, &mut pos), Some("loading")); - - let _ = std::fs::remove_dir_all(&dir); - } - - #[test] - fn engine_serve_http_args_forward_engine_recipe_json() { - let engine_recipe_json = r#"{"contract_version":"0.1.0","engine":"vllm","required_flags":["--enable-auto-tool-choice"]}"#; - let args = engine_serve_http_args( - "vllm", - "svc-1", - "Qwen/Qwen3.5-4B", - "127.0.0.1", - 11435, - "gpu_required", - &[], - Some("therock-release:gfx120X-all"), - Some("env-1"), - Some(engine_recipe_json), - Path::new("state.json"), - ); - - assert!( - args.windows(2) - .any(|pair| pair[0] == "--engine-recipe-json" && pair[1] == engine_recipe_json) - ); - assert!( - args.windows(2).any(|pair| { - pair[0] == "--runtime-id" && pair[1] == "therock-release:gfx120X-all" - }) - ); - assert!( - args.windows(2) - .any(|pair| pair[0] == "--state-path" && pair[1] == "state.json") - ); - } - - #[test] - fn engine_serve_http_args_emit_gpu_indices_when_pinned() { - let args = engine_serve_http_args( - "vllm", - "svc-1", - "Qwen/Qwen3.5-4B", - "127.0.0.1", - 11435, - "gpu_required", - &[1], - None, - None, - None, - Path::new("state.json"), - ); - - assert!( - args.windows(2) - .any(|pair| pair[0] == "--gpu" && pair[1] == "1") - ); - - let auto = engine_serve_http_args( - "vllm", - "svc-1", - "Qwen/Qwen3.5-4B", - "127.0.0.1", - 11435, - "gpu_required", - &[], - None, - None, - None, - Path::new("state.json"), - ); - assert!(!auto.iter().any(|arg| arg == "--gpu")); - } - - #[tokio::test] - async fn local_webhook_requires_enabled_automation_loop() { - let (_root, paths) = temp_app_paths("local-webhook-requires-loop"); - let error = run_daemon(&paths, false, Some(0)).await.unwrap_err(); - - assert!(error.to_string().contains("requires --automations-enabled")); - } + use crate::persistence::load_service_record; + use crate::test_support::{UNCONFIRMED_STOP_PID, seed_keyed_service, temp_app_paths}; + use crate::watchers::stop_requested; + use std::fs; + #[cfg(unix)] + use std::process::{Command as ProcessCommand, Stdio}; #[test] fn stop_managed_service_removes_endpoint_key_file() -> Result<()> { @@ -1517,7 +467,7 @@ mod tests { assert!(!key_path.exists()); let result = stop_managed_service(&paths, service_id); - let reloaded = crate::watchers::load_service_record(&paths, service_id); + let reloaded = crate::persistence::load_service_record(&paths, service_id); fs::remove_dir_all(root).ok(); let value = result?; @@ -2129,174 +1079,4 @@ mod tests { ); Ok(()) } - - /// The identity check is only as good as the token the launcher writes down. - /// `rocm_core::identity_state` treats a missing token as a best-effort match - /// — the legacy `(None, _)` branch — so a record written without one is - /// signalled blind, which is the original defect restored in full. Every - /// other test here hands itself a record with the token already set, and so - /// proves nothing about the code that is supposed to put it there. - /// - /// This drives the real `supervise_service` and reads the manifest it - /// persisted. It is stopped at that first write by making the service log - /// path a directory, so the `fs::File::create` immediately after it fails: - /// going further reaches the engine spawn, and the failure path past that - /// calls `std::process::exit`, which inside a test binary would take the - /// whole suite with it. - /// - /// Linux-only: `rocm_core::process_start_ticks` reads `/proc` and returns - /// `None` everywhere else, so on other platforms there is no token to write. - #[cfg(target_os = "linux")] - #[test] - fn supervise_service_persists_the_supervisor_identity_token() -> Result<()> { - let (root, paths) = temp_app_paths("supervise-records-identity"); - paths.ensure()?; - fs::create_dir_all(paths.services_dir())?; - - let service_id = "svc-supervise-identity"; - let log_path = identity_probe_record(&paths, service_id, 11443).log_path; - // A directory where the supervisor expects to create a file. - fs::create_dir_all(&log_path)?; - - let outcome = supervise_service( - &paths, - service_id.to_owned(), - "llamacpp".to_owned(), - "a-model".to_owned(), - "a-model".to_owned(), - None, - None, - "127.0.0.1".to_owned(), - 11443, - "gpu_required".to_owned(), - None, - None, - ); - let persisted = load_service_record(&paths, service_id); - fs::remove_dir_all(root).ok(); - - let error = - outcome.expect_err("the log path is a directory, so the engine spawn is unreachable"); - assert!( - format!("{error:#}").contains("failed to create"), - "stopped somewhere other than the log file: {error:#}" - ); - assert!( - persisted?.supervisor_start_ticks.is_some(), - "supervise_service must persist the supervisor's start-time token beside its PID" - ); - Ok(()) - } - - /// The supervisor's last write lands after its engine child exits — which - /// is what a stop that could not verify the supervisor, but did kill the - /// engine, leaves behind. Writing back the record the supervisor built at - /// startup would erase the stop marker the stop persisted, and a `failed` - /// record with no marker is exactly what the daemon's recovery restarts. - #[test] - fn supervisor_exit_write_keeps_a_standing_stop_marker() -> Result<()> { - let (root, paths) = temp_app_paths("supervise-exit-keeps-marker"); - paths.ensure()?; - let service_id = "svc-supervise-exit-marker"; - // The supervisor's own copy, as `supervise_service` builds it: no marker. - let mut supervisor = identity_probe_record(&paths, service_id, 11446); - supervisor.record_supervisor_identity(std::process::id()); - supervisor.status = "running".to_owned(); - // What is on disk once a stop has run: same processes, marker set. - let mut on_disk = supervisor.clone(); - on_disk.stop_requested_unix_ms = Some(1); - on_disk.write()?; - - record_supervised_exit(&paths, &supervisor, false)?; - let persisted = load_service_record(&paths, service_id); - let recoverable = find_recoverable_service(&paths); - fs::remove_dir_all(root).ok(); - - let persisted = persisted?; - assert_eq!( - persisted.status, "failed", - "the supervisor's own field lands" - ); - assert_eq!( - persisted.stop_requested_unix_ms, - Some(1), - "the supervisor's exit write must not withdraw a standing stop" - ); - assert!( - recoverable?.is_none(), - "a stopped service must not become a recovery candidate" - ); - Ok(()) - } - - /// A record that names another supervisor — a confirmed stop cleared the - /// PIDs, or a restart recorded its own — or that was removed outright, is - /// no longer this supervisor's to write. - #[test] - fn supervisor_exit_write_leaves_a_record_it_no_longer_owns() -> Result<()> { - let (root, paths) = temp_app_paths("supervise-exit-not-owner"); - paths.ensure()?; - let service_id = "svc-supervise-exit-not-owner"; - let mut supervisor = identity_probe_record(&paths, service_id, 11447); - supervisor.record_supervisor_identity(std::process::id()); - supervisor.status = "running".to_owned(); - // A confirmed stop's final write: status stopped, PIDs cleared. - let mut stopped = supervisor.clone(); - stopped.status = "stopped".to_owned(); - stopped.supervisor_pid = 0; - stopped.supervisor_start_ticks = None; - stopped.write()?; - - record_supervised_exit(&paths, &supervisor, false)?; - let after_stop = load_service_record(&paths, service_id); - - fs::remove_file(paths.service_manifest_path(service_id))?; - record_supervised_exit(&paths, &supervisor, false)?; - let resurrected = paths.service_manifest_path(service_id).exists(); - fs::remove_dir_all(root).ok(); - - assert_eq!(after_stop?.status, "stopped"); - assert!(!resurrected, "a removed record must not be written back"); - Ok(()) - } - - /// The supervisor's first write rebuilds the record from its arguments. - /// The restart that launched it cleared any stop marker first, so a marker - /// on disk here was set by a stop issued since, and must survive. - #[cfg(target_os = "linux")] - #[test] - fn supervise_service_first_write_keeps_a_standing_stop_marker() -> Result<()> { - let (root, paths) = temp_app_paths("supervise-first-write-keeps-marker"); - paths.ensure()?; - fs::create_dir_all(paths.services_dir())?; - - let service_id = "svc-supervise-first-write-marker"; - let mut seeded = identity_probe_record(&paths, service_id, 11448); - seeded.stop_requested_unix_ms = Some(1); - seeded.write()?; - // Stop `supervise_service` right after its first write; see - // `supervise_service_persists_the_supervisor_identity_token`. - fs::create_dir_all(&seeded.log_path)?; - - let outcome = supervise_service( - &paths, - service_id.to_owned(), - "llamacpp".to_owned(), - "a-model".to_owned(), - "a-model".to_owned(), - None, - None, - "127.0.0.1".to_owned(), - 11448, - "gpu_required".to_owned(), - None, - None, - ); - let persisted = load_service_record(&paths, service_id); - fs::remove_dir_all(root).ok(); - - assert!(outcome.is_err(), "the log path is a directory"); - assert_eq!(persisted?.stop_requested_unix_ms, Some(1)); - Ok(()) - } } diff --git a/apps/rocmd/src/supervise.rs b/apps/rocmd/src/supervise.rs new file mode 100644 index 0000000000..887744ea06 --- /dev/null +++ b/apps/rocmd/src/supervise.rs @@ -0,0 +1,1321 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +use anyhow::{Context, Result}; +use rocm_core::{AppPaths, ManagedServiceRecord, daemon_binary_path}; +use std::fs; +use std::path::Path; +use std::process::{Command as ProcessCommand, Stdio}; +use std::thread; +use std::time::Duration; + +fn parse_gpu_indices_arg(value: Option<&str>) -> Result> { + let Some(raw) = value else { + return Ok(Vec::new()); + }; + match rocm_engine_protocol::GpuSelection::parse_cli_value(raw).map_err(anyhow::Error::msg)? { + rocm_engine_protocol::GpuSelection::Auto => Ok(Vec::new()), + rocm_engine_protocol::GpuSelection::Index(index) => Ok(vec![index]), + } +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn supervise_service( + paths: &AppPaths, + service_id: String, + engine: String, + model_ref: String, + canonical_model_id: String, + runtime_id: Option, + env_id: Option, + host: String, + port: u16, + device_policy: String, + gpu: Option, + engine_recipe_json: Option, +) -> Result<()> { + paths.ensure()?; + fs::create_dir_all(paths.engine_logs_dir(&engine))?; + fs::create_dir_all(paths.engine_state_dir(&engine))?; + fs::create_dir_all(paths.services_dir())?; + + let gpu_indices = parse_gpu_indices_arg(gpu.as_deref())?; + let _ = daemon_binary_path(); + + let mut record = ManagedServiceRecord::new( + paths, + service_id, + engine.clone(), + model_ref, + canonical_model_id.clone(), + host, + port, + "managed", + std::process::id(), + runtime_id.clone(), + env_id.clone(), + Some(device_policy.clone()), + ); + // `new` recorded this PID; this pairs it with its start-time token. + record.record_supervisor_identity(std::process::id()); + record.gpu_indices = gpu_indices; + record.engine_recipe_json = engine_recipe_json.clone(); + // Carried over from whatever is on disk. `ManagedServiceRecord::new` starts + // this false, so rebuilding a record here without restoring it would not + // just skip the check now — it would write the weakened record back and + // disarm every later `rocm services restart` as well. + // + // Propagated, not defaulted. This read arms the guard below, so it is not + // best-effort the way an identical-looking call feeding a printed warning + // would be. `load_managed_services` already *skips* unparseable records, so + // an `Err` here is a real I/O failure — and a missing directory is `Ok` + // anyway. Swallowing it would say "no service ever required a key", the + // key-file fallback is false precisely when a service has been stopped, and + // the weakened record would then be written back at the bottom of this + // function. That is the outcome the comment above says must not happen. + let existing = crate::persistence::load_managed_services(paths) + .context( + "could not read the service registry to check whether this service requires an \ + endpoint API key; refusing to recover it rather than assume it does not", + )? + .into_iter() + .find(|existing| existing.service_id == record.service_id); + let previously_required = existing + .as_ref() + .is_some_and(|existing| existing.requires_api_key); + // Only what the registry recorded. The `|| key-file-is-present` clause that + // used to be here re-derived the flag the same way `spawn_managed_engine_child` + // did, and was wrong for the same reason: a public bind always has a key file + // whether or not auth was ever demanded, so recovery re-armed this on services + // that never asked for it and refused them with the wrong remediation. + record.requires_api_key = previously_required; + // A standing stop request is carried over too. The restart that launched + // this supervisor clears the marker before it does so, so one found here was + // set by a stop issued since — and `ManagedServiceRecord::new` starting it + // at `None` would quietly withdraw that stop and hand the service straight + // back to the daemon's recovery. + record.stop_requested_unix_ms = existing.and_then(|existing| existing.stop_requested_unix_ms); + // Refuse a keyless public respawn before the manifest write, so a refused + // attempt leaves the recorded restart_count and timestamps intact instead of + // clobbering them with a record no live process will ever back. The spawn + // site below re-checks against the key actually threaded onto the command. + crate::engine_ipc::ensure_public_service_has_endpoint_key( + &record.host, + rocm_engine_protocol::endpoint_key_file_if_present(paths, &record.service_id) + .and_then(|path| rocm_engine_protocol::endpoint_api_key_file_if_valid(&path)) + .is_some(), + record.requires_api_key, + )?; + record.write()?; + + let log_file = fs::File::create(&record.log_path) + .with_context(|| format!("failed to create {}", record.log_path.display()))?; + let log_file_err = log_file + .try_clone() + .context("failed to clone service log file handle")?; + + let rocm_binary = + std::env::current_exe().context("failed to resolve current rocm executable path")?; + let mut command = ProcessCommand::new(rocm_binary); + command + .args(engine_serve_http_args( + &engine, + &record.service_id, + &canonical_model_id, + &record.host, + record.port, + &device_policy, + &record.gpu_indices, + runtime_id.as_deref(), + env_id.as_deref(), + engine_recipe_json.as_deref(), + &record.engine_state_path, + )) + .stdin(Stdio::null()) + .stdout(Stdio::from(log_file)) + .stderr(Stdio::from(log_file_err)); + // Re-thread the endpoint key file (public bind only) onto the engine child, + // same as the initial `rocm serve` spawn. This path also runs on daemon + // recovery (`restart_managed_service` re-execs `rocmd supervise`), so + // without this a previously-authenticated public service would come back + // up anonymous after a crash/recover cycle. + // If the key is gone the child would listen on the recorded public host with + // no auth, so fail closed instead — an unreachable service is recoverable, + // an anonymous public one is not. + let endpoint_key_applied = + crate::engine_ipc::apply_endpoint_key_env(&mut command, paths, &record.service_id); + crate::engine_ipc::ensure_public_service_has_endpoint_key( + &record.host, + endpoint_key_applied, + record.requires_api_key, + )?; + let mut child = command + .spawn() + .with_context(|| format!("failed to spawn engine supervisor child for {engine}"))?; + + // Every write from here on goes through `update_supervised_record`: it + // applies only this supervisor's own fields to the record as it is on disk + // now, so a stop marker, or anything else another writer persisted meanwhile, + // survives it. + // + // Captured while the child is known alive, so a later stop verifies this + // exact process instead of whatever has since inherited its PID. + let engine_pid = child.id(); + update_supervised_record(paths, &record, |current| { + current.record_engine_identity(engine_pid); + current.status = "running".to_owned(); + })?; + + let became_ready = wait_for_service_ready( + paths, + &record.engine, + &record.service_id, + &record.log_path, + Duration::from_mins(3), + |phase| { + let _ = update_supervised_record(paths, &record, |current| { + current.startup_phase = Some(phase.to_owned()); + }); + }, + ); + if became_ready { + update_supervised_record(paths, &record, |current| { + current.status = "ready".to_owned(); + // The phase only describes the coming-up window; clear it once ready. + current.startup_phase = None; + })?; + } + + let exit_status = child.wait().context("failed waiting for engine child")?; + record_supervised_exit(paths, &record, exit_status.success())?; + + if exit_status.success() { + Ok(()) + } else { + std::process::exit(exit_status.code().unwrap_or(1)); + } +} + +/// Apply one of [`supervise_service`]'s own changes to its service's record as +/// it is on disk now, rather than writing back the copy the supervisor built at +/// startup. +/// +/// `write` replaces the whole file, and the supervisor's copy is stale the +/// moment anyone else writes: a stop records `stop_requested_unix_ms` before it +/// signals anything, and `rocm`'s liveness refresh and the daemon's restart +/// write here too. Writing the startup snapshot back would erase that stop +/// marker — and a record reading `failed` with no marker is exactly what the +/// daemon's recovery restarts, undoing the stop. So `update` touches only the +/// fields the supervisor owns, and never the marker. +/// +/// Nothing is written when the record no longer belongs to this supervisor: +/// +/// - it is gone — `rocm services remove` deleted it, and writing would +/// resurrect a service the operator removed; +/// - it names a different supervisor process — a confirmed stop clears the +/// recorded PIDs, and a restart records its own. Either way the record now +/// describes something this supervisor is not, and its view of the service +/// (`running`, `failed`, …) would overwrite a truer one. +fn update_supervised_record( + paths: &AppPaths, + supervisor: &ManagedServiceRecord, + update: impl FnOnce(&mut ManagedServiceRecord), +) -> Result<()> { + if !paths.service_manifest_path(&supervisor.service_id).exists() { + return Ok(()); + } + let mut current = crate::persistence::load_service_record(paths, &supervisor.service_id)?; + if current.supervisor_pid != supervisor.supervisor_pid + || current.supervisor_start_ticks != supervisor.supervisor_start_ticks + { + return Ok(()); + } + update(&mut current); + current.write() +} + +/// [`supervise_service`]'s last write: the engine child has exited, so record +/// how. Separate so a test can drive exactly this write. +fn record_supervised_exit( + paths: &AppPaths, + supervisor: &ManagedServiceRecord, + success: bool, +) -> Result<()> { + update_supervised_record(paths, supervisor, |current| { + current.status = if success { "stopped" } else { "failed" }.to_owned(); + }) +} + +#[allow(clippy::too_many_arguments)] +fn engine_serve_http_args( + engine: &str, + service_id: &str, + canonical_model_id: &str, + host: &str, + port: u16, + device_policy: &str, + gpu_indices: &[u32], + runtime_id: Option<&str>, + env_id: Option<&str>, + engine_recipe_json: Option<&str>, + state_path: &Path, +) -> Vec { + let mut args = vec![ + "__engine-serve-http".to_owned(), + engine.to_owned(), + service_id.to_owned(), + canonical_model_id.to_owned(), + "--host".to_owned(), + host.to_owned(), + "--port".to_owned(), + port.to_string(), + "--device-policy".to_owned(), + device_policy.to_owned(), + ]; + if let Some(csv) = rocm_engine_protocol::gpu_indices_to_csv(gpu_indices) { + args.extend(["--gpu".to_owned(), csv]); + } + args.extend(crate::common::optional_arg("--runtime-id", runtime_id)); + args.extend(crate::common::optional_arg("--env-id", env_id)); + args.extend(crate::common::optional_arg( + "--engine-recipe-json", + engine_recipe_json, + )); + args.extend(["--state-path".to_owned(), state_path.display().to_string()]); + args +} + +/// Keep only the final visible segment of a `\r`-redrawn progress line. +/// +/// Progress tools (pip, tqdm, Hugging Face) redraw a line in place with a bare +/// carriage return and no newline, so the segment after the last `\r` is its +/// final visible state. Lines without `\r` pass through unchanged. (Same +/// collapse rule the dashboard job console applies to streamed job output.) +fn last_cr_segment(line: &str) -> &str { + line.rsplit('\r').next().unwrap_or(line) +} + +/// Classify a single serve-log line into a coarse startup phase token +/// (`downloading`/`loading`/`warmup`), or `None` when the line carries no phase +/// signal. Case-insensitive substring match over the common vLLM / llama.cpp / +/// Hugging Face startup vocabulary. Checked warmup → loading → downloading so +/// the latest lifecycle stage a line mentions wins. +fn classify_startup_phase(line: &str) -> Option<&'static str> { + let lower = line.to_ascii_lowercase(); + if lower.contains("capturing cuda graph") + || lower.contains("capturing the model") + || lower.contains("warming up") + || lower.contains("warmup") + { + Some("warmup") + } else if lower.contains("loading weights") + || lower.contains("loading model") + || lower.contains("load_tensors") + || lower.contains("llama_model_loader") + || lower.contains("model loading took") + { + Some("loading") + } else if lower.contains("downloading") || lower.contains("fetching") { + Some("downloading") + } else { + None + } +} + +/// Read log bytes appended since `*pos`, advance `*pos`, and return the most +/// recent recognizable startup phase in that new output (later lines win, so a +/// download → load → warmup progression advances naturally). +/// +/// Best-effort: any I/O error (file not created yet, transient read) yields +/// `None`. A shrunk file (rotation/truncation) resets the cursor to the top. +fn read_new_log_phase(log_path: &Path, pos: &mut u64) -> Option<&'static str> { + use std::io::{Read, Seek, SeekFrom}; + let mut file = fs::File::open(log_path).ok()?; + let len = file.metadata().ok()?.len(); + if len < *pos { + *pos = 0; + } + if len == *pos { + return None; + } + file.seek(SeekFrom::Start(*pos)).ok()?; + let mut bytes = Vec::new(); + let read = file.read_to_end(&mut bytes).ok()?; + *pos += read as u64; + let text = String::from_utf8_lossy(&bytes); + let mut phase = None; + for line in text.lines() { + if let Some(found) = classify_startup_phase(last_cr_segment(line)) { + phase = Some(found); + } + } + phase +} + +fn engine_healthcheck_ready(paths: &AppPaths, engine: &str, service_id: &str) -> Result { + Ok(crate::engine_ipc::healthcheck_response_ready( + &crate::engine_ipc::engine_healthcheck_response(paths, engine, service_id)?, + )) +} + +/// Poll a freshly-spawned service until its healthcheck reports ready (or the +/// timeout elapses), tailing its log file meanwhile and reporting each coarse +/// startup phase transition via `on_phase`. +fn wait_for_service_ready( + paths: &AppPaths, + engine: &str, + service_id: &str, + log_path: &Path, + timeout: Duration, + mut on_phase: impl FnMut(&str), +) -> bool { + let start = std::time::Instant::now(); + let mut log_pos: u64 = 0; + let mut last_phase: Option<&'static str> = None; + while start.elapsed() < timeout { + if let Some(phase) = read_new_log_phase(log_path, &mut log_pos) + && last_phase != Some(phase) + { + last_phase = Some(phase); + on_phase(phase); + } + if engine_healthcheck_ready(paths, engine, service_id).unwrap_or(false) { + return true; + } + thread::sleep(Duration::from_millis(200)); + } + false +} + +pub(crate) fn restart_managed_service( + paths: &AppPaths, + record: &mut ManagedServiceRecord, +) -> Result<()> { + let rocmd_binary = + std::env::current_exe().context("failed to resolve current rocmd executable path")?; + let log_file = fs::OpenOptions::new() + .create(true) + .append(true) + .open(&record.log_path) + .with_context(|| format!("failed to open {}", record.log_path.display()))?; + let log_file_err = log_file + .try_clone() + .context("failed to clone service log file handle")?; + + record.status = "recovering".to_owned(); + // Counts the restart and drops the previous run's inference verification. + // The respawned child writes a fresh record of its own, and "recovering" is + // outside the statuses that probe, so a stale verdict would not currently be + // acted on — but this record is written again below, after the spawn, and + // that write can land after the child's. Clearing here keeps the invariant + // true at the one site that reuses a record across restarts. + record.reset_for_restart(); + // No supervisor exists until the spawn below succeeds, so none is recorded. + // Recording this daemon's own PID here, as a stand-in, left the manifest + // naming the daemon as the service's supervisor whenever the spawn failed: + // a stop run from any other process (`rocmd sandbox-tool stop_server`, + // `rocm services stop`) verifies that identity, finds it live, and + // terminates the daemon's whole process tree. The spawned supervisor's + // identity is recorded right after the spawn. + record.supervisor_pid = 0; + record.supervisor_start_ticks = None; + // The engine PID belonged to the run being replaced, and nothing here + // starts an engine: the supervisor spawned below records its own, together + // with its token. Carried forward, the old PID would sit beside a + // supervisor it no longer matches — and on a record `rocm` wrote, with no + // token at all, so the next stop could only signal it blind. + record.engine_pid = None; + record.engine_start_ticks = None; + // Restarting is a request to run, so it supersedes any earlier stop request, + // as `rocm services restart` treats it too. + record.stop_requested_unix_ms = None; + record.write()?; + + let mut child = detached_rocmd_command(&rocmd_binary) + .args(recovery_supervise_args(record)) + .stdin(Stdio::null()) + .stdout(Stdio::from(log_file)) + .stderr(Stdio::from(log_file_err)) + .spawn() + .context("failed to spawn recovery supervisor")?; + + record_spawned_supervisor(paths, record, child.id())?; + + thread::sleep(Duration::from_millis(200)); + if let Some(status) = child + .try_wait() + .context("failed to check recovery supervisor startup state")? + { + // Not a write-back of `record`: a stop issued since the spawn has + // persisted its marker, and a `failed` record without one is what + // recovery restarts. Same discipline as the supervisor's own writes. + update_supervised_record(paths, record, |current| { + current.status = "failed".to_owned(); + })?; + anyhow::bail!( + "recovery supervisor exited immediately with status {status}; inspect {}", + record.log_path.display() + ); + } + + Ok(()) +} + +/// Record the supervisor [`restart_managed_service`] just spawned, as `pid`. +/// +/// Between the restart's first write — which clears the recorded PIDs, since +/// no supervisor exists yet — and this one, the record names no process at +/// all. A stop that starts in that window persists its marker first and then +/// finds nothing to signal; writing the restart's whole copy back here would +/// erase that marker while the new supervisor comes up, and leave the stop's +/// own final write comparing against a record that no longer shows the +/// process it has to answer for. So only the supervisor's identity is applied, +/// to the record as it is on disk now, through the same ownership guard as the +/// supervisor's own writes: against the copy from before this assignment, so +/// it is the cleared PID that has to still be there. If the new supervisor has +/// already recorded itself, or the record was removed, nothing is written. +fn record_spawned_supervisor( + paths: &AppPaths, + record: &mut ManagedServiceRecord, + pid: u32, +) -> Result<()> { + let unrecorded = record.clone(); + record.record_supervisor_identity(pid); + let (supervisor_pid, supervisor_start_ticks) = + (record.supervisor_pid, record.supervisor_start_ticks); + update_supervised_record(paths, &unrecorded, |current| { + current.supervisor_pid = supervisor_pid; + current.supervisor_start_ticks = supervisor_start_ticks; + }) +} + +fn recovery_supervise_args(record: &ManagedServiceRecord) -> Vec { + let mut args = vec![ + "supervise".to_owned(), + record.service_id.clone(), + "--engine".to_owned(), + record.engine.clone(), + "--model-ref".to_owned(), + record.model_ref.clone(), + "--canonical-model-id".to_owned(), + record.canonical_model_id.clone(), + "--host".to_owned(), + record.host.clone(), + "--port".to_owned(), + record.port.to_string(), + "--device-policy".to_owned(), + record + .device_policy + .as_deref() + .unwrap_or("gpu_required") + .to_owned(), + ]; + args.extend(crate::common::optional_arg( + "--runtime-id", + record.runtime_id.as_deref(), + )); + args.extend(crate::common::optional_arg( + "--env-id", + record.env_id.as_deref(), + )); + if let Some(csv) = rocm_engine_protocol::gpu_indices_to_csv(&record.gpu_indices) { + args.extend(["--gpu".to_owned(), csv]); + } + args.extend(crate::common::optional_arg( + "--engine-recipe-json", + record.engine_recipe_json.as_deref(), + )); + args +} + +#[cfg(unix)] +fn detached_rocmd_command(rocmd_binary: &std::path::Path) -> ProcessCommand { + let mut command = ProcessCommand::new("setsid"); + command.arg(rocmd_binary); + command +} + +#[cfg(not(unix))] +fn detached_rocmd_command(rocmd_binary: &std::path::Path) -> ProcessCommand { + ProcessCommand::new(rocmd_binary) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::persistence::load_service_record; + #[cfg(target_os = "linux")] + use crate::test_support::UNCONFIRMED_STOP_PID; + use crate::test_support::{identity_probe_record, temp_app_paths, unique_test_root}; + use crate::watchers::find_recoverable_service; + + /// Drive `supervise_service` far enough to reach the key guard, and return + /// what it did. + /// + /// The guard sits before the manifest write and well before any spawn, so a + /// refusal returns without starting a process — which is what makes the real + /// call site testable at all. The arguments below are the shape a recovery + /// re-exec passes: a loopback bind, no GPU, no recipe. + /// + /// This exists because testing `ensure_public_service_has_endpoint_key` + /// directly with literal arguments cannot catch the defect that actually + /// happened twice in this crate's history — the guard being *wired up* with + /// the wrong value at its call site. + /// + /// Bounded, and the bound is the assertion. A guard that fails to refuse + /// does not return an error — it falls through to the engine spawn and + /// supervises a child that never exits, so an unbounded call would hang the + /// suite instead of failing it. Both callers below are regression tests for + /// a fail-*open*, which is exactly the shape that turns into a hang. + fn supervise_at_the_guard(paths: &AppPaths, service_id: &str) -> Result<()> { + let paths = paths.clone(); + let service_id = service_id.to_owned(); + let (sender, receiver) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let outcome = supervise_service( + &paths, + service_id, + "llamacpp".to_owned(), + "a-model".to_owned(), + "a-model".to_owned(), + None, + None, + "127.0.0.1".to_owned(), + 11434, + "gpu_required".to_owned(), + None, + None, + ); + let _ = sender.send(outcome.map_err(|error| format!("{error:#}"))); + }); + receiver + .recv_timeout(std::time::Duration::from_secs(30)) + .unwrap_or_else(|_| { + panic!( + "supervise_service did not return within 30s: the key guard let the call \ + through and it reached the engine spawn, which is the fail-open this test \ + exists to catch" + ) + }) + .map_err(anyhow::Error::msg) + } + + /// Write a service record into the registry the way a live service would + /// have left it behind. + fn seed_registry(paths: &AppPaths, service_id: &str, requires_api_key: bool) { + fs::create_dir_all(paths.services_dir()).unwrap(); + let mut record = ManagedServiceRecord::new( + paths, + service_id.to_owned(), + "llamacpp".to_owned(), + "a-model".to_owned(), + "a-model".to_owned(), + "127.0.0.1".to_owned(), + 11434, + "managed", + std::process::id(), + None, + None, + Some("gpu_required".to_owned()), + ); + record.requires_api_key = requires_api_key; + record.write().unwrap(); + } + + /// Drive `supervise_service` and report whether the key guard let it past. + /// + /// Decided on what the call *returns*, not on any file. The obvious + /// observable — the manifest appearing — is useless here, because + /// `seed_registry` has already written one, so polling for it passes + /// whatever the guard does. That mistake was made first and caught by + /// mutating the code the test claims to protect. + /// + /// A call the guard admits does not return: it carries on to the engine + /// spawn. So the guard's refusal is the only thing that comes back quickly, + /// and it is identified by its message rather than by the mere fact of an + /// error — a later, unrelated failure must not read as a refusal. + fn guard_admits(paths: &AppPaths, service_id: &str) -> bool { + let owned_paths = paths.clone(); + let owned_id = service_id.to_owned(); + let (sender, receiver) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let outcome = supervise_service( + &owned_paths, + owned_id, + "llamacpp".to_owned(), + "a-model".to_owned(), + "a-model".to_owned(), + None, + None, + "127.0.0.1".to_owned(), + 11434, + "gpu_required".to_owned(), + None, + None, + ); + let _ = sender.send(outcome.map_err(|error| format!("{error:#}"))); + }); + + match receiver.recv_timeout(std::time::Duration::from_secs(10)) { + // The key guard refused, by its own words. + Ok(Err(rendered)) if rendered.contains("without authentication") => false, + // Anything else means it got past the guard: it either finished, or + // failed later for a reason that is not this guard, or is still + // running because it reached the spawn. + _ => true, + } + } + + #[test] + fn a_service_that_never_required_a_key_is_not_refused_for_lacking_one() { + // The other direction of the guard, and the one no test covered. + // Hardcoding `record.requires_api_key = true` at the restore site passes + // every other test in this crate, because they all seed a service that + // *does* require a key. This is the case that catches it. + // + // Two records are seeded, not one: with a single record the + // `existing.service_id == record.service_id` half of the lookup does + // nothing, so dropping that comparison would go unnoticed and one + // service's requirement would leak onto another's. + let (root, paths) = temp_app_paths("supervise-no-key-needed"); + seed_registry(&paths, "svc-needs-key", true); + seed_registry(&paths, "svc-plain", false); + + assert!( + guard_admits(&paths, "svc-plain"), + "a loopback service that never asked for a key must not be refused for lacking one" + ); + + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn supervising_a_service_that_required_a_key_refuses_when_the_key_is_gone() { + // The real call site, not the guard in isolation. `supervise_service` + // rebuilds the record with `ManagedServiceRecord::new`, which starts + // `requires_api_key` false, and restores it from the registry. Passing + // the wrong value here — a literal, or the freshly-built field before it + // is restored — is exactly the miswiring that shipped twice in this + // crate and that a literal-argument unit test cannot see. + let (root, paths) = temp_app_paths("supervise-requires-key"); + seed_registry(&paths, "svc-needs-key", true); + + let error = supervise_at_the_guard(&paths, "svc-needs-key") + .expect_err("a service that required a key must not be recovered without one"); + assert!( + format!("{error:#}").contains("without authentication"), + "{error:#}" + ); + + // And the refusal must not have weakened what is on disk. The guard runs + // before `record.write()` precisely so a refused attempt leaves the + // recorded requirement armed for the next attempt. + let stored = crate::persistence::load_managed_services(&paths).unwrap(); + let stored = stored + .iter() + .find(|candidate| candidate.service_id == "svc-needs-key") + .expect("the seeded record must survive a refused recovery"); + assert!( + stored.requires_api_key, + "a refused recovery must not disarm the requirement" + ); + + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn a_registry_that_cannot_be_read_refuses_recovery_rather_than_assuming_no_key() { + // `load_managed_services` already skips records it cannot parse, so an + // `Err` from it is a real I/O failure — and a missing directory is `Ok`. + // Defaulting it away therefore says "no service ever required a key", + // which is fail-open on an auth gate and, worse, gets written back. + // + // The failure is provoked portably: a directory named like a record + // makes the `fs::read` inside the loop fail rather than the read_dir. + let (root, paths) = temp_app_paths("supervise-unreadable-registry"); + fs::create_dir_all(paths.services_dir().join("not-a-record.json")).unwrap(); + + let error = supervise_at_the_guard(&paths, "svc-unknown") + .expect_err("an unreadable registry must refuse, not assume no key was required"); + let rendered = format!("{error:#}"); + assert!( + rendered.contains("could not read the service registry"), + "{rendered}" + ); + + // Nothing was written: a registry we could not read is not a registry we + // may add a weakened record to. + assert!( + !paths.service_manifest_path("svc-unknown").exists(), + "a refused recovery must not persist a record" + ); + + let _ = fs::remove_dir_all(&root); + } + + #[test] + fn last_cr_segment_keeps_final_progress_redraw() { + // A tqdm/HF-style in-place redraw collapses to its last segment. + assert_eq!( + last_cr_segment("Downloading: 10%\rDownloading: 55%\rDownloading: 100%"), + "Downloading: 100%" + ); + // A plain line is unchanged. + assert_eq!( + last_cr_segment("Loading model weights"), + "Loading model weights" + ); + } + + #[test] + fn classify_startup_phase_maps_engine_vocabulary() { + assert_eq!( + classify_startup_phase("Downloading shards: 100%"), + Some("downloading") + ); + assert_eq!( + classify_startup_phase("Fetching 12 files"), + Some("downloading") + ); + assert_eq!( + classify_startup_phase("INFO: Loading model weights took 4.2s"), + Some("loading") + ); + assert_eq!( + classify_startup_phase("llama_model_loader: loaded meta data"), + Some("loading") + ); + assert_eq!( + classify_startup_phase("Capturing CUDA graph shapes"), + Some("warmup") + ); + assert_eq!( + classify_startup_phase("Warming up the engine"), + Some("warmup") + ); + // Ordinary chatter carries no phase signal. + assert_eq!( + classify_startup_phase("Uvicorn running on http://..."), + None + ); + } + + #[test] + fn classify_startup_phase_emits_only_dashboard_known_tokens() { + // These tokens are the wire contract with the dashboard's + // `StartupPhase::from_token` (rocm-dash-core); emitting anything else + // would be silently dropped there. rocmd can't link that crate, so the + // contract is pinned here by literal. + for line in [ + "Downloading shards", + "Loading model weights", + "Capturing CUDA graph", + ] { + let token = classify_startup_phase(line).expect("line is a phase signal"); + assert!( + matches!(token, "downloading" | "loading" | "warmup"), + "token {token:?} must be one the dashboard understands" + ); + } + } + + #[test] + fn read_new_log_phase_advances_and_tracks_latest() { + use std::io::Write as _; + // Workspace-local test root (rooted at CARGO_MANIFEST_DIR, not the + // ambient temp dir) — same helper the other rocmd tests use. + let dir = unique_test_root(&format!("rocmd-phase-{}", std::process::id())); + let log = dir.join("svc.log"); + std::fs::write(&log, "boot\nDownloading shards: 100%\n").unwrap(); + + let mut pos = 0_u64; + assert_eq!(read_new_log_phase(&log, &mut pos), Some("downloading")); + // No new bytes → no phase, cursor unchanged. + let after_first = pos; + assert_eq!(read_new_log_phase(&log, &mut pos), None); + assert_eq!(pos, after_first); + + // Appending a later stage advances the phase. + let mut f = std::fs::OpenOptions::new().append(true).open(&log).unwrap(); + writeln!(f, "Loading model weights took 3s").unwrap(); + assert_eq!(read_new_log_phase(&log, &mut pos), Some("loading")); + + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn engine_serve_http_args_forward_engine_recipe_json() { + let engine_recipe_json = r#"{"contract_version":"0.1.0","engine":"vllm","required_flags":["--enable-auto-tool-choice"]}"#; + let args = engine_serve_http_args( + "vllm", + "svc-1", + "Qwen/Qwen3.5-4B", + "127.0.0.1", + 11435, + "gpu_required", + &[], + Some("therock-release:gfx120X-all"), + Some("env-1"), + Some(engine_recipe_json), + Path::new("state.json"), + ); + + assert!( + args.windows(2) + .any(|pair| pair[0] == "--engine-recipe-json" && pair[1] == engine_recipe_json) + ); + assert!( + args.windows(2).any(|pair| { + pair[0] == "--runtime-id" && pair[1] == "therock-release:gfx120X-all" + }) + ); + assert!( + args.windows(2) + .any(|pair| pair[0] == "--state-path" && pair[1] == "state.json") + ); + } + + #[test] + fn engine_serve_http_args_emit_gpu_indices_when_pinned() { + let args = engine_serve_http_args( + "vllm", + "svc-1", + "Qwen/Qwen3.5-4B", + "127.0.0.1", + 11435, + "gpu_required", + &[1], + None, + None, + None, + Path::new("state.json"), + ); + + assert!( + args.windows(2) + .any(|pair| pair[0] == "--gpu" && pair[1] == "1") + ); + + let auto = engine_serve_http_args( + "vllm", + "svc-1", + "Qwen/Qwen3.5-4B", + "127.0.0.1", + 11435, + "gpu_required", + &[], + None, + None, + None, + Path::new("state.json"), + ); + assert!(!auto.iter().any(|arg| arg == "--gpu")); + } + + /// The identity check is only as good as the token the launcher writes down. + /// `rocm_core::identity_state` treats a missing token as a best-effort match + /// — the legacy `(None, _)` branch — so a record written without one is + /// signalled blind, which is the original defect restored in full. Every + /// other test here hands itself a record with the token already set, and so + /// proves nothing about the code that is supposed to put it there. + /// + /// This drives the real `supervise_service` and reads the manifest it + /// persisted. It is stopped at that first write by making the service log + /// path a directory, so the `fs::File::create` immediately after it fails: + /// going further reaches the engine spawn, and the failure path past that + /// calls `std::process::exit`, which inside a test binary would take the + /// whole suite with it. + /// + /// Linux-only: `rocm_core::process_start_ticks` reads `/proc` and returns + /// `None` everywhere else, so on other platforms there is no token to write. + #[cfg(target_os = "linux")] + #[test] + fn supervise_service_persists_the_supervisor_identity_token() -> Result<()> { + let (root, paths) = temp_app_paths("supervise-records-identity"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-supervise-identity"; + let log_path = identity_probe_record(&paths, service_id, 11443).log_path; + // A directory where the supervisor expects to create a file. + fs::create_dir_all(&log_path)?; + + let outcome = supervise_service( + &paths, + service_id.to_owned(), + "llamacpp".to_owned(), + "a-model".to_owned(), + "a-model".to_owned(), + None, + None, + "127.0.0.1".to_owned(), + 11443, + "gpu_required".to_owned(), + None, + None, + ); + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let error = + outcome.expect_err("the log path is a directory, so the engine spawn is unreachable"); + assert!( + format!("{error:#}").contains("failed to create"), + "stopped somewhere other than the log file: {error:#}" + ); + assert!( + persisted?.supervisor_start_ticks.is_some(), + "supervise_service must persist the supervisor's start-time token beside its PID" + ); + Ok(()) + } + + /// The supervisor's last write lands after its engine child exits — which + /// is what a stop that could not verify the supervisor, but did kill the + /// engine, leaves behind. Writing back the record the supervisor built at + /// startup would erase the stop marker the stop persisted, and a `failed` + /// record with no marker is exactly what the daemon's recovery restarts. + #[test] + fn supervisor_exit_write_keeps_a_standing_stop_marker() -> Result<()> { + let (root, paths) = temp_app_paths("supervise-exit-keeps-marker"); + paths.ensure()?; + let service_id = "svc-supervise-exit-marker"; + // The supervisor's own copy, as `supervise_service` builds it: no marker. + let mut supervisor = identity_probe_record(&paths, service_id, 11446); + supervisor.record_supervisor_identity(std::process::id()); + supervisor.status = "running".to_owned(); + // What is on disk once a stop has run: same processes, marker set. + let mut on_disk = supervisor.clone(); + on_disk.stop_requested_unix_ms = Some(1); + on_disk.write()?; + + record_supervised_exit(&paths, &supervisor, false)?; + let persisted = load_service_record(&paths, service_id); + let recoverable = find_recoverable_service(&paths); + fs::remove_dir_all(root).ok(); + + let persisted = persisted?; + assert_eq!( + persisted.status, "failed", + "the supervisor's own field lands" + ); + assert_eq!( + persisted.stop_requested_unix_ms, + Some(1), + "the supervisor's exit write must not withdraw a standing stop" + ); + assert!( + recoverable?.is_none(), + "a stopped service must not become a recovery candidate" + ); + Ok(()) + } + + /// A record that names another supervisor — a confirmed stop cleared the + /// PIDs, or a restart recorded its own — or that was removed outright, is + /// no longer this supervisor's to write. + #[test] + fn supervisor_exit_write_leaves_a_record_it_no_longer_owns() -> Result<()> { + let (root, paths) = temp_app_paths("supervise-exit-not-owner"); + paths.ensure()?; + let service_id = "svc-supervise-exit-not-owner"; + let mut supervisor = identity_probe_record(&paths, service_id, 11447); + supervisor.record_supervisor_identity(std::process::id()); + supervisor.status = "running".to_owned(); + // A confirmed stop's final write: status stopped, PIDs cleared. + let mut stopped = supervisor.clone(); + stopped.status = "stopped".to_owned(); + stopped.supervisor_pid = 0; + stopped.supervisor_start_ticks = None; + stopped.write()?; + + record_supervised_exit(&paths, &supervisor, false)?; + let after_stop = load_service_record(&paths, service_id); + + fs::remove_file(paths.service_manifest_path(service_id))?; + record_supervised_exit(&paths, &supervisor, false)?; + let resurrected = paths.service_manifest_path(service_id).exists(); + fs::remove_dir_all(root).ok(); + + assert_eq!(after_stop?.status, "stopped"); + assert!(!resurrected, "a removed record must not be written back"); + Ok(()) + } + + /// The supervisor's first write rebuilds the record from its arguments. + /// The restart that launched it cleared any stop marker first, so a marker + /// on disk here was set by a stop issued since, and must survive. + #[cfg(target_os = "linux")] + #[test] + fn supervise_service_first_write_keeps_a_standing_stop_marker() -> Result<()> { + let (root, paths) = temp_app_paths("supervise-first-write-keeps-marker"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-supervise-first-write-marker"; + let mut seeded = identity_probe_record(&paths, service_id, 11448); + seeded.stop_requested_unix_ms = Some(1); + seeded.write()?; + // Stop `supervise_service` right after its first write; see + // `supervise_service_persists_the_supervisor_identity_token`. + fs::create_dir_all(&seeded.log_path)?; + + let outcome = supervise_service( + &paths, + service_id.to_owned(), + "llamacpp".to_owned(), + "a-model".to_owned(), + "a-model".to_owned(), + None, + None, + "127.0.0.1".to_owned(), + 11448, + "gpu_required".to_owned(), + None, + None, + ); + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + assert!(outcome.is_err(), "the log path is a directory"); + assert_eq!(persisted?.stop_requested_unix_ms, Some(1)); + Ok(()) + } + + #[test] + fn recovery_supervise_args_preserve_engine_recipe_json() { + let (_root, paths) = temp_app_paths("recovery-engine-recipe"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-1", + "vllm", + "qwen", + "Qwen/Qwen3.5-4B", + "127.0.0.1", + 11435, + "managed", + 123, + Some("therock-release:gfx120X-all".to_owned()), + Some("env-1".to_owned()), + Some("gpu_required".to_owned()), + ); + let engine_recipe_json = r#"{"contract_version":"0.1.0","engine":"vllm","required_flags":["--enable-auto-tool-choice"]}"#; + record.engine_recipe_json = Some(engine_recipe_json.to_owned()); + + let args = recovery_supervise_args(&record); + + assert!( + args.windows(2) + .any(|pair| pair[0] == "--engine-recipe-json" && pair[1] == engine_recipe_json) + ); + assert!( + args.windows(2) + .any(|pair| { pair[0] == "--canonical-model-id" && pair[1] == "Qwen/Qwen3.5-4B" }) + ); + } + + /// The same guarantee for the daemon's recovery path, which records a PID of + /// its own. See `supervise_service_persists_the_supervisor_identity_token` + /// for why an unrecorded token is the defect rather than a cosmetic gap. + /// + /// The restart re-execs `rocmd supervise`, which here is the test binary: + /// it rejects those arguments and exits at once, so the restart reports + /// failure — after it has written the record this test reads. + #[cfg(target_os = "linux")] + #[test] + fn restart_managed_service_persists_the_supervisor_identity_token() -> Result<()> { + let (root, paths) = temp_app_paths("restart-records-identity"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-restart-identity"; + let mut record = identity_probe_record(&paths, service_id, 11444); + record.status = "failed".to_owned(); + record.write()?; + assert_eq!( + load_service_record(&paths, service_id)?.supervisor_start_ticks, + None, + "precondition: the seeded record carries no token" + ); + + let _ = restart_managed_service(&paths, &mut record); + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + assert!( + persisted?.supervisor_start_ticks.is_some(), + "restart_managed_service must persist the supervisor's start-time token beside its PID" + ); + Ok(()) + } + + /// A restart moves the supervisor PID; the engine PID it carried belonged + /// to the run being replaced. On a record `rocm serve --background` wrote, + /// that PID has no token, so carrying it past the point where the two PIDs + /// diverge leaves the next stop an entry it can only signal blind. + /// + /// Linux-only for the same reason as the test above: the restart re-execs + /// the test binary, which is only known to reject those arguments there. + #[cfg(target_os = "linux")] + #[test] + fn restart_managed_service_drops_the_replaced_runs_engine_pid() -> Result<()> { + let (root, paths) = temp_app_paths("restart-drops-engine-pid"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-restart-drops-engine-pid"; + let mut record = identity_probe_record(&paths, service_id, 11455); + // The shape `rocm serve --background` writes: one PID in both roles. + record.supervisor_pid = UNCONFIRMED_STOP_PID; + record.engine_pid = Some(UNCONFIRMED_STOP_PID); + record.engine_start_ticks = None; + record.stop_requested_unix_ms = Some(1); + record.status = "failed".to_owned(); + record.write()?; + + let _ = restart_managed_service(&paths, &mut record); + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let persisted = persisted?; + assert_ne!(persisted.supervisor_pid, UNCONFIRMED_STOP_PID); + assert_eq!( + persisted.engine_pid, None, + "the replaced run's engine PID must not outlive the restart" + ); + assert_eq!(persisted.engine_start_ticks, None); + assert_eq!( + persisted.stop_requested_unix_ms, None, + "a restart supersedes an earlier stop request" + ); + Ok(()) + } + + /// A stop that lands while a restart waits to see whether its supervisor + /// survived startup must outlive the restart's `failed` write: a `failed` + /// record without the marker is exactly what recovery restarts. + /// + /// The restarted supervisor is the test binary, which rejects the + /// `supervise` arguments and exits at once, so the restart reaches that + /// write. A thread plays the stop: once the restart has recorded the + /// spawned supervisor, it persists a marker, as a stop does first. + #[cfg(target_os = "linux")] + #[test] + fn restart_failure_write_keeps_a_stop_marker_set_meanwhile() -> Result<()> { + let (root, paths) = temp_app_paths("restart-failure-keeps-marker"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + let service_id = "svc-restart-failure-marker"; + let mut record = identity_probe_record(&paths, service_id, 11456); + record.status = "failed".to_owned(); + record.write()?; + + let stop = { + let paths = paths.clone(); + thread::spawn(move || -> Result { + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while std::time::Instant::now() < deadline { + // `write` is not atomic, so a read can land mid-write and + // see a truncated file; just look again. + let Ok(mut on_disk) = load_service_record(&paths, service_id) else { + thread::sleep(Duration::from_millis(5)); + continue; + }; + if on_disk.supervisor_pid != 0 && on_disk.supervisor_pid != std::process::id() { + on_disk.stop_requested_unix_ms = Some(1); + on_disk.write()?; + return Ok(true); + } + thread::sleep(Duration::from_millis(5)); + } + Ok(false) + }) + }; + let outcome = restart_managed_service(&paths, &mut record); + let stop_landed = stop.join().expect("stop thread"); + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + assert!( + stop_landed?, + "precondition: the stop ran during the restart" + ); + assert!(outcome.is_err(), "the test binary is not a supervisor"); + let persisted = persisted?; + assert_eq!(persisted.status, "failed"); + assert_eq!( + persisted.stop_requested_unix_ms, + Some(1), + "the restart's failure write must not withdraw a stop issued meanwhile" + ); + Ok(()) + } + + /// Between a restart's first write, which clears the recorded PIDs, and the + /// one recording the supervisor it spawned, a stop can persist its marker. + /// Writing the restart's own copy back would erase it while the new + /// supervisor comes up — so only the identity is applied. + #[test] + fn restart_records_its_supervisor_without_erasing_a_stop_marker() -> Result<()> { + let (root, paths) = temp_app_paths("restart-spawn-keeps-marker"); + paths.ensure()?; + let service_id = "svc-restart-spawn-marker"; + // The restart's copy after its first write: no supervisor, no marker. + let mut restarting = identity_probe_record(&paths, service_id, 11457); + restarting.status = "recovering".to_owned(); + restarting.supervisor_pid = 0; + restarting.supervisor_start_ticks = None; + // A stop started since: same (empty) processes, marker set. + let mut on_disk = restarting.clone(); + on_disk.stop_requested_unix_ms = Some(1); + on_disk.write()?; + + record_spawned_supervisor(&paths, &mut restarting, std::process::id())?; + let persisted = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let persisted = persisted?; + assert_eq!(restarting.supervisor_pid, std::process::id()); + assert_eq!(persisted.supervisor_pid, std::process::id()); + assert_eq!( + persisted.supervisor_start_ticks, + restarting.supervisor_start_ticks + ); + assert_eq!( + persisted.stop_requested_unix_ms, + Some(1), + "recording the spawned supervisor must not withdraw a standing stop" + ); + Ok(()) + } + + /// The spawned supervisor may record itself before the restart does, and + /// the record may be removed outright; either way the restart's write is + /// not the one to land. + #[test] + fn restart_leaves_a_record_naming_another_supervisor_or_none() -> Result<()> { + let (root, paths) = temp_app_paths("restart-spawn-not-owner"); + paths.ensure()?; + let service_id = "svc-restart-spawn-not-owner"; + let mut restarting = identity_probe_record(&paths, service_id, 11458); + restarting.supervisor_pid = 0; + restarting.supervisor_start_ticks = None; + let mut on_disk = restarting.clone(); + on_disk.supervisor_pid = 4_242_424; + on_disk.write()?; + + record_spawned_supervisor(&paths, &mut restarting.clone(), std::process::id())?; + let after_supervisor = load_service_record(&paths, service_id); + + fs::remove_file(paths.service_manifest_path(service_id))?; + record_spawned_supervisor(&paths, &mut restarting, std::process::id())?; + let resurrected = paths.service_manifest_path(service_id).exists(); + fs::remove_dir_all(root).ok(); + + assert_eq!(after_supervisor?.supervisor_pid, 4_242_424); + assert!(!resurrected, "a removed record must not be written back"); + Ok(()) + } +} diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index 5ec222daa4..cd10cc0a41 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -2,17 +2,19 @@ // // SPDX-License-Identifier: MIT +use crate::common::payload_string; +use crate::persistence::load_service_record; +use crate::supervise::restart_managed_service; use anyhow::{Context, Result, bail}; use rocm_core::{ AppPaths, AutomationProposalRecord, AutomationRuntimeState, AutomationTriggerEvent, CodexBridgeGpuSnapshot, ManagedServiceRecord, RocmCliConfig, WatcherMode, - WatcherRuntimeSnapshot, append_automation_proposal, builtin_watchers, + WatcherRuntimeSnapshot, append_automation_proposal, builtin_watchers, format_host_port, resolve_model_recipe_artifact, unix_time_millis, }; use serde_json::Value; use serde_json::json; -use std::fs; -use std::process::{Command as ProcessCommand, Stdio}; +use std::net::{SocketAddr, TcpStream}; use std::thread; use std::time::Duration; @@ -63,7 +65,7 @@ fn collect_automation_events( state: &AutomationRuntimeState, ) -> Result> { collect_automation_events_with_gpu_snapshot(paths, state, || { - crate::common::gather_gpu_snapshot_for_config(config) + crate::host_snapshot::gather_gpu_snapshot_for_config(config) }) } @@ -237,11 +239,11 @@ fn handle_therock_update_event( handle_therock_update_event_with_runner(paths, mode, state, event, |paths| { crate::sandbox::run_sandbox_tool( paths, - crate::cli::SandboxToolArg::CheckUpdates, + crate::sandbox_tool::SandboxToolArg::CheckUpdates, None, None, None, - crate::cli::SandboxToolPolicy::default(), + crate::sandbox_tool::SandboxToolPolicy::default(), ) }) } @@ -362,7 +364,7 @@ fn restricted_check_updates_result(value: &Value) -> Result Result Result Option { - payload - .get(key) - .and_then(Value::as_str) - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(str::to_owned) -} - #[cfg(test)] fn evaluate_server_recover( paths: &AppPaths, @@ -1207,12 +1200,12 @@ fn service_record_matches_recovery_event( manifest_service_recovery_reason(record, unix_time_millis()).is_some() } "service.endpoint_recoverable" => endpoint_service_recovery_reason(record).is_some(), - "service.healthcheck_recoverable" => { - crate::common::engine_healthcheck_response(paths, &record.engine, &record.service_id) - .is_ok_and(|healthcheck| { - crate::common::healthcheck_response_recoverable(&healthcheck) - }) - } + "service.healthcheck_recoverable" => crate::engine_ipc::engine_healthcheck_response( + paths, + &record.engine, + &record.service_id, + ) + .is_ok_and(|healthcheck| crate::engine_ipc::healthcheck_response_recoverable(&healthcheck)), _ => false, } } @@ -1312,7 +1305,7 @@ fn handle_server_recover_event_with_record( // Report it and stop, rather than letting a permanent failure // propagate out of `evaluate_watchers` and take the whole daemon — // and every other watcher — down on each 30s recovery tick. - if let Err(error) = crate::common::ensure_public_service_has_endpoint_key( + if let Err(error) = crate::engine_ipc::ensure_public_service_has_endpoint_key( &record.host, rocm_engine_protocol::endpoint_key_file_if_present(paths, &record.service_id) .and_then(|path| rocm_engine_protocol::endpoint_api_key_file_if_valid(&path)) @@ -1416,7 +1409,7 @@ pub(crate) fn find_recoverable_service( return Ok(Some((record, reason))); } if matches!(record.status.as_str(), "ready" | "running") { - let Ok(healthcheck) = crate::common::engine_healthcheck_response( + let Ok(healthcheck) = crate::engine_ipc::engine_healthcheck_response( paths, &record.engine, &record.service_id, @@ -1426,7 +1419,7 @@ pub(crate) fn find_recoverable_service( } continue; }; - if crate::common::healthcheck_response_recoverable(&healthcheck) { + if crate::engine_ipc::healthcheck_response_recoverable(&healthcheck) { return Ok(Some(( record, format!("healthcheck_status_{}", healthcheck.status), @@ -1455,33 +1448,24 @@ pub(crate) const fn stop_requested(record: &ManagedServiceRecord) -> bool { } fn endpoint_service_recovery_reason(record: &ManagedServiceRecord) -> Option { - (!crate::common::wait_for_port(&record.host, record.port, ENDPOINT_HEALTH_TIMEOUT)) + (!wait_for_port(&record.host, record.port, ENDPOINT_HEALTH_TIMEOUT)) .then(|| "endpoint_status_unreachable".to_owned()) } -pub(crate) fn load_service_record( - paths: &AppPaths, - service_id: &str, -) -> Result { - rocm_core::ServiceId::new(service_id) - .with_context(|| format!("invalid managed service id `{service_id}`"))?; - let manifest_path = paths.service_manifest_path(service_id); - let bytes = fs::read(&manifest_path).with_context(|| { - format!( - "managed service `{service_id}` not found at {}", - manifest_path.display() - ) - })?; - let record = serde_json::from_slice::(&bytes) - .with_context(|| format!("failed to parse {}", manifest_path.display()))?; - if record.service_id != service_id { - bail!( - "managed service manifest {} contains service_id `{}`, expected `{service_id}`", - manifest_path.display(), - record.service_id - ); +fn wait_for_port(host: &str, port: u16, timeout: Duration) -> bool { + let address: SocketAddr = match format_host_port(host, port).parse() { + Ok(value) => value, + Err(_) => return false, + }; + + let start = std::time::Instant::now(); + while start.elapsed() < timeout { + if TcpStream::connect_timeout(&address, Duration::from_millis(200)).is_ok() { + return true; + } + thread::sleep(Duration::from_millis(200)); } - Ok(record) + false } fn manifest_service_recovery_reason( @@ -1501,147 +1485,6 @@ fn manifest_service_recovery_reason( } } -pub(crate) fn restart_managed_service( - paths: &AppPaths, - record: &mut ManagedServiceRecord, -) -> Result<()> { - let rocmd_binary = - std::env::current_exe().context("failed to resolve current rocmd executable path")?; - let log_file = fs::OpenOptions::new() - .create(true) - .append(true) - .open(&record.log_path) - .with_context(|| format!("failed to open {}", record.log_path.display()))?; - let log_file_err = log_file - .try_clone() - .context("failed to clone service log file handle")?; - - record.status = "recovering".to_owned(); - // Counts the restart and drops the previous run's inference verification. - // The respawned child writes a fresh record of its own, and "recovering" is - // outside the statuses that probe, so a stale verdict would not currently be - // acted on — but this record is written again below, after the spawn, and - // that write can land after the child's. Clearing here keeps the invariant - // true at the one site that reuses a record across restarts. - record.reset_for_restart(); - // No supervisor exists until the spawn below succeeds, so none is recorded. - // Recording this daemon's own PID here, as a stand-in, left the manifest - // naming the daemon as the service's supervisor whenever the spawn failed: - // a stop run from any other process (`rocmd sandbox-tool stop_server`, - // `rocm services stop`) verifies that identity, finds it live, and - // terminates the daemon's whole process tree. The spawned supervisor's - // identity is recorded right after the spawn. - record.supervisor_pid = 0; - record.supervisor_start_ticks = None; - // The engine PID belonged to the run being replaced, and nothing here - // starts an engine: the supervisor spawned below records its own, together - // with its token. Carried forward, the old PID would sit beside a - // supervisor it no longer matches — and on a record `rocm` wrote, with no - // token at all, so the next stop could only signal it blind. - record.engine_pid = None; - record.engine_start_ticks = None; - // Restarting is a request to run, so it supersedes any earlier stop request, - // as `rocm services restart` treats it too. - record.stop_requested_unix_ms = None; - record.write()?; - - let mut child = detached_rocmd_command(&rocmd_binary) - .args(recovery_supervise_args(record)) - .stdin(Stdio::null()) - .stdout(Stdio::from(log_file)) - .stderr(Stdio::from(log_file_err)) - .spawn() - .context("failed to spawn recovery supervisor")?; - - record_spawned_supervisor(paths, record, child.id())?; - - thread::sleep(Duration::from_millis(200)); - if let Some(status) = child - .try_wait() - .context("failed to check recovery supervisor startup state")? - { - // Not a write-back of `record`: a stop issued since the spawn has - // persisted its marker, and a `failed` record without one is what - // recovery restarts. Same discipline as the supervisor's own writes. - crate::service::update_supervised_record(paths, record, |current| { - current.status = "failed".to_owned(); - })?; - anyhow::bail!( - "recovery supervisor exited immediately with status {status}; inspect {}", - record.log_path.display() - ); - } - - Ok(()) -} - -/// Record the supervisor [`restart_managed_service`] just spawned, as `pid`. -/// -/// Between the restart's first write — which clears the recorded PIDs, since -/// no supervisor exists yet — and this one, the record names no process at -/// all. A stop that starts in that window persists its marker first and then -/// finds nothing to signal; writing the restart's whole copy back here would -/// erase that marker while the new supervisor comes up, and leave the stop's -/// own final write comparing against a record that no longer shows the -/// process it has to answer for. So only the supervisor's identity is applied, -/// to the record as it is on disk now, through the same ownership guard as the -/// supervisor's own writes: against the copy from before this assignment, so -/// it is the cleared PID that has to still be there. If the new supervisor has -/// already recorded itself, or the record was removed, nothing is written. -fn record_spawned_supervisor( - paths: &AppPaths, - record: &mut ManagedServiceRecord, - pid: u32, -) -> Result<()> { - let unrecorded = record.clone(); - record.record_supervisor_identity(pid); - let (supervisor_pid, supervisor_start_ticks) = - (record.supervisor_pid, record.supervisor_start_ticks); - crate::service::update_supervised_record(paths, &unrecorded, |current| { - current.supervisor_pid = supervisor_pid; - current.supervisor_start_ticks = supervisor_start_ticks; - }) -} - -fn recovery_supervise_args(record: &ManagedServiceRecord) -> Vec { - let mut args = vec![ - "supervise".to_owned(), - record.service_id.clone(), - "--engine".to_owned(), - record.engine.clone(), - "--model-ref".to_owned(), - record.model_ref.clone(), - "--canonical-model-id".to_owned(), - record.canonical_model_id.clone(), - "--host".to_owned(), - record.host.clone(), - "--port".to_owned(), - record.port.to_string(), - "--device-policy".to_owned(), - record - .device_policy - .as_deref() - .unwrap_or("gpu_required") - .to_owned(), - ]; - args.extend(crate::common::optional_arg( - "--runtime-id", - record.runtime_id.as_deref(), - )); - args.extend(crate::common::optional_arg( - "--env-id", - record.env_id.as_deref(), - )); - if let Some(csv) = rocm_engine_protocol::gpu_indices_to_csv(&record.gpu_indices) { - args.extend(["--gpu".to_owned(), csv]); - } - args.extend(crate::common::optional_arg( - "--engine-recipe-json", - record.engine_recipe_json.as_deref(), - )); - args -} - fn queue_proposal( paths: &AppPaths, watcher_id: &str, @@ -1713,26 +1556,14 @@ fn proposal_arguments_for_action(action: &str, service_id: Option<&str>) -> Valu } } -#[cfg(unix)] -fn detached_rocmd_command(rocmd_binary: &std::path::Path) -> ProcessCommand { - let mut command = ProcessCommand::new("setsid"); - command.arg(rocmd_binary); - command -} - -#[cfg(not(unix))] -fn detached_rocmd_command(rocmd_binary: &std::path::Path) -> ProcessCommand { - ProcessCommand::new(rocmd_binary) -} - #[cfg(test)] mod tests { use super::*; - use crate::test_support::identity_probe_record; use crate::test_support::{ UNCONFIRMED_STOP_PID, seed_keyed_service, stop_with_outcome, temp_app_paths, }; use rocm_core::{AuditEventRecord, AutomationEventRecord}; + use std::fs; fn test_watcher_snapshot( id: &str, @@ -3488,38 +3319,6 @@ mod tests { Ok(()) } - #[test] - fn recovery_supervise_args_preserve_engine_recipe_json() { - let (_root, paths) = temp_app_paths("recovery-engine-recipe"); - let mut record = ManagedServiceRecord::new( - &paths, - "svc-1", - "vllm", - "qwen", - "Qwen/Qwen3.5-4B", - "127.0.0.1", - 11435, - "managed", - 123, - Some("therock-release:gfx120X-all".to_owned()), - Some("env-1".to_owned()), - Some("gpu_required".to_owned()), - ); - let engine_recipe_json = r#"{"contract_version":"0.1.0","engine":"vllm","required_flags":["--enable-auto-tool-choice"]}"#; - record.engine_recipe_json = Some(engine_recipe_json.to_owned()); - - let args = recovery_supervise_args(&record); - - assert!( - args.windows(2) - .any(|pair| pair[0] == "--engine-recipe-json" && pair[1] == engine_recipe_json) - ); - assert!( - args.windows(2) - .any(|pair| { pair[0] == "--canonical-model-id" && pair[1] == "Qwen/Qwen3.5-4B" }) - ); - } - #[test] fn watcher_policy_maps_modes_to_decisions() { assert_eq!( @@ -3540,41 +3339,6 @@ mod tests { ); } - /// The same guarantee for the daemon's recovery path, which records a PID of - /// its own. See `supervise_service_persists_the_supervisor_identity_token` - /// for why an unrecorded token is the defect rather than a cosmetic gap. - /// - /// The restart re-execs `rocmd supervise`, which here is the test binary: - /// it rejects those arguments and exits at once, so the restart reports - /// failure — after it has written the record this test reads. - #[cfg(target_os = "linux")] - #[test] - fn restart_managed_service_persists_the_supervisor_identity_token() -> Result<()> { - let (root, paths) = temp_app_paths("restart-records-identity"); - paths.ensure()?; - fs::create_dir_all(paths.services_dir())?; - - let service_id = "svc-restart-identity"; - let mut record = identity_probe_record(&paths, service_id, 11444); - record.status = "failed".to_owned(); - record.write()?; - assert_eq!( - load_service_record(&paths, service_id)?.supervisor_start_ticks, - None, - "precondition: the seeded record carries no token" - ); - - let _ = restart_managed_service(&paths, &mut record); - let persisted = load_service_record(&paths, service_id); - fs::remove_dir_all(root).ok(); - - assert!( - persisted?.supervisor_start_ticks.is_some(), - "restart_managed_service must persist the supervisor's start-time token beside its PID" - ); - Ok(()) - } - /// An unconfirmed stop leaves the status at `ready` on purpose, which is /// exactly what the daemon's recovery scan looks for. The stop request it /// records is the only thing telling it apart from a crash, so recovery @@ -3671,173 +3435,4 @@ mod tests { assert!(event_without_request); Ok(()) } - - /// A restart moves the supervisor PID; the engine PID it carried belonged - /// to the run being replaced. On a record `rocm serve --background` wrote, - /// that PID has no token, so carrying it past the point where the two PIDs - /// diverge leaves the next stop an entry it can only signal blind. - /// - /// Linux-only for the same reason as the test above: the restart re-execs - /// the test binary, which is only known to reject those arguments there. - #[cfg(target_os = "linux")] - #[test] - fn restart_managed_service_drops_the_replaced_runs_engine_pid() -> Result<()> { - let (root, paths) = temp_app_paths("restart-drops-engine-pid"); - paths.ensure()?; - fs::create_dir_all(paths.services_dir())?; - - let service_id = "svc-restart-drops-engine-pid"; - let mut record = identity_probe_record(&paths, service_id, 11455); - // The shape `rocm serve --background` writes: one PID in both roles. - record.supervisor_pid = UNCONFIRMED_STOP_PID; - record.engine_pid = Some(UNCONFIRMED_STOP_PID); - record.engine_start_ticks = None; - record.stop_requested_unix_ms = Some(1); - record.status = "failed".to_owned(); - record.write()?; - - let _ = restart_managed_service(&paths, &mut record); - let persisted = load_service_record(&paths, service_id); - fs::remove_dir_all(root).ok(); - - let persisted = persisted?; - assert_ne!(persisted.supervisor_pid, UNCONFIRMED_STOP_PID); - assert_eq!( - persisted.engine_pid, None, - "the replaced run's engine PID must not outlive the restart" - ); - assert_eq!(persisted.engine_start_ticks, None); - assert_eq!( - persisted.stop_requested_unix_ms, None, - "a restart supersedes an earlier stop request" - ); - Ok(()) - } - - /// A stop that lands while a restart waits to see whether its supervisor - /// survived startup must outlive the restart's `failed` write: a `failed` - /// record without the marker is exactly what recovery restarts. - /// - /// The restarted supervisor is the test binary, which rejects the - /// `supervise` arguments and exits at once, so the restart reaches that - /// write. A thread plays the stop: once the restart has recorded the - /// spawned supervisor, it persists a marker, as a stop does first. - #[cfg(target_os = "linux")] - #[test] - fn restart_failure_write_keeps_a_stop_marker_set_meanwhile() -> Result<()> { - let (root, paths) = temp_app_paths("restart-failure-keeps-marker"); - paths.ensure()?; - fs::create_dir_all(paths.services_dir())?; - - let service_id = "svc-restart-failure-marker"; - let mut record = identity_probe_record(&paths, service_id, 11456); - record.status = "failed".to_owned(); - record.write()?; - - let stop = { - let paths = paths.clone(); - thread::spawn(move || -> Result { - let deadline = std::time::Instant::now() + Duration::from_secs(10); - while std::time::Instant::now() < deadline { - // `write` is not atomic, so a read can land mid-write and - // see a truncated file; just look again. - let Ok(mut on_disk) = load_service_record(&paths, service_id) else { - thread::sleep(Duration::from_millis(5)); - continue; - }; - if on_disk.supervisor_pid != 0 && on_disk.supervisor_pid != std::process::id() { - on_disk.stop_requested_unix_ms = Some(1); - on_disk.write()?; - return Ok(true); - } - thread::sleep(Duration::from_millis(5)); - } - Ok(false) - }) - }; - let outcome = restart_managed_service(&paths, &mut record); - let stop_landed = stop.join().expect("stop thread"); - let persisted = load_service_record(&paths, service_id); - fs::remove_dir_all(root).ok(); - - assert!( - stop_landed?, - "precondition: the stop ran during the restart" - ); - assert!(outcome.is_err(), "the test binary is not a supervisor"); - let persisted = persisted?; - assert_eq!(persisted.status, "failed"); - assert_eq!( - persisted.stop_requested_unix_ms, - Some(1), - "the restart's failure write must not withdraw a stop issued meanwhile" - ); - Ok(()) - } - - /// Between a restart's first write, which clears the recorded PIDs, and the - /// one recording the supervisor it spawned, a stop can persist its marker. - /// Writing the restart's own copy back would erase it while the new - /// supervisor comes up — so only the identity is applied. - #[test] - fn restart_records_its_supervisor_without_erasing_a_stop_marker() -> Result<()> { - let (root, paths) = temp_app_paths("restart-spawn-keeps-marker"); - paths.ensure()?; - let service_id = "svc-restart-spawn-marker"; - // The restart's copy after its first write: no supervisor, no marker. - let mut restarting = identity_probe_record(&paths, service_id, 11457); - restarting.status = "recovering".to_owned(); - restarting.supervisor_pid = 0; - restarting.supervisor_start_ticks = None; - // A stop started since: same (empty) processes, marker set. - let mut on_disk = restarting.clone(); - on_disk.stop_requested_unix_ms = Some(1); - on_disk.write()?; - - record_spawned_supervisor(&paths, &mut restarting, std::process::id())?; - let persisted = load_service_record(&paths, service_id); - fs::remove_dir_all(root).ok(); - - let persisted = persisted?; - assert_eq!(restarting.supervisor_pid, std::process::id()); - assert_eq!(persisted.supervisor_pid, std::process::id()); - assert_eq!( - persisted.supervisor_start_ticks, - restarting.supervisor_start_ticks - ); - assert_eq!( - persisted.stop_requested_unix_ms, - Some(1), - "recording the spawned supervisor must not withdraw a standing stop" - ); - Ok(()) - } - - /// The spawned supervisor may record itself before the restart does, and - /// the record may be removed outright; either way the restart's write is - /// not the one to land. - #[test] - fn restart_leaves_a_record_naming_another_supervisor_or_none() -> Result<()> { - let (root, paths) = temp_app_paths("restart-spawn-not-owner"); - paths.ensure()?; - let service_id = "svc-restart-spawn-not-owner"; - let mut restarting = identity_probe_record(&paths, service_id, 11458); - restarting.supervisor_pid = 0; - restarting.supervisor_start_ticks = None; - let mut on_disk = restarting.clone(); - on_disk.supervisor_pid = 4_242_424; - on_disk.write()?; - - record_spawned_supervisor(&paths, &mut restarting.clone(), std::process::id())?; - let after_supervisor = load_service_record(&paths, service_id); - - fs::remove_file(paths.service_manifest_path(service_id))?; - record_spawned_supervisor(&paths, &mut restarting, std::process::id())?; - let resurrected = paths.service_manifest_path(service_id).exists(); - fs::remove_dir_all(root).ok(); - - assert_eq!(after_supervisor?.supervisor_pid, 4_242_424); - assert!(!resurrected, "a removed record must not be written back"); - Ok(()) - } } diff --git a/apps/rocmd/src/webhook.rs b/apps/rocmd/src/webhook.rs index 3c455b0430..e26c6f7f7e 100644 --- a/apps/rocmd/src/webhook.rs +++ b/apps/rocmd/src/webhook.rs @@ -162,12 +162,12 @@ pub(crate) fn local_webhook_event_from_request( bail!("server-recover webhook events require service_id"); } "cache-warm" - if crate::watchers::payload_string(&request.payload, "artifact_ref").is_none() => + if crate::common::payload_string(&request.payload, "artifact_ref").is_none() => { bail!("cache-warm webhook events require payload.artifact_ref"); } "driver-upgrade" - if crate::watchers::payload_string(&request.payload, "component").as_deref() + if crate::common::payload_string(&request.payload, "component").as_deref() != Some("driver") => { bail!("driver-upgrade webhook events require payload.component=driver"); diff --git a/crates/rocm-core/src/process.rs b/crates/rocm-core/src/process.rs index afbea28608..758d19762a 100644 --- a/crates/rocm-core/src/process.rs +++ b/crates/rocm-core/src/process.rs @@ -2010,9 +2010,11 @@ mod tests { // pre-setting it to the marker is cmd's equivalent of the unix // `read || echo`. A stdin left open blocks in `set /p` instead. // - // Both callers in `apps/rocmd/src/common.rs` run on Windows, and - // `run_with_timeout` has no `cfg(windows)` branch, so this covers the - // same promise with the same two-sidedness as the unix case. + // Both callers in `rocmd` run on Windows -- the amd-smi probe in + // `apps/rocmd/src/host_snapshot.rs` and the `rocm` capture that + // `apps/rocmd/src/common.rs` makes through `run_rocm_capture_with_env` + // -- and `run_with_timeout` has no `cfg(windows)` branch, so this covers + // the same promise with the same two-sidedness as the unix case. let fixture = WindowsFixture::script( "stdin", &format!( diff --git a/crates/rocm-core/src/rocm_install.rs b/crates/rocm-core/src/rocm_install.rs index 8ddab2ad5b..7830e4d4e6 100644 --- a/crates/rocm-core/src/rocm_install.rs +++ b/crates/rocm-core/src/rocm_install.rs @@ -28,7 +28,7 @@ pub struct LegacyRocmSummary { /// declares no version anywhere. /// /// Optional and defaulted so the daemon's serialised snapshot - /// (`apps/rocmd/src/lib.rs`) written before this field existed still loads. + /// (`apps/rocmd/src/host_snapshot.rs`) written before this field existed still loads. #[serde(default)] pub version: Option, } diff --git a/crates/rocm-dash-daemon/src/registry.rs b/crates/rocm-dash-daemon/src/registry.rs index cf321d757e..6f02aa8f4c 100644 --- a/crates/rocm-dash-daemon/src/registry.rs +++ b/crates/rocm-dash-daemon/src/registry.rs @@ -89,7 +89,7 @@ pub fn is_scrapeable_status(status: &str) -> bool { /// for safety. /// /// **`running` + `startup_phase` overlap:** the rocm-cli supervisor -/// (`apps/rocmd/src/lib.rs::supervise_service`) flips the on-disk record to +/// (`apps/rocmd/src/supervise.rs::supervise_service`) flips the on-disk record to /// `running` immediately after spawning the engine child — *before* it starts /// polling and parsing `startup_phase` from the serve log — and only clears /// `startup_phase` once the service reaches `ready`. So a record can be @@ -432,7 +432,7 @@ mod tests { #[test] fn discovered_from_record_surfaces_startup_phase_while_running() { - // The real producer (`apps/rocmd/src/lib.rs::supervise_service`) flips + // The real producer (`apps/rocmd/src/supervise.rs::supervise_service`) flips // the on-disk record to `running` *before* it starts polling and // parsing `startup_phase` from the serve log, and only clears // `startup_phase` once the service reaches `ready`. Exercise that diff --git a/docs/architecture.md b/docs/architecture.md index ecb4709327..75bdaa2106 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -83,16 +83,21 @@ Further `apps/rocm` modules, not part of the `main.rs` split: Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): -- [`cli.rs`](../apps/rocmd/src/cli.rs) — the `Cli`/`Command` clap definitions, `SandboxToolArg`/`SandboxToolPolicy`, the bridge-snapshot diagnostic printer, and the top-level dispatch in `run_cli`/`run_bin_cli`/`run_from_args`. -- [`common.rs`](../apps/rocmd/src/common.rs) — helpers the other modules call into: GPU/amd-smi snapshotting, the bridge-snapshot diagnostic, the `rocm`-subprocess capture wrappers the sandbox and MCP clusters both call (they only choose the child's `ROCM_CLI_*_DIR` environment; `CommandCapture` and the spawn itself live in [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), shared with `rocm`'s chat tool bridge, and the bounded-execution half in [`process.rs`](../crates/rocm-core/src/process.rs)), and small arg/healthcheck/endpoint-key utilities. Not every helper here has more than one caller (`wait_for_port`, for example, is called only from [`watchers.rs`](../apps/rocmd/src/watchers.rs)). -- [`mcp.rs`](../apps/rocmd/src/mcp.rs) — the MCP stdio server, tool schema table, tool dispatch, and the argv-building and result-shaping helpers behind the MCP tools (the `rocm` subprocess itself is spawned through [`common.rs`](../apps/rocmd/src/common.rs)). -- [`persistence.rs`](../apps/rocmd/src/persistence.rs) — `record_event`/`load_managed_services`, the automation-event/audit-log and managed-service-registry I/O shared across the daemon's sandbox, MCP, service-lifecycle, and watcher code. -- [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) — bubblewrap/native sandbox execution, atomic-write helpers, artifact prefetch policy gating, and the sandbox-tool result shaping for `check_updates`/`driver_plan`. -- [`service.rs`](../apps/rocmd/src/service.rs) — managed-service PID lifecycle/stop, `run_daemon`'s foreground loop, `supervise_service`'s spawn-and-recover path, and serve-log startup-phase polling. +- [`cli.rs`](../apps/rocmd/src/cli.rs) — the `Cli`/`Command` clap definitions, the bridge-snapshot diagnostic printer, the pretty-JSON printer for the subcommands that print a result (`print_json`), and the top-level dispatch in `run_cli`/`run_bin_cli`/`run_from_args`. +- [`common.rs`](../apps/rocmd/src/common.rs) — helpers the other modules call into: the read-only `rocm update` message, the `rocm`-subprocess capture wrapper (`run_rocm_capture_for_paths`) the sandbox and MCP clusters both call (it only chooses the child's `ROCM_CLI_*_DIR` environment; `CommandCapture` and the spawn itself live in [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), shared with `rocm`'s chat tool bridge, and the bounded-execution half in [`process.rs`](../crates/rocm-core/src/process.rs)), and small arg/JSON-payload utilities (`payload_string`, for example, is read by both [`watchers.rs`](../apps/rocmd/src/watchers.rs) and [`webhook.rs`](../apps/rocmd/src/webhook.rs)). +- [`daemon.rs`](../apps/rocmd/src/daemon.rs) — the foreground `rocmd run` loop (`run_daemon`: evaluate the watchers each tick, dispatch local webhook events, write the runtime state, stop on a signal) and `rocmd status` (`print_status`), with the watcher tick interval. It uses [`watchers.rs`](../apps/rocmd/src/watchers.rs), [`webhook.rs`](../apps/rocmd/src/webhook.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs); [`cli.rs`](../apps/rocmd/src/cli.rs) calls it. +- [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) — talking to an engine and guarding its endpoint key: `engine_request` (the stdio request to an engine process), the healthcheck call and its readiness and recoverability checks, and the endpoint-key helpers `apply_endpoint_key_env` and `ensure_public_service_has_endpoint_key`. It uses no other module of the crate; [`service.rs`](../apps/rocmd/src/service.rs), [`supervise.rs`](../apps/rocmd/src/supervise.rs) and [`watchers.rs`](../apps/rocmd/src/watchers.rs) call it. +- [`host_snapshot.rs`](../apps/rocmd/src/host_snapshot.rs) — the host snapshot: `build_bridge_snapshot` (the bridge snapshot printed by `rocmd bridge-snapshot` and returned by the `bridge_snapshot` MCP tool, assembled from the examine summary, the GPU snapshot, the automation runtime state and recent events, the engine inventory and the managed services), the amd-smi GPU snapshot (`gather_gpu_snapshot_for_config`, which honours the telemetry setting, over time-bounded amd-smi probes) and `bridge_engine_inventory`. It uses only [`persistence.rs`](../apps/rocmd/src/persistence.rs); [`cli.rs`](../apps/rocmd/src/cli.rs), [`mcp.rs`](../apps/rocmd/src/mcp.rs) and [`watchers.rs`](../apps/rocmd/src/watchers.rs) call it. +- [`mcp.rs`](../apps/rocmd/src/mcp.rs) — the MCP stdio server, tool schema table, tool dispatch, and the argv-building and result-shaping helpers behind the MCP tools (the `rocm` subprocess itself is spawned through [`common.rs`](../apps/rocmd/src/common.rs), via `run_rocm_capture` here). +- [`persistence.rs`](../apps/rocmd/src/persistence.rs) — `record_event`/`load_managed_services`/`load_service_record`, the automation-event/audit-log and managed-service-registry I/O shared across the daemon's sandbox, MCP, service-lifecycle, and watcher code. +- [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) — bubblewrap/native sandbox execution, atomic-write helpers, artifact prefetch policy gating, and the sandbox-tool result shaping for `check_updates`/`driver_plan`/`stop_server`. +- [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs) — the sandbox-tool vocabulary: the `SandboxToolArg` value enum naming each sandbox tool, and `SandboxToolPolicy`, the artifact-download policy built from the `--allow-*` flags (including the Hugging Face token read from the environment when authenticated downloads are allowed). It uses no other module of the crate, so modules that need the vocabulary do not import [`cli.rs`](../apps/rocmd/src/cli.rs). +- [`service.rs`](../apps/rocmd/src/service.rs) — managed-service stop: terminating the recorded processes of a service after verifying their identity, and the stop report (`stop_managed_service`, `stop_report_confirmed`) that the sandbox and MCP tools read. +- [`supervise.rs`](../apps/rocmd/src/supervise.rs) — launching and re-launching a managed service's supervisor: `supervise_service` (the `rocmd supervise` entry point: launch the engine child, record its identity, wait for readiness, record the exit) with the serve-log startup-phase polling and readiness wait it uses, and `restart_managed_service` (the recovery respawn: re-launch `rocmd supervise` detached for a service and record the new supervisor). Writes after a supervisor's first go through the private `update_supervised_record`, which re-reads the record and leaves a stop marker or another supervisor's record alone. Its production code uses only [`common.rs`](../apps/rocmd/src/common.rs), [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs). [`cli.rs`](../apps/rocmd/src/cli.rs) calls `supervise_service`; [`watchers.rs`](../apps/rocmd/src/watchers.rs) (the server-recover handler) and [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) (the `restart_server` tool) call `restart_managed_service`. - [`watchers.rs`](../apps/rocmd/src/watchers.rs) — event collection/dispatch for all built-in watchers (TheRock update, GPU metrics/thermal-pressure, cache-warm, driver-upgrade, server-recover), plus managed-service recovery classification and automation-proposal queuing. - [`webhook.rs`](../apps/rocmd/src/webhook.rs) — the local webhook source: its axum routes, request validation, and watcher-kind allow-list. -[`lib.rs`](../apps/rocmd/src/lib.rs) itself is now top-level glue: module declarations, the three externally-consumed entry points re-exported from [`cli.rs`](../apps/rocmd/src/cli.rs) (`run_bin_cli`/`run_from_args`, plus `parse_args`, which `apps/rocm`'s test-only [`advised_commands.rs`](../apps/rocm/src/advised_commands.rs) uses to check advised `rocmd` commands), the `WATCHER_TICK_INTERVAL`/`ARTIFACT_PREFETCH_TIMEOUT` constants, and a test module. +[`lib.rs`](../apps/rocmd/src/lib.rs) itself is now top-level glue: module declarations and the three externally-consumed entry points re-exported from [`cli.rs`](../apps/rocmd/src/cli.rs) (`run_bin_cli`/`run_from_args`, plus `parse_args`, which `apps/rocm`'s test-only [`advised_commands.rs`](../apps/rocm/src/advised_commands.rs) uses to check advised `rocmd` commands). ### `crates/rocm-core` — core library