diff --git a/apps/rocmd/src/test_support.rs b/apps/rocmd/src/test_support.rs index 424ad9d5c1..2dca74747d 100644 --- a/apps/rocmd/src/test_support.rs +++ b/apps/rocmd/src/test_support.rs @@ -11,7 +11,10 @@ //! split into focused modules (see `docs/architecture.md`). use anyhow::Result; -use rocm_core::{AppPaths, ManagedServiceRecord, unix_time_millis}; +use rocm_core::{ + AppPaths, AutomationRuntimeState, ManagedServiceRecord, WatcherMode, WatcherRuntimeSnapshot, + unix_time_millis, +}; use std::fs; use std::path::PathBuf; use std::sync::atomic::{AtomicU64, Ordering}; @@ -186,6 +189,47 @@ pub(crate) fn assert_unconfirmed_stop_kept_the_service( assert!(key_kept, "an unconfirmed stop must keep the endpoint key"); } +/// One enabled watcher snapshot in `mode`, for tests that build an automation +/// runtime state by hand. +pub(crate) fn test_watcher_snapshot( + id: &str, + mode: WatcherMode, + last_event_unix_ms: Option, +) -> WatcherRuntimeSnapshot { + WatcherRuntimeSnapshot { + id: id.to_owned(), + enabled: true, + mode, + summary: "test watcher".to_owned(), + last_event: None, + last_event_unix_ms, + } +} + +/// A running automation runtime state carrying `active_watchers`. +pub(crate) fn test_runtime_state( + active_watchers: Vec, +) -> AutomationRuntimeState { + AutomationRuntimeState { + running: true, + automations_enabled: true, + daemon_pid: 1, + started_at_unix_ms: 1, + last_tick_unix_ms: 1, + local_webhook_endpoint: None, + active_watchers, + } +} + +/// How many pending proposals to queue behind the one under test so it sits +/// beyond a recent-window read: more than the reviewed-retention cap, and +/// more than the 100 the code once read, so the tests keep guarding that +/// window even if the cap is ever lowered below it. +pub(crate) fn proposals_beyond_a_recent_window() -> usize { + const WINDOW_THE_CODE_ONCE_READ: usize = 100; + rocm_core::REVIEWED_PROPOSALS_RETAINED.max(WINDOW_THE_CODE_ONCE_READ) + 1 +} + #[cfg(test)] mod tests { use super::*; diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index e32a8c79eb..c725cd4c9c 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -2,30 +2,35 @@ // // SPDX-License-Identifier: MIT -use crate::common::payload_string; -use crate::persistence::load_service_record; -use crate::supervise::restart_managed_service; -use anyhow::{Context, Result, bail}; +use anyhow::Result; use rocm_core::{ - AppPaths, AutomationProposalRecord, AutomationRuntimeState, AutomationTriggerEvent, - CodexBridgeGpuSnapshot, ManagedServiceRecord, RocmCliConfig, WatcherMode, - WatcherRuntimeSnapshot, append_automation_proposal, builtin_watchers, format_host_port, - resolve_model_recipe_artifact, unix_time_millis, + AppPaths, AutomationRuntimeState, AutomationTriggerEvent, CodexBridgeGpuSnapshot, + RocmCliConfig, WatcherRuntimeSnapshot, builtin_watchers, unix_time_millis, }; -use serde_json::Value; use serde_json::json; -use std::net::{SocketAddr, TcpStream}; -use std::thread; -use std::time::Duration; -const SERVER_RECOVER_BACKOFF_MS: u128 = 30_000; -const SERVER_TRANSIENT_STALE_MS: u128 = 5 * 60 * 1_000; -const ENDPOINT_HEALTH_TIMEOUT: Duration = Duration::from_millis(250); -const THEROCK_UPDATE_INTERVAL_MS: u128 = 6 * 60 * 60 * 1000; -const GPU_METRICS_INTERVAL_MS: u128 = 60 * 1000; -const GPU_THERMAL_HOTSPOT_PRESSURE_C: f64 = 95.0; -const GPU_THERMAL_MEMORY_PRESSURE_C: f64 = 95.0; -const GPU_MEMORY_VRAM_PRESSURE_PERCENT: f64 = 95.0; +mod cache_warm; +mod driver_upgrade; +mod gpu_metrics; +mod gpu_thermal_protect; +mod policy; +mod server_recover; +mod therock_update; + +use cache_warm::handle_cache_warm_event; +use driver_upgrade::handle_driver_upgrade_event; +use gpu_metrics::{ + GPU_METRICS_INTERVAL_MS, gpu_metrics_due, gpu_pressure_events, gpu_snapshot_summary, + handle_gpu_metrics_event, +}; +use gpu_thermal_protect::{gpu_thermal_protect_due, handle_gpu_thermal_protect_event}; +pub(crate) use server_recover::find_recoverable_service; +#[cfg(test)] +pub(crate) use server_recover::stop_requested; +use server_recover::{ + handle_server_recover_event, server_recover_due, service_recovery_event_kind, +}; +use therock_update::{THEROCK_UPDATE_INTERVAL_MS, handle_therock_update_event, therock_update_due}; pub(crate) fn reconcile_watcher_snapshots( config: &RocmCliConfig, @@ -230,1558 +235,189 @@ fn events_for_watcher_exact<'a>( }) } -fn handle_therock_update_event( - paths: &AppPaths, - mode: WatcherMode, - state: &mut AutomationRuntimeState, - event: &AutomationTriggerEvent, -) -> Result<()> { - handle_therock_update_event_with_runner(paths, mode, state, event, |paths| { - crate::sandbox::run_sandbox_tool( - paths, - crate::sandbox_tool::SandboxToolArg::CheckUpdates, - None, - None, - None, - crate::sandbox_tool::SandboxToolPolicy::default(), - ) - }) -} - -fn handle_therock_update_event_with_runner( - paths: &AppPaths, - mode: WatcherMode, - state: &mut AutomationRuntimeState, - _event: &AutomationTriggerEvent, - update_runner: F, -) -> Result<()> -where - F: FnOnce(&AppPaths) -> Result, -{ - let policy = watcher_policy_action("therock-update", mode); - let action = match policy { - WatcherPolicyAction::Observe => "observe_schedule", - WatcherPolicyAction::QueueProposal => "queue_update_proposal", - WatcherPolicyAction::RunContained => "run_update_check", - }; - let message = match policy { - WatcherPolicyAction::Observe => { - "scheduled TheRock update check reminder emitted; run `rocm update` to inspect the selected channel" - } - WatcherPolicyAction::QueueProposal => { - "scheduled TheRock update check reminder emitted; queueing read-only update-check proposal for review" - } - WatcherPolicyAction::RunContained => { - "scheduled TheRock update check is approved for contained read-only execution" - } - }; - match policy { - WatcherPolicyAction::Observe | WatcherPolicyAction::QueueProposal => { - crate::persistence::record_event( - paths, - state, - "therock-update", - "info", - action, - message, - None, - )?; - if policy == WatcherPolicyAction::QueueProposal { - queue_proposal( - paths, - "therock-update", - action, - "Check TheRock updates", - "Run `rocm update` to inspect available CLI, runtime, engine, and recipe updates before applying changes.", - None, - )?; - } - } - WatcherPolicyAction::RunContained => match update_runner(paths) { - Ok(output) => match restricted_check_updates_result(&output) { - Ok(result) => { - crate::persistence::record_event( - paths, - state, - "therock-update", - if result.exit_status == 0 { - "info" - } else { - "error" - }, - action, - &format!( - "{message}; restricted check_updates status={}; {}", - result.status, - crate::common::update_check_message(result.status) - ), - None, - )?; - if result.update_available { - record_update_available_notification(paths, state, result.status)?; - } - } - Err(error) => { - crate::persistence::record_event( - paths, - state, - "therock-update", - "error", - "update_check_failed", - &format!( - "scheduled TheRock update check failed during contained restricted execution: {error}; no updates were applied" - ), - None, - )?; - } - }, - Err(error) => { - crate::persistence::record_event( - paths, - state, - "therock-update", - "error", - "update_check_failed", - &format!( - "scheduled TheRock update check failed during contained read-only execution: {error}; no updates were applied" - ), - None, - )?; - } - }, - } - Ok(()) -} - -struct RestrictedCheckUpdatesResult<'a> { - status: &'a str, - update_available: bool, - exit_status: i64, -} - -fn restricted_check_updates_result(value: &Value) -> Result> { - let tool = value - .get("tool") - .and_then(Value::as_str) - .context("restricted update check did not report a tool name")?; - if tool != crate::sandbox_tool::SandboxToolArg::CheckUpdates.as_cli_value() { - bail!("restricted update check returned `{tool}`, expected `check_updates`"); - } - let status = value - .get("status") - .and_then(Value::as_str) - .unwrap_or("checked"); - let update_available = value - .get("update_available") - .and_then(Value::as_bool) - .unwrap_or(matches!(status, "update_available" | "repair_available")); - let exit_status = value - .get("exit_status") - .and_then(Value::as_i64) - .unwrap_or_else(|| i64::from(status == "error")); - Ok(RestrictedCheckUpdatesResult { - status, - update_available, - exit_status, - }) -} - -fn record_update_available_notification( - paths: &AppPaths, - state: &mut AutomationRuntimeState, - status: &str, -) -> Result<()> { - let message = if status == "repair_available" { - "A ROCm runtime repair is available because its package composition changed. Preview it before applying. No updates were applied." - } else { - "A ROCm runtime update is available. Preview it before applying. No updates were applied." - }; - crate::persistence::record_event( - paths, - state, - "therock-update", - "info", - "notify_if_newer", - message, - None, - )?; - crate::sandbox::record_notification_audit( - paths, - "watcher:therock-update", - "notify_if_newer", - Some("therock-update"), - message, - ) -} - -fn therock_update_due(state: &AutomationRuntimeState, now: u128) -> bool { - let Some(snapshot) = state - .active_watchers - .iter() - .find(|watcher| watcher.id == "therock-update" && watcher.enabled) - else { - return false; - }; - snapshot - .last_event_unix_ms - .is_none_or(|last_event| now.saturating_sub(last_event) >= THEROCK_UPDATE_INTERVAL_MS) -} - -fn gpu_metrics_due(state: &AutomationRuntimeState, now: u128) -> bool { - let Some(snapshot) = state - .active_watchers - .iter() - .find(|watcher| watcher.id == "gpu-metrics" && watcher.enabled) - else { - return false; - }; - snapshot - .last_event_unix_ms - .is_none_or(|last_event| now.saturating_sub(last_event) >= GPU_METRICS_INTERVAL_MS) -} - -fn gpu_thermal_protect_due(state: &AutomationRuntimeState, now: u128) -> bool { - let Some(snapshot) = state - .active_watchers - .iter() - .find(|watcher| watcher.id == "gpu-thermal-protect" && watcher.enabled) - else { - return false; - }; - snapshot - .last_event_unix_ms - .is_none_or(|last_event| now.saturating_sub(last_event) >= GPU_METRICS_INTERVAL_MS) -} +#[cfg(test)] +mod tests { + use super::*; + use crate::persistence::load_service_record; + use crate::test_support::{temp_app_paths, test_runtime_state, test_watcher_snapshot}; + use rocm_core::{AutomationEventRecord, ManagedServiceRecord, WatcherMode}; + use serde_json::Value; + use std::fs; -fn handle_gpu_metrics_event( - paths: &AppPaths, - mode: WatcherMode, - state: &mut AutomationRuntimeState, - event: &AutomationTriggerEvent, -) -> Result<()> { - let summary = event - .payload - .get("summary") - .and_then(Value::as_str) - .unwrap_or("summary unavailable"); - let level = if event.kind == "gpu.metrics" { - "info" - } else { - "warn" - }; - let mode_note = match mode { - WatcherMode::Observe => "observe mode records telemetry only", - WatcherMode::Propose => { - "propose mode has no GPU mutation policy yet, so telemetry is recorded only" - } - WatcherMode::Contained => { - "contained mode has no GPU mutation policy yet, so telemetry is recorded only" - } - }; - let reason = event - .reason - .as_deref() - .filter(|value| !value.trim().is_empty()) - .unwrap_or("no detail"); - let source = match event.source.as_str() { - "gpu_telemetry" => "local amd-smi telemetry", - "local_webhook" => "local webhook", - other => other, - }; + #[test] + fn event_collector_emits_schedule_tick_for_due_update() -> Result<()> { + let (root, paths) = temp_app_paths("event-bus-schedule"); + let state = test_runtime_state(vec![test_watcher_snapshot( + "therock-update", + WatcherMode::Observe, + None, + )]); - crate::persistence::record_event( - paths, - state, - "gpu-metrics", - level, - "record_gpu_metrics", - &format!( - "GPU metrics event from {source}: {summary}; reason={reason}; {mode_note}; no mutating action was taken" - ), - None, - ) -} + let events = collect_automation_events(&paths, &RocmCliConfig::default(), &state)?; + fs::remove_dir_all(root).ok(); -fn gpu_snapshot_summary(snapshot: &CodexBridgeGpuSnapshot) -> String { - let mut parts = Vec::new(); - parts.push(format!("amd_smi_available={}", snapshot.amd_smi_available)); - parts.push(format!( - "static_snapshot={}", - if snapshot.static_snapshot.is_some() { - "available" - } else { - "missing" - } - )); - parts.push(format!( - "monitor_snapshot={}", - if snapshot.monitor_snapshot.is_some() { - "available" - } else { - "missing" - } - )); - if let Some(count) = snapshot.static_snapshot.as_ref().and_then(gpu_data_count) { - parts.push(format!("gpu_count={count}")); - } - if let Some(note) = snapshot.note.as_deref() - && !note.trim().is_empty() - { - parts.push(format!("note={note}")); + let event = events + .iter() + .find(|event| event.watcher_hint.as_deref() == Some("therock-update")) + .expect("schedule tick event should be emitted"); + assert_eq!(event.kind, "schedule.tick"); + assert_eq!(event.source, "scheduler"); + assert_eq!(event.reason.as_deref(), Some("therock_update_interval_due")); + Ok(()) } - parts.join(" ") -} - -fn gpu_data_count(value: &Value) -> Option { - value - .get("gpu_data") - .and_then(Value::as_array) - .map(Vec::len) -} - -#[derive(Debug, Clone, Copy)] -struct GpuPressureReading { - gpu_index: Option, - hotspot_temperature_c: Option, - memory_temperature_c: Option, - vram_percent: Option, -} -fn gpu_pressure_events( - now: u128, - snapshot: &CodexBridgeGpuSnapshot, -) -> Vec { - let Some(monitor_snapshot) = snapshot.monitor_snapshot.as_ref() else { - return Vec::new(); - }; - monitor_entries(monitor_snapshot) - .into_iter() - .filter_map(gpu_pressure_reading) - .filter_map(|reading| gpu_pressure_event(now, reading)) - .collect() -} + #[test] + fn event_collector_emits_recoverable_service_event() -> Result<()> { + let (root, paths) = temp_app_paths("event-bus-service"); + paths.ensure()?; + let mut failed = ManagedServiceRecord::new( + &paths, + "svc-failed", + "vllm", + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + 11435, + "managed", + 123, + None, + None, + None, + ); + failed.status = "failed".to_owned(); + failed.write()?; + let state = test_runtime_state(vec![test_watcher_snapshot( + "server-recover", + WatcherMode::Propose, + None, + )]); -fn gpu_pressure_event(now: u128, reading: GpuPressureReading) -> Option { - let (kind, reason, metric_label, value, threshold) = if let Some(value) = - reading.hotspot_temperature_c - && value >= GPU_THERMAL_HOTSPOT_PRESSURE_C - { - ( - "gpu.thermal_pressure", - "hotspot_temperature_threshold", - "hotspot temperature", - value, - GPU_THERMAL_HOTSPOT_PRESSURE_C, - ) - } else if let Some(value) = reading.memory_temperature_c - && value >= GPU_THERMAL_MEMORY_PRESSURE_C - { - ( - "gpu.thermal_pressure", - "memory_temperature_threshold", - "memory temperature", - value, - GPU_THERMAL_MEMORY_PRESSURE_C, - ) - } else if let Some(value) = reading.vram_percent - && value >= GPU_MEMORY_VRAM_PRESSURE_PERCENT - { - ( - "gpu.memory_pressure", - "vram_pressure_threshold", - "VRAM use", - value, - GPU_MEMORY_VRAM_PRESSURE_PERCENT, - ) - } else { - return None; - }; - let gpu_label = reading - .gpu_index - .map_or_else(|| "the GPU".to_owned(), |gpu| format!("GPU {gpu}")); - let unit = if metric_label == "VRAM use" { - "%" - } else { - " C" - }; - let summary = format!( - "{gpu_label} {metric_label} is {}{} (limit {}{})", - display_metric(value), - unit, - display_metric(threshold), - unit - ); - Some(AutomationTriggerEvent { - at_unix_ms: now, - kind: kind.to_owned(), - source: "gpu_telemetry".to_owned(), - watcher_hint: Some("gpu-thermal-protect".to_owned()), - service_id: None, - reason: Some(reason.to_owned()), - payload: json!({ - "gpu": reading.gpu_index, - "hotspot_temperature_c": reading.hotspot_temperature_c, - "memory_temperature_c": reading.memory_temperature_c, - "vram_percent": reading.vram_percent, - "hotspot_threshold_c": GPU_THERMAL_HOTSPOT_PRESSURE_C, - "memory_temperature_threshold_c": GPU_THERMAL_MEMORY_PRESSURE_C, - "vram_threshold_percent": GPU_MEMORY_VRAM_PRESSURE_PERCENT, - "recommended_action": "stop_serving_load", - "summary": summary, - }), - }) -} + let events = collect_automation_events(&paths, &RocmCliConfig::default(), &state)?; + fs::remove_dir_all(root).ok(); -fn monitor_entries(value: &Value) -> Vec<&Value> { - if let Some(entries) = value.as_array() { - return entries.iter().collect(); + let event = events + .iter() + .find(|event| event.watcher_hint.as_deref() == Some("server-recover")) + .expect("recoverable service event should be emitted"); + assert_eq!(event.kind, "service.manifest_recoverable"); + assert_eq!(event.service_id.as_deref(), Some("svc-failed")); + assert_eq!(event.reason.as_deref(), Some("manifest_status_failed")); + Ok(()) } - value - .get("gpu_data") - .and_then(Value::as_array) - .map(|entries| entries.iter().collect()) - .unwrap_or_default() -} - -fn gpu_pressure_reading(entry: &Value) -> Option { - let reading = GpuPressureReading { - gpu_index: metric_u64(entry, &["gpu", "gpu_id", "gpu_index"]), - hotspot_temperature_c: metric_f64( - entry, - &[ - "hotspot_temperature", - "hotspot_temperature_c", - "temperature_hotspot", - ], - ), - memory_temperature_c: metric_f64( - entry, - &[ - "memory_temperature", - "memory_temperature_c", - "temperature_memory", - ], - ), - vram_percent: metric_f64( - entry, - &["vram_percent", "vram_usage_percent", "vram_used_percent"], - ), - }; - (reading.hotspot_temperature_c.is_some() - || reading.memory_temperature_c.is_some() - || reading.vram_percent.is_some()) - .then_some(reading) -} - -fn metric_f64(entry: &Value, keys: &[&str]) -> Option { - keys.iter() - .find_map(|key| entry.get(*key).and_then(value_as_metric_f64)) -} -fn metric_u64(entry: &Value, keys: &[&str]) -> Option { - keys.iter() - .find_map(|key| entry.get(*key).and_then(value_as_metric_u64)) -} + #[test] + fn event_collector_emits_endpoint_recoverable_service_event() -> Result<()> { + let (root, paths) = temp_app_paths("event-bus-endpoint"); + paths.ensure()?; + let mut service = ManagedServiceRecord::new( + &paths, + "svc-endpoint", + "missing-engine", + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + 1, + "managed", + 123, + None, + None, + None, + ); + service.status = "ready".to_owned(); + service.write()?; + let state = test_runtime_state(vec![test_watcher_snapshot( + "server-recover", + WatcherMode::Propose, + None, + )]); -fn value_as_metric_f64(value: &Value) -> Option { - match value { - Value::Number(number) => number.as_f64(), - Value::String(text) => text.trim().parse::().ok(), - Value::Object(map) => map - .get("value") - .or_else(|| map.get("val")) - .and_then(value_as_metric_f64), - _ => None, - } -} + let events = collect_automation_events(&paths, &RocmCliConfig::default(), &state)?; + fs::remove_dir_all(root).ok(); -fn value_as_metric_u64(value: &Value) -> Option { - match value { - Value::Number(number) => number.as_u64(), - Value::String(text) => text.trim().parse::().ok(), - Value::Object(map) => map - .get("value") - .or_else(|| map.get("val")) - .and_then(value_as_metric_u64), - _ => None, + let event = events + .iter() + .find(|event| event.watcher_hint.as_deref() == Some("server-recover")) + .expect("endpoint recoverable service event should be emitted"); + assert_eq!(event.kind, "service.endpoint_recoverable"); + assert_eq!(event.service_id.as_deref(), Some("svc-endpoint")); + assert_eq!(event.reason.as_deref(), Some("endpoint_status_unreachable")); + Ok(()) } -} -fn display_metric(value: f64) -> String { - if value.fract().abs() < f64::EPSILON { - format!("{value:.0}") - } else { - format!("{value:.1}") - } -} + #[test] + fn event_collector_emits_gpu_metrics_event_when_enabled() -> Result<()> { + let (root, paths) = temp_app_paths("event-bus-gpu-metrics"); + let state = test_runtime_state(vec![test_watcher_snapshot( + "gpu-metrics", + WatcherMode::Observe, + None, + )]); -fn handle_gpu_thermal_protect_event( - paths: &AppPaths, - mode: WatcherMode, - state: &mut AutomationRuntimeState, - event: &AutomationTriggerEvent, -) -> Result<()> { - let summary = payload_string(&event.payload, "summary") - .unwrap_or_else(|| "GPU pressure is high".to_owned()); - let reason = event.reason.as_deref().unwrap_or("gpu_pressure_threshold"); + let events = collect_automation_events_with_gpu_snapshot(&paths, &state, || { + CodexBridgeGpuSnapshot { + amd_smi_available: true, + static_snapshot: Some(json!({ + "gpu_data": [ + { "gpu": 0, "asic": { "market_name": "AMD Radeon Test" } } + ] + })), + monitor_snapshot: Some(json!({ "gpu_data": [] })), + note: None, + } + })?; + fs::remove_dir_all(root).ok(); - if matches!(mode, WatcherMode::Observe) { - return crate::persistence::record_event( - paths, - state, - "gpu-thermal-protect", - "warn", - "observe_gpu_pressure", - &format!( - "{summary}; observe mode records this only and does not stop any model server" - ), - event.service_id.clone(), + let event = events + .iter() + .find(|event| event.watcher_hint.as_deref() == Some("gpu-metrics")) + .expect("gpu metrics event should be emitted"); + assert_eq!(event.kind, "gpu.metrics"); + assert_eq!(event.source, "gpu_telemetry"); + assert_eq!(event.reason.as_deref(), Some("amd_smi_snapshot_available")); + assert_eq!( + event + .payload + .get("monitor_available") + .and_then(Value::as_bool), + Some(true) ); + assert!( + event + .payload + .get("summary") + .and_then(Value::as_str) + .is_some_and(|summary| summary.contains("gpu_count=1")) + ); + Ok(()) } - let Some(record) = resolve_gpu_pressure_service_target(paths, event)? else { - return crate::persistence::record_event( - paths, - state, + #[test] + fn event_collector_emits_gpu_thermal_pressure_event_when_enabled() -> Result<()> { + let (root, paths) = temp_app_paths("event-bus-gpu-thermal-pressure"); + let state = test_runtime_state(vec![test_watcher_snapshot( "gpu-thermal-protect", - "warn", - "gpu_pressure_no_clear_target", - &format!( - "{summary}; rocm-cli did not choose a model server to stop because there was no single clear running managed server" - ), + WatcherMode::Propose, None, - ); - }; - - if pending_stop_proposal_exists(paths, &record.service_id)? { - return crate::persistence::record_event( - paths, - state, - "gpu-thermal-protect", - "info", - "stop_proposal_already_pending", - &format!( - "{summary}; a reviewed stop request is already waiting for {}", - record.service_id - ), - Some(record.service_id), - ); - } - - let action = "queue_stop_server_proposal"; - let mode_note = if matches!(mode, WatcherMode::Contained) { - "contained mode still asks before stopping anything" - } else { - "asking before stopping anything" - }; - let message = format!( - "{summary}; {mode_note}; selected managed server {} ({})", - record.service_id, record.endpoint_url - ); - crate::persistence::record_event( - paths, - state, - "gpu-thermal-protect", - "warn", - action, - &message, - Some(record.service_id.clone()), - )?; - queue_proposal_with_arguments( - paths, - "gpu-thermal-protect", - action, - "Review GPU pressure stop", - &message, - Some(record.service_id.clone()), - json!({ - "service_id": record.service_id, - "model_ref": record.model_ref, - "canonical_model_id": record.canonical_model_id, - "endpoint_url": record.endpoint_url, - "engine": record.engine, - "pressure_kind": event.kind, - "pressure_reason": reason, - "pressure_summary": summary, - "gpu": event.payload.get("gpu").cloned().unwrap_or(Value::Null), - "hotspot_temperature_c": event.payload.get("hotspot_temperature_c").cloned().unwrap_or(Value::Null), - "memory_temperature_c": event.payload.get("memory_temperature_c").cloned().unwrap_or(Value::Null), - "vram_percent": event.payload.get("vram_percent").cloned().unwrap_or(Value::Null), - "hotspot_threshold_c": GPU_THERMAL_HOTSPOT_PRESSURE_C, - "memory_temperature_threshold_c": GPU_THERMAL_MEMORY_PRESSURE_C, - "vram_threshold_percent": GPU_MEMORY_VRAM_PRESSURE_PERCENT, - }), - ) -} - -fn resolve_gpu_pressure_service_target( - paths: &AppPaths, - event: &AutomationTriggerEvent, -) -> Result> { - if let Some(service_id) = event - .service_id - .as_deref() - .map(str::trim) - .filter(|value| !value.is_empty()) - { - let record = load_service_record(paths, service_id)?; - return Ok(active_pressure_target(record)); - } - - let active = crate::persistence::load_managed_services(paths)? - .into_iter() - .filter_map(active_pressure_target) - .collect::>(); - if active.len() == 1 { - Ok(active.into_iter().next()) - } else { - Ok(None) - } -} + )]); -fn active_pressure_target(record: ManagedServiceRecord) -> Option { - (record.mode == "managed" && matches!(record.status.as_str(), "ready" | "running")) - .then_some(record) -} - -fn pending_stop_proposal_exists(paths: &AppPaths, service_id: &str) -> Result { - pending_proposal_exists( - paths, - "gpu-thermal-protect", - service_id, - "queue_stop_server_proposal", - "stop_server", - ) -} - -fn pending_restart_proposal_exists(paths: &AppPaths, service_id: &str) -> Result { - pending_proposal_exists( - paths, - "server-recover", - service_id, - "queue_restart_proposal", - "restart_server", - ) -} - -/// Whether `watcher_id` already has a pending proposal for `service_id`, either -/// queued under `action` or carrying `tool`. -/// -/// Reads the whole proposals store rather than a recent window: a pending -/// proposal is never dropped, so it can be arbitrarily old, and a window that -/// missed it would let the watcher queue a duplicate once the original aged -/// out of that window. Compaction bounds the reviewed proposals to a fixed -/// count; pending ones are kept. -fn pending_proposal_exists( - paths: &AppPaths, - watcher_id: &str, - service_id: &str, - action: &str, - tool: &str, -) -> Result { - Ok(rocm_core::load_automation_proposals(paths)? - .into_iter() - .any(|proposal| { - proposal.status == "pending" - && proposal.watcher_id == watcher_id - && proposal.service_id.as_deref() == Some(service_id) - && (proposal.action == action || proposal.tool.as_deref() == Some(tool)) - })) -} - -fn handle_cache_warm_event( - paths: &AppPaths, - mode: WatcherMode, - state: &mut AutomationRuntimeState, - event: &AutomationTriggerEvent, -) -> Result<()> { - handle_cache_warm_event_with_resolver(paths, mode, state, event, |artifact_ref| { - resolve_model_recipe_artifact(artifact_ref).map(|resolved| resolved.is_some()) - }) -} - -fn handle_cache_warm_event_with_resolver( - paths: &AppPaths, - mode: WatcherMode, - state: &mut AutomationRuntimeState, - event: &AutomationTriggerEvent, - mut artifact_exists: F, -) -> Result<()> -where - F: FnMut(&str) -> Result, -{ - let Some(artifact_ref) = payload_string(&event.payload, "artifact_ref") else { - return crate::persistence::record_event( - paths, - state, - "cache-warm", - "warn", - "cache_warm_missing_artifact", - "cache warm event did not include artifact_ref; no prefetch proposal was queued", - None, - ); - }; - match artifact_exists(&artifact_ref) { - Ok(true) => {} - Ok(false) => { - return crate::persistence::record_event( - paths, - state, - "cache-warm", - "warn", - "cache_warm_unknown_artifact", - &format!( - "cache warm requested unknown registry artifact {artifact_ref}; no prefetch proposal was queued" - ), - None, - ); - } - Err(error) => { - return crate::persistence::record_event( - paths, - state, - "cache-warm", - "error", - "cache_warm_registry_error", - &format!( - "cache warm could not verify registry artifact {artifact_ref}: {error}; no prefetch proposal was queued" - ), - None, - ); - } - } - match mode { - WatcherMode::Observe => crate::persistence::record_event( - paths, - state, - "cache-warm", - "info", - "observe_cache_warm_request", - &format!( - "observed cache warm request for {artifact_ref}; observe mode does not queue or download artifacts" - ), - None, - ), - WatcherMode::Propose | WatcherMode::Contained => { - let action = "queue_prefetch_proposal"; - let message = if matches!(mode, WatcherMode::Contained) { - format!( - "cache warm requested for {artifact_ref}; contained mode still queues a review because artifact downloads require explicit source-policy approval" - ) - } else { - format!( - "cache warm requested for {artifact_ref}; queueing a reviewed prefetch proposal" - ) - }; - crate::persistence::record_event( - paths, - state, - "cache-warm", - "info", - action, - &message, - None, - )?; - queue_proposal_with_arguments( - paths, - "cache-warm", - action, - "Prefetch model artifact", - &message, - None, - json!({ - "artifact_ref": artifact_ref, - }), - ) - } - } -} - -fn handle_driver_upgrade_event( - paths: &AppPaths, - mode: WatcherMode, - state: &mut AutomationRuntimeState, - event: &AutomationTriggerEvent, -) -> Result<()> { - handle_driver_upgrade_event_with_runner(paths, mode, state, event, |paths| { - crate::sandbox::run_sandbox_tool( - paths, - crate::sandbox_tool::SandboxToolArg::DriverPlan, - None, - None, - None, - crate::sandbox_tool::SandboxToolPolicy::default(), - ) - }) -} - -fn handle_driver_upgrade_event_with_runner( - paths: &AppPaths, - mode: WatcherMode, - state: &mut AutomationRuntimeState, - event: &AutomationTriggerEvent, - driver_plan_runner: F, -) -> Result<()> -where - F: FnOnce(&AppPaths) -> Result, -{ - if payload_string(&event.payload, "component").as_deref() != Some("driver") { - return crate::persistence::record_event( - paths, - state, - "driver-upgrade", - "warn", - "driver_upgrade_ignored_component", - "driver-upgrade event did not include payload.component=driver; no driver plan proposal was queued", - None, - ); - } - - match mode { - WatcherMode::Observe => crate::persistence::record_event( - paths, - state, - "driver-upgrade", - "info", - "observe_driver_update", - "observed local driver update signal; observe mode does not queue or run a driver plan", - None, - ), - WatcherMode::Propose => { - let action = "prepare_driver_plan"; - let message = - "local driver update signal received; queueing a reviewed read-only driver plan"; - crate::persistence::record_event( - paths, - state, - "driver-upgrade", - "info", - action, - message, - None, - )?; - queue_proposal( - paths, - "driver-upgrade", - action, - "Review driver install plan", - message, - None, - ) - } - WatcherMode::Contained => match driver_plan_runner(paths) { - Ok(output) => match restricted_driver_plan_result(&output) { - Ok(result) => crate::persistence::record_event( - paths, - state, - "driver-upgrade", - if result.exit_status == 0 { - "info" - } else { - "error" - }, - "run_driver_plan", - &format!( - "local driver update signal received; contained restricted driver_plan status={}; no driver commands were executed", - result.status - ), - None, - ), - Err(error) => crate::persistence::record_event( - paths, - state, - "driver-upgrade", - "error", - "driver_plan_failed", - &format!( - "local driver update signal received, but contained restricted driver_plan failed: {error}; no driver commands were executed" - ), - None, - ), - }, - Err(error) => crate::persistence::record_event( - paths, - state, - "driver-upgrade", - "error", - "driver_plan_failed", - &format!( - "local driver update signal received, but contained restricted driver_plan failed: {error}; no driver commands were executed" - ), - None, - ), - }, - } -} - -struct RestrictedDriverPlanResult<'a> { - status: &'a str, - exit_status: i64, -} - -fn restricted_driver_plan_result(value: &Value) -> Result> { - let tool = value - .get("tool") - .and_then(Value::as_str) - .context("restricted driver plan did not report a tool name")?; - if tool != crate::sandbox_tool::SandboxToolArg::DriverPlan.as_cli_value() { - bail!("restricted driver plan returned `{tool}`, expected `driver_plan`"); - } - let status = value - .get("status") - .and_then(Value::as_str) - .unwrap_or("planned"); - let exit_status = value - .get("exit_status") - .and_then(Value::as_i64) - .unwrap_or_else(|| i64::from(status == "error")); - Ok(RestrictedDriverPlanResult { - status, - exit_status, - }) -} - -#[cfg(test)] -fn evaluate_server_recover( - paths: &AppPaths, - mode: WatcherMode, - state: &mut AutomationRuntimeState, -) -> Result<()> { - let now = unix_time_millis(); - if !server_recover_due(state, now) { - return Ok(()); - } - - let Some((mut record, recovery_reason)) = find_recoverable_service(paths)? else { - return Ok(()); - }; - let kind = service_recovery_event_kind(&recovery_reason); - let event = AutomationTriggerEvent { - at_unix_ms: now, - kind: kind.to_owned(), - source: "managed_service".to_owned(), - watcher_hint: Some("server-recover".to_owned()), - service_id: Some(record.service_id.clone()), - reason: Some(recovery_reason), - payload: json!({ - "engine": record.engine.clone(), - "status": record.status.clone(), - "endpoint": record.endpoint_url.clone(), - }), - }; - handle_server_recover_event_with_record(paths, mode, state, &event, &mut record) -} - -fn handle_server_recover_event( - paths: &AppPaths, - mode: WatcherMode, - state: &mut AutomationRuntimeState, - event: &AutomationTriggerEvent, -) -> Result<()> { - let service_id = event - .service_id - .as_deref() - .context("server-recover event is missing service_id")?; - let mut record = load_service_record(paths, service_id)?; - if !service_record_matches_recovery_event(paths, &record, event) { - crate::persistence::record_event( - paths, - state, - "server-recover", - "info", - "ignore_nonrecoverable_service", - &format!( - "managed service {} does not currently need recovery; restart not attempted", - record.service_id - ), - Some(record.service_id.clone()), - )?; - return Ok(()); - } - handle_server_recover_event_with_record(paths, mode, state, event, &mut record) -} - -fn service_record_matches_recovery_event( - paths: &AppPaths, - record: &ManagedServiceRecord, - event: &AutomationTriggerEvent, -) -> bool { - // Re-checked here as well as in the scan: a stop can be requested between - // the event being raised and it being handled. - if stop_requested(record) { - return false; - } - match event.kind.as_str() { - "service.manifest_recoverable" => { - manifest_service_recovery_reason(record, unix_time_millis()).is_some() - } - "service.endpoint_recoverable" => endpoint_service_recovery_reason(record).is_some(), - "service.healthcheck_recoverable" => crate::engine_ipc::engine_healthcheck_response( - paths, - &record.engine, - &record.service_id, - ) - .is_ok_and(|healthcheck| crate::engine_ipc::healthcheck_response_recoverable(&healthcheck)), - _ => false, - } -} - -fn handle_server_recover_event_with_record( - paths: &AppPaths, - mode: WatcherMode, - state: &mut AutomationRuntimeState, - event: &AutomationTriggerEvent, - record: &mut ManagedServiceRecord, -) -> Result<()> { - let now = unix_time_millis(); - let recovery_reason = event.reason.as_deref().unwrap_or("recoverable_event"); - let recovery_reason_display = display_recovery_reason(recovery_reason); - - match watcher_policy_action("server-recover", mode) { - WatcherPolicyAction::Observe => crate::persistence::record_event( - paths, - state, - "server-recover", - "warn", - "observe_failure", - &format!( - "observed managed service {} needing recovery ({recovery_reason_display}); restart not attempted in observe mode", - record.service_id, - ), - Some(record.service_id.clone()), - ), - WatcherPolicyAction::QueueProposal => { - // An unreadable proposals store is reported and skipped, never - // propagated: the daemon loop exits on an error out of - // `evaluate_watchers`, so a store that stays corrupt would take it - // down on every recovery tick. - match pending_restart_proposal_exists(paths, &record.service_id) { - Ok(true) => { - return crate::persistence::record_event( - paths, - state, - "server-recover", - "info", - "restart_proposal_already_pending", - &format!( - "managed service {} needs recovery ({recovery_reason_display}); a restart request awaiting review is already queued", - record.service_id, - ), - Some(record.service_id.clone()), - ); - } - Ok(false) => {} - Err(error) => { - return crate::persistence::record_event( - paths, - state, - "server-recover", - "error", - "restart_proposal_check_failed", - &format!( - "managed service {} needs recovery ({recovery_reason_display}); \ - not queueing a restart request because the proposals store could \ - not be checked for one already pending: {error:#}", - record.service_id, - ), - Some(record.service_id.clone()), - ); - } - } - let message = format!( - "managed service {} needs recovery ({recovery_reason_display}); queueing restart proposal", - record.service_id, - ); - crate::persistence::record_event( - paths, - state, - "server-recover", - "warn", - "queue_restart_proposal", - &message, - Some(record.service_id.clone()), - )?; - queue_proposal( - paths, - "server-recover", - "queue_restart_proposal", - "Restart managed service", - &message, - Some(record.service_id.clone()), - ) - } - WatcherPolicyAction::RunContained => { - if let Some(last_restart) = record.last_restart_unix_ms - && now.saturating_sub(last_restart) < SERVER_RECOVER_BACKOFF_MS - { - return Ok(()); - } - // A public service whose endpoint key is gone can never be recovered: - // the respawn guard in `supervise_service` refuses it by design. - // Report it and stop, rather than letting a permanent failure - // propagate out of `evaluate_watchers` and take the whole daemon — - // and every other watcher — down on each 30s recovery tick. - if let Err(error) = crate::engine_ipc::ensure_public_service_has_endpoint_key( - &record.host, - rocm_engine_protocol::endpoint_key_file_if_present(paths, &record.service_id) - .and_then(|path| rocm_engine_protocol::endpoint_api_key_file_if_valid(&path)) - .is_some(), - record.requires_api_key, - ) { - return crate::persistence::record_event( - paths, - state, - "server-recover", - "error", - "restart_managed_service_refused", - &format!( - "cannot recover managed service {} on {}:{} after \ - {recovery_reason_display}: {error}", - record.service_id, record.host, record.port - ), - Some(record.service_id.clone()), - ); - } - restart_managed_service(paths, &mut *record)?; - crate::persistence::record_event( - paths, - state, - "server-recover", - "info", - "restart_managed_service", - &format!( - "restarted managed service {} on {}:{} after {recovery_reason_display}", - record.service_id, record.host, record.port - ), - Some(record.service_id.clone()), - ) - } - } -} - -fn display_recovery_reason(reason: &str) -> String { - match reason { - "manifest_status_failed" => "manifest reports failed".to_owned(), - "manifest_status_exited" => "manifest reports exited".to_owned(), - "manifest_status_unreachable" => "manifest reports unreachable".to_owned(), - "manifest_status_starting_stale" => "service has been starting for too long".to_owned(), - "manifest_status_recovering_stale" => "service has been recovering for too long".to_owned(), - "endpoint_status_unreachable" => "endpoint port is unreachable".to_owned(), - other if other.starts_with("healthcheck_status_") => format!( - "engine healthcheck reports {}", - other.trim_start_matches("healthcheck_status_") - ), - other => other.replace('_', " "), - } -} - -fn service_recovery_event_kind(recovery_reason: &str) -> &'static str { - if recovery_reason.starts_with("healthcheck_status_") { - "service.healthcheck_recoverable" - } else if recovery_reason.starts_with("endpoint_status_") { - "service.endpoint_recoverable" - } else { - "service.manifest_recoverable" - } -} - -fn server_recover_due(state: &AutomationRuntimeState, now: u128) -> bool { - let Some(snapshot) = state - .active_watchers - .iter() - .find(|watcher| watcher.id == "server-recover" && watcher.enabled) - else { - return false; - }; - snapshot - .last_event_unix_ms - .is_none_or(|last_event| now.saturating_sub(last_event) >= SERVER_RECOVER_BACKOFF_MS) -} - -#[derive(Debug, Clone, Copy, Eq, PartialEq)] -enum WatcherPolicyAction { - Observe, - QueueProposal, - RunContained, -} - -const fn watcher_policy_action(watcher_id: &str, mode: WatcherMode) -> WatcherPolicyAction { - match (watcher_id, mode) { - (_, WatcherMode::Observe) => WatcherPolicyAction::Observe, - (_, WatcherMode::Propose) => WatcherPolicyAction::QueueProposal, - (_, WatcherMode::Contained) => WatcherPolicyAction::RunContained, - } -} - -pub(crate) fn find_recoverable_service( - paths: &AppPaths, -) -> Result> { - let now = unix_time_millis(); - for record in crate::persistence::load_managed_services(paths)? { - if record.mode != "managed" || stop_requested(&record) { - continue; - } - if let Some(reason) = manifest_service_recovery_reason(&record, now) { - return Ok(Some((record, reason))); - } - if matches!(record.status.as_str(), "ready" | "running") { - let Ok(healthcheck) = crate::engine_ipc::engine_healthcheck_response( - paths, - &record.engine, - &record.service_id, - ) else { - if let Some(reason) = endpoint_service_recovery_reason(&record) { - return Ok(Some((record, reason))); - } - continue; - }; - if crate::engine_ipc::healthcheck_response_recoverable(&healthcheck) { - return Ok(Some(( - record, - format!("healthcheck_status_{}", healthcheck.status), - ))); - } - if let Some(reason) = endpoint_service_recovery_reason(&record) { - return Ok(Some((record, reason))); - } - } - } - Ok(None) -} - -/// Whether an operator asked for this service to stop and that stop is still -/// standing. -/// -/// Such a service is never a recovery candidate, whatever its status says. A -/// stop that could not confirm every recorded process gone deliberately leaves -/// the status at `ready`/`running` — it did not happen, so it must not be -/// claimed — and a stop still in flight has not reached its verdict yet. Either -/// way the endpoint has stopped answering, which on its own is exactly what -/// recovery restarts. `stop_requested_unix_ms` is what tells the operator's -/// stop apart from a crash; a confirmed stop, and any fresh launch, clear it. -pub(crate) const fn stop_requested(record: &ManagedServiceRecord) -> bool { - record.stop_requested_unix_ms.is_some() -} - -fn endpoint_service_recovery_reason(record: &ManagedServiceRecord) -> Option { - (!wait_for_port(&record.host, record.port, ENDPOINT_HEALTH_TIMEOUT)) - .then(|| "endpoint_status_unreachable".to_owned()) -} - -fn wait_for_port(host: &str, port: u16, timeout: Duration) -> bool { - let address: SocketAddr = match format_host_port(host, port).parse() { - Ok(value) => value, - Err(_) => return false, - }; - - let start = std::time::Instant::now(); - while start.elapsed() < timeout { - if TcpStream::connect_timeout(&address, Duration::from_millis(200)).is_ok() { - return true; - } - thread::sleep(Duration::from_millis(200)); - } - false -} - -fn manifest_service_recovery_reason( - record: &ManagedServiceRecord, - now_unix_ms: u128, -) -> Option { - match record.status.as_str() { - "failed" | "exited" | "unreachable" => Some(format!("manifest_status_{}", record.status)), - "starting" | "recovering" => { - let started_at = record - .last_restart_unix_ms - .unwrap_or(record.created_at_unix_ms); - (now_unix_ms.saturating_sub(started_at) >= SERVER_TRANSIENT_STALE_MS) - .then(|| format!("manifest_status_{}_stale", record.status)) - } - _ => None, - } -} - -fn queue_proposal( - paths: &AppPaths, - watcher_id: &str, - action: &str, - title: &str, - message: &str, - service_id: Option, -) -> Result<()> { - queue_proposal_with_arguments( - paths, - watcher_id, - action, - title, - message, - service_id.clone(), - proposal_arguments_for_action(action, service_id.as_deref()), - ) -} - -fn queue_proposal_with_arguments( - paths: &AppPaths, - watcher_id: &str, - action: &str, - title: &str, - message: &str, - service_id: Option, - arguments: Value, -) -> Result<()> { - append_automation_proposal( - paths, - &AutomationProposalRecord { - at_unix_ms: unix_time_millis(), - proposal_id: String::new(), - watcher_id: watcher_id.to_owned(), - action: action.to_owned(), - title: title.to_owned(), - message: message.to_owned(), - status: "pending".to_owned(), - service_id, - tool: proposal_tool_for_action(action).map(str::to_owned), - arguments, - reviewed_at_unix_ms: None, - }, - ) -} - -fn proposal_tool_for_action(action: &str) -> Option<&'static str> { - match action { - "queue_restart_proposal" => Some("restart_server"), - "queue_stop_server_proposal" => Some("stop_server"), - "queue_update_proposal" => Some("check_updates"), - "queue_prefetch_proposal" => Some("prefetch_artifact"), - "prepare_driver_plan" => Some("driver_plan"), - _ => None, - } -} - -fn proposal_arguments_for_action(action: &str, service_id: Option<&str>) -> Value { - match action { - "queue_restart_proposal" => json!({ - "service_id": service_id, - }), - "queue_stop_server_proposal" => json!({ - "service_id": service_id, - }), - "queue_update_proposal" => json!({}), - "prepare_driver_plan" => json!({}), - _ => Value::Null, - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_support::{ - UNCONFIRMED_STOP_PID, seed_keyed_service, stop_with_outcome, temp_app_paths, - }; - use rocm_core::{AuditEventRecord, AutomationEventRecord}; - use std::fs; - - fn test_watcher_snapshot( - id: &str, - mode: WatcherMode, - last_event_unix_ms: Option, - ) -> WatcherRuntimeSnapshot { - WatcherRuntimeSnapshot { - id: id.to_owned(), - enabled: true, - mode, - summary: "test watcher".to_owned(), - last_event: None, - last_event_unix_ms, - } - } - - fn test_runtime_state(active_watchers: Vec) -> AutomationRuntimeState { - AutomationRuntimeState { - running: true, - automations_enabled: true, - daemon_pid: 1, - started_at_unix_ms: 1, - last_tick_unix_ms: 1, - local_webhook_endpoint: None, - active_watchers, - } - } - - #[test] - fn event_collector_emits_schedule_tick_for_due_update() -> Result<()> { - let (root, paths) = temp_app_paths("event-bus-schedule"); - let state = test_runtime_state(vec![test_watcher_snapshot( - "therock-update", - WatcherMode::Observe, - None, - )]); - - let events = collect_automation_events(&paths, &RocmCliConfig::default(), &state)?; - fs::remove_dir_all(root).ok(); - - let event = events - .iter() - .find(|event| event.watcher_hint.as_deref() == Some("therock-update")) - .expect("schedule tick event should be emitted"); - assert_eq!(event.kind, "schedule.tick"); - assert_eq!(event.source, "scheduler"); - assert_eq!(event.reason.as_deref(), Some("therock_update_interval_due")); - Ok(()) - } - - #[test] - fn event_collector_emits_recoverable_service_event() -> Result<()> { - let (root, paths) = temp_app_paths("event-bus-service"); - paths.ensure()?; - let mut failed = ManagedServiceRecord::new( - &paths, - "svc-failed", - "vllm", - "qwen", - "Qwen/Qwen3.5", - "127.0.0.1", - 11435, - "managed", - 123, - None, - None, - None, - ); - failed.status = "failed".to_owned(); - failed.write()?; - let state = test_runtime_state(vec![test_watcher_snapshot( - "server-recover", - WatcherMode::Propose, - None, - )]); - - let events = collect_automation_events(&paths, &RocmCliConfig::default(), &state)?; - fs::remove_dir_all(root).ok(); - - let event = events - .iter() - .find(|event| event.watcher_hint.as_deref() == Some("server-recover")) - .expect("recoverable service event should be emitted"); - assert_eq!(event.kind, "service.manifest_recoverable"); - assert_eq!(event.service_id.as_deref(), Some("svc-failed")); - assert_eq!(event.reason.as_deref(), Some("manifest_status_failed")); - Ok(()) - } - - #[test] - fn event_collector_emits_endpoint_recoverable_service_event() -> Result<()> { - let (root, paths) = temp_app_paths("event-bus-endpoint"); - paths.ensure()?; - let mut service = ManagedServiceRecord::new( - &paths, - "svc-endpoint", - "missing-engine", - "qwen", - "Qwen/Qwen3.5", - "127.0.0.1", - 1, - "managed", - 123, - None, - None, - None, - ); - service.status = "ready".to_owned(); - service.write()?; - let state = test_runtime_state(vec![test_watcher_snapshot( - "server-recover", - WatcherMode::Propose, - None, - )]); - - let events = collect_automation_events(&paths, &RocmCliConfig::default(), &state)?; - fs::remove_dir_all(root).ok(); - - let event = events - .iter() - .find(|event| event.watcher_hint.as_deref() == Some("server-recover")) - .expect("endpoint recoverable service event should be emitted"); - assert_eq!(event.kind, "service.endpoint_recoverable"); - assert_eq!(event.service_id.as_deref(), Some("svc-endpoint")); - assert_eq!(event.reason.as_deref(), Some("endpoint_status_unreachable")); - Ok(()) - } - - #[test] - fn event_collector_emits_gpu_metrics_event_when_enabled() -> Result<()> { - let (root, paths) = temp_app_paths("event-bus-gpu-metrics"); - let state = test_runtime_state(vec![test_watcher_snapshot( - "gpu-metrics", - WatcherMode::Observe, - None, - )]); - - let events = collect_automation_events_with_gpu_snapshot(&paths, &state, || { - CodexBridgeGpuSnapshot { - amd_smi_available: true, - static_snapshot: Some(json!({ - "gpu_data": [ - { "gpu": 0, "asic": { "market_name": "AMD Radeon Test" } } - ] - })), - monitor_snapshot: Some(json!({ "gpu_data": [] })), - note: None, - } - })?; - fs::remove_dir_all(root).ok(); - - let event = events - .iter() - .find(|event| event.watcher_hint.as_deref() == Some("gpu-metrics")) - .expect("gpu metrics event should be emitted"); - assert_eq!(event.kind, "gpu.metrics"); - assert_eq!(event.source, "gpu_telemetry"); - assert_eq!(event.reason.as_deref(), Some("amd_smi_snapshot_available")); - assert_eq!( - event - .payload - .get("monitor_available") - .and_then(Value::as_bool), - Some(true) - ); - assert!( - event - .payload - .get("summary") - .and_then(Value::as_str) - .is_some_and(|summary| summary.contains("gpu_count=1")) - ); - Ok(()) - } - - #[test] - fn event_collector_emits_gpu_thermal_pressure_event_when_enabled() -> Result<()> { - let (root, paths) = temp_app_paths("event-bus-gpu-thermal-pressure"); - let state = test_runtime_state(vec![test_watcher_snapshot( - "gpu-thermal-protect", - WatcherMode::Propose, - None, - )]); - - let events = collect_automation_events_with_gpu_snapshot(&paths, &state, || { - CodexBridgeGpuSnapshot { - amd_smi_available: true, - static_snapshot: None, - monitor_snapshot: Some(json!({ - "gpu_data": [ - { - "gpu": 0, - "hotspot_temperature": { "value": 96.0 }, - "memory_temperature": { "value": 88.0 }, - "vram_percent": { "value": 72.0 } - } - ] - })), - note: None, - } - })?; - fs::remove_dir_all(root).ok(); + let events = collect_automation_events_with_gpu_snapshot(&paths, &state, || { + CodexBridgeGpuSnapshot { + amd_smi_available: true, + static_snapshot: None, + monitor_snapshot: Some(json!({ + "gpu_data": [ + { + "gpu": 0, + "hotspot_temperature": { "value": 96.0 }, + "memory_temperature": { "value": 88.0 }, + "vram_percent": { "value": 72.0 } + } + ] + })), + note: None, + } + })?; + fs::remove_dir_all(root).ok(); let event = events .iter() @@ -1791,1747 +427,304 @@ mod tests { assert_eq!(event.source, "gpu_telemetry"); assert_eq!( event.reason.as_deref(), - Some("hotspot_temperature_threshold") - ); - assert!( - event - .payload - .get("summary") - .and_then(Value::as_str) - .is_some_and(|summary| summary.contains("GPU 0 hotspot temperature is 96 C")) - ); - Ok(()) - } - - #[test] - fn event_collector_skips_gpu_pressure_below_thresholds() -> Result<()> { - let (root, paths) = temp_app_paths("event-bus-gpu-pressure-cool"); - let state = test_runtime_state(vec![test_watcher_snapshot( - "gpu-thermal-protect", - WatcherMode::Propose, - None, - )]); - - let events = collect_automation_events_with_gpu_snapshot(&paths, &state, || { - CodexBridgeGpuSnapshot { - amd_smi_available: true, - static_snapshot: None, - monitor_snapshot: Some(json!([ - { - "gpu": 0, - "hotspot_temperature": 80.0, - "memory_temperature": 82.0, - "vram_percent": 50.0 - } - ])), - note: None, - } - })?; - fs::remove_dir_all(root).ok(); - - assert!( - !events - .iter() - .any(|event| event.watcher_hint.as_deref() == Some("gpu-thermal-protect")) - ); - Ok(()) - } - - #[test] - fn gpu_metrics_event_records_read_only_status_without_proposal() -> Result<()> { - let (root, paths) = temp_app_paths("gpu-metrics-record"); - paths.ensure()?; - let mut state = test_runtime_state(vec![test_watcher_snapshot( - "gpu-metrics", - WatcherMode::Contained, - None, - )]); - let mut config = RocmCliConfig::default(); - let watcher = config.watcher_config_mut("gpu-metrics"); - watcher.enabled = true; - watcher.mode = Some(WatcherMode::Contained); - let events = vec![AutomationTriggerEvent { - at_unix_ms: 1, - kind: "gpu.metrics_unavailable".to_owned(), - source: "gpu_telemetry".to_owned(), - watcher_hint: Some("gpu-metrics".to_owned()), - service_id: None, - reason: Some("amd-smi missing".to_owned()), - payload: json!({ - "summary": "amd_smi_available=false static_snapshot=missing monitor_snapshot=missing", - }), - }]; - - evaluate_watchers_for_events(&paths, &config, &mut state, &events)?; - let event_text = fs::read_to_string(paths.automation_events_path())?; - let event = serde_json::from_str::(event_text.trim())?; - let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; - fs::remove_dir_all(root).ok(); - - assert_eq!(event.watcher_id, "gpu-metrics"); - assert_eq!(event.action, "record_gpu_metrics"); - assert!(event.message.contains("telemetry is recorded only")); - assert!(event.message.contains("amd-smi missing")); - assert!(event.message.contains("no mutating action was taken")); - assert!(proposals.is_empty()); - Ok(()) - } - - #[test] - fn gpu_thermal_protect_propose_queues_reviewed_stop_for_one_running_service() -> Result<()> { - let (root, paths) = temp_app_paths("gpu-thermal-protect-propose"); - paths.ensure()?; - let mut record = ManagedServiceRecord::new( - &paths, - "svc-hot", - "vllm", - "tiny", - "Tiny/Test", - "127.0.0.1", - 11435, - "managed", - 123, - None, - None, - None, - ); - record.status = "ready".to_owned(); - record.write()?; - let mut state = test_runtime_state(vec![test_watcher_snapshot( - "gpu-thermal-protect", - WatcherMode::Propose, - None, - )]); - let mut config = RocmCliConfig::default(); - let watcher = config.watcher_config_mut("gpu-thermal-protect"); - watcher.enabled = true; - watcher.mode = Some(WatcherMode::Propose); - let event = AutomationTriggerEvent { - at_unix_ms: 1, - kind: "gpu.thermal_pressure".to_owned(), - source: "gpu_telemetry".to_owned(), - watcher_hint: Some("gpu-thermal-protect".to_owned()), - service_id: None, - reason: Some("hotspot_temperature_threshold".to_owned()), - payload: json!({ - "gpu": 0, - "summary": "GPU 0 hotspot temperature is 96 C (limit 95 C)", - "hotspot_temperature_c": 96.0, - }), - }; - - evaluate_watchers_for_events(&paths, &config, &mut state, &[event])?; - let event_text = fs::read_to_string(paths.automation_events_path())?; - let event = serde_json::from_str::(event_text.trim())?; - let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; - let saved = load_service_record(&paths, "svc-hot")?; - fs::remove_dir_all(root).ok(); - - assert_eq!(event.watcher_id, "gpu-thermal-protect"); - assert_eq!(event.action, "queue_stop_server_proposal"); - assert_eq!(event.service_id.as_deref(), Some("svc-hot")); - assert!(event.message.contains("asking before stopping anything")); - assert_eq!(saved.status, "ready"); - assert_eq!(proposals.len(), 1); - assert_eq!(proposals[0].tool.as_deref(), Some("stop_server")); - assert_eq!(proposals[0].service_id.as_deref(), Some("svc-hot")); - assert_eq!( - proposals[0] - .arguments - .get("pressure_reason") - .and_then(Value::as_str), - Some("hotspot_temperature_threshold") - ); - Ok(()) - } - - #[test] - fn gpu_thermal_protect_contained_still_queues_reviewed_stop() -> Result<()> { - let (root, paths) = temp_app_paths("gpu-thermal-protect-contained"); - paths.ensure()?; - let mut record = ManagedServiceRecord::new( - &paths, - "svc-hot", - "vllm", - "qwen", - "Qwen/Test", - "127.0.0.1", - 11436, - "managed", - 123, - None, - None, - None, - ); - record.status = "running".to_owned(); - record.write()?; - let mut state = test_runtime_state(vec![test_watcher_snapshot( - "gpu-thermal-protect", - WatcherMode::Contained, - None, - )]); - let event = AutomationTriggerEvent { - at_unix_ms: 1, - kind: "gpu.memory_pressure".to_owned(), - source: "gpu_telemetry".to_owned(), - watcher_hint: Some("gpu-thermal-protect".to_owned()), - service_id: Some("svc-hot".to_owned()), - reason: Some("vram_pressure_threshold".to_owned()), - payload: json!({ - "summary": "GPU 0 VRAM use is 96% (limit 95%)", - "vram_percent": 96.0, - }), - }; - - handle_gpu_thermal_protect_event(&paths, WatcherMode::Contained, &mut state, &event)?; - let event_text = fs::read_to_string(paths.automation_events_path())?; - let event = serde_json::from_str::(event_text.trim())?; - let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; - let saved = load_service_record(&paths, "svc-hot")?; - fs::remove_dir_all(root).ok(); - - assert_eq!(event.action, "queue_stop_server_proposal"); - assert!(event.message.contains("contained mode still asks")); - assert_eq!(saved.status, "running"); - assert_eq!(proposals.len(), 1); - assert_eq!(proposals[0].tool.as_deref(), Some("stop_server")); - Ok(()) - } - - #[test] - fn gpu_thermal_protect_observe_records_without_proposal() -> Result<()> { - let (root, paths) = temp_app_paths("gpu-thermal-protect-observe"); - paths.ensure()?; - let mut state = test_runtime_state(vec![test_watcher_snapshot( - "gpu-thermal-protect", - WatcherMode::Observe, - None, - )]); - let event = AutomationTriggerEvent { - at_unix_ms: 1, - kind: "gpu.thermal_pressure".to_owned(), - source: "gpu_telemetry".to_owned(), - watcher_hint: Some("gpu-thermal-protect".to_owned()), - service_id: None, - reason: Some("memory_temperature_threshold".to_owned()), - payload: json!({ - "summary": "GPU memory temperature is high", - }), - }; - - handle_gpu_thermal_protect_event(&paths, WatcherMode::Observe, &mut state, &event)?; - let event_text = fs::read_to_string(paths.automation_events_path())?; - let event = serde_json::from_str::(event_text.trim())?; - let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; - fs::remove_dir_all(root).ok(); - - assert_eq!(event.action, "observe_gpu_pressure"); - assert!(event.message.contains("does not stop any model server")); - assert!(proposals.is_empty()); - Ok(()) - } - - #[test] - fn gpu_thermal_protect_ambiguous_services_records_no_action() -> Result<()> { - let (root, paths) = temp_app_paths("gpu-thermal-protect-ambiguous"); - paths.ensure()?; - for service_id in ["svc-a", "svc-b"] { - let mut record = ManagedServiceRecord::new( - &paths, - service_id, - "vllm", - "qwen", - "Qwen/Test", - "127.0.0.1", - 11435, - "managed", - 123, - None, - None, - None, - ); - record.status = "ready".to_owned(); - record.write()?; - } - let mut state = test_runtime_state(vec![test_watcher_snapshot( - "gpu-thermal-protect", - WatcherMode::Propose, - None, - )]); - let event = AutomationTriggerEvent { - at_unix_ms: 1, - kind: "gpu.thermal_pressure".to_owned(), - source: "gpu_telemetry".to_owned(), - watcher_hint: Some("gpu-thermal-protect".to_owned()), - service_id: None, - reason: Some("hotspot_temperature_threshold".to_owned()), - payload: json!({ - "summary": "GPU 0 hotspot temperature is 96 C (limit 95 C)", - }), - }; - - handle_gpu_thermal_protect_event(&paths, WatcherMode::Propose, &mut state, &event)?; - let event_text = fs::read_to_string(paths.automation_events_path())?; - let event = serde_json::from_str::(event_text.trim())?; - let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; - fs::remove_dir_all(root).ok(); - - assert_eq!(event.action, "gpu_pressure_no_clear_target"); - assert!(event.message.contains("did not choose a model server")); - assert!(proposals.is_empty()); - Ok(()) - } - - #[test] - fn gpu_thermal_protect_does_not_duplicate_pending_stop_proposals() -> Result<()> { - let (root, paths) = temp_app_paths("gpu-thermal-protect-dedupe"); - paths.ensure()?; - let mut record = ManagedServiceRecord::new( - &paths, - "svc-hot", - "vllm", - "tiny", - "Tiny/Test", - "127.0.0.1", - 11435, - "managed", - 123, - None, - None, - None, - ); - record.status = "ready".to_owned(); - record.write()?; - let mut state = test_runtime_state(vec![test_watcher_snapshot( - "gpu-thermal-protect", - WatcherMode::Propose, - None, - )]); - let event = AutomationTriggerEvent { - at_unix_ms: 1, - kind: "gpu.thermal_pressure".to_owned(), - source: "gpu_telemetry".to_owned(), - watcher_hint: Some("gpu-thermal-protect".to_owned()), - service_id: Some("svc-hot".to_owned()), - reason: Some("hotspot_temperature_threshold".to_owned()), - payload: json!({ - "summary": "GPU 0 hotspot temperature is 96 C (limit 95 C)", - }), - }; - - handle_gpu_thermal_protect_event(&paths, WatcherMode::Propose, &mut state, &event)?; - handle_gpu_thermal_protect_event(&paths, WatcherMode::Propose, &mut state, &event)?; - let events = rocm_core::load_recent_automation_events(&paths, 2)?; - let proposals = rocm_core::load_recent_automation_proposals(&paths, 10)?; - fs::remove_dir_all(root).ok(); - - assert_eq!(proposals.len(), 1); - assert_eq!(proposals[0].tool.as_deref(), Some("stop_server")); - assert!( - events - .iter() - .any(|event| event.action == "stop_proposal_already_pending") - ); - Ok(()) - } - - /// How many pending proposals to queue behind the one under test so it sits - /// beyond a recent-window read: more than the reviewed-retention cap, and - /// more than the 100 the code once read, so the tests keep guarding that - /// window even if the cap is ever lowered below it. - fn proposals_beyond_a_recent_window() -> usize { - const WINDOW_THE_CODE_ONCE_READ: usize = 100; - rocm_core::REVIEWED_PROPOSALS_RETAINED.max(WINDOW_THE_CODE_ONCE_READ) + 1 - } - - #[test] - fn gpu_thermal_protect_dedupe_sees_a_pending_stop_older_than_the_recent_window() -> Result<()> { - let (root, paths) = temp_app_paths("gpu-thermal-protect-dedupe-old"); - let mut record = ManagedServiceRecord::new( - &paths, - "svc-hot", - "vllm", - "tiny", - "Tiny/Test", - "127.0.0.1", - 11435, - "managed", - 123, - None, - None, - None, - ); - record.status = "ready".to_owned(); - record.write()?; - let mut state = test_runtime_state(vec![test_watcher_snapshot( - "gpu-thermal-protect", - WatcherMode::Propose, - None, - )]); - let event = AutomationTriggerEvent { - at_unix_ms: 1, - kind: "gpu.thermal_pressure".to_owned(), - source: "gpu_telemetry".to_owned(), - watcher_hint: Some("gpu-thermal-protect".to_owned()), - service_id: Some("svc-hot".to_owned()), - reason: Some("hotspot_temperature_threshold".to_owned()), - payload: json!({ - "summary": "GPU 0 hotspot temperature is 96 C (limit 95 C)", - }), - }; - - handle_gpu_thermal_protect_event(&paths, WatcherMode::Propose, &mut state, &event)?; - // More newer proposals than the store keeps reviewed ones (all pending - // here, so none is compacted away), which puts the stop proposal beyond - // a recent-window read the size of that cap. A pending proposal is - // never dropped from the store, so it is still waiting for review. - for index in 0..proposals_beyond_a_recent_window() { - queue_proposal_with_arguments( - &paths, - "gpu-thermal-protect", - "queue_stop_server_proposal", - "Stop another server", - "unrelated proposal", - Some(format!("svc-other-{index}")), - json!({}), - )?; - } - handle_gpu_thermal_protect_event(&paths, WatcherMode::Propose, &mut state, &event)?; - let stops_for_hot_server = rocm_core::load_automation_proposals(&paths)? - .into_iter() - .filter(|proposal| { - proposal.tool.as_deref() == Some("stop_server") - && proposal.service_id.as_deref() == Some("svc-hot") - && proposal.status == "pending" - }) - .count(); - let events = rocm_core::load_recent_automation_events(&paths, 2)?; - fs::remove_dir_all(root).ok(); - - assert_eq!(stops_for_hot_server, 1); - assert!( - events - .iter() - .any(|event| event.action == "stop_proposal_already_pending") - ); - Ok(()) - } - - #[test] - fn server_recover_does_not_duplicate_pending_restart_proposals() -> Result<()> { - let (root, paths) = temp_app_paths("server-recover-dedupe"); - let mut record = ManagedServiceRecord::new( - &paths, - "svc-unhealthy", - "vllm", - "tiny", - "Tiny/Test", - "127.0.0.1", - 11436, - "managed", - 123, - None, - None, - None, - ); - record.status = "unreachable".to_owned(); - record.write()?; - let mut state = test_runtime_state(vec![test_watcher_snapshot( - "server-recover", - WatcherMode::Propose, - None, - )]); - let event = AutomationTriggerEvent { - at_unix_ms: 1, - kind: "service.manifest_recoverable".to_owned(), - source: "managed_service".to_owned(), - watcher_hint: Some("server-recover".to_owned()), - service_id: Some(record.service_id.clone()), - reason: Some("manifest_status_failed".to_owned()), - payload: json!({}), - }; - - handle_server_recover_event_with_record( - &paths, - WatcherMode::Propose, - &mut state, - &event, - &mut record, - )?; - handle_server_recover_event_with_record( - &paths, - WatcherMode::Propose, - &mut state, - &event, - &mut record, - )?; - let events = rocm_core::load_recent_automation_events(&paths, 2)?; - let proposals = rocm_core::load_recent_automation_proposals(&paths, 10)?; - fs::remove_dir_all(root).ok(); - - assert_eq!(proposals.len(), 1); - assert_eq!(proposals[0].tool.as_deref(), Some("restart_server")); - // The wording describes a proposal still awaiting review; the single - // pending proposal asserted here is what makes that true. - assert_eq!(proposals[0].status, "pending"); - assert!(events.iter().any(|event| { - event.action == "restart_proposal_already_pending" - && event.message.contains("a restart request awaiting review") - })); - Ok(()) - } - - #[test] - fn server_recover_dedupe_sees_a_pending_restart_older_than_the_recent_window() -> Result<()> { - let (root, paths) = temp_app_paths("server-recover-dedupe-old"); - let mut record = ManagedServiceRecord::new( - &paths, - "svc-unhealthy", - "vllm", - "tiny", - "Tiny/Test", - "127.0.0.1", - 11436, - "managed", - 123, - None, - None, - None, - ); - record.status = "unreachable".to_owned(); - record.write()?; - let mut state = test_runtime_state(vec![test_watcher_snapshot( - "server-recover", - WatcherMode::Propose, - None, - )]); - let event = AutomationTriggerEvent { - at_unix_ms: 1, - kind: "service.manifest_recoverable".to_owned(), - source: "managed_service".to_owned(), - watcher_hint: Some("server-recover".to_owned()), - service_id: Some(record.service_id.clone()), - reason: Some("manifest_status_failed".to_owned()), - payload: json!({}), - }; - - handle_server_recover_event_with_record( - &paths, - WatcherMode::Propose, - &mut state, - &event, - &mut record, - )?; - // More newer proposals than the store keeps reviewed ones (all pending - // here, so none is compacted away), which puts the restart proposal - // beyond a recent-window read the size of that cap. - for index in 0..proposals_beyond_a_recent_window() { - queue_proposal_with_arguments( - &paths, - "gpu-thermal-protect", - "queue_stop_server_proposal", - "Stop another server", - "unrelated proposal", - Some(format!("svc-other-{index}")), - json!({}), - )?; - } - handle_server_recover_event_with_record( - &paths, - WatcherMode::Propose, - &mut state, - &event, - &mut record, - )?; - let restarts = rocm_core::load_automation_proposals(&paths)? - .into_iter() - .filter(|proposal| proposal.tool.as_deref() == Some("restart_server")) - .count(); - fs::remove_dir_all(root).ok(); - - assert_eq!(restarts, 1); - Ok(()) - } - - #[test] - fn server_recover_dedupe_ignores_proposals_that_do_not_block_a_restart() -> Result<()> { - let (root, paths) = temp_app_paths("server-recover-dedupe-filters"); - let mut record = ManagedServiceRecord::new( - &paths, - "svc-unhealthy", - "vllm", - "tiny", - "Tiny/Test", - "127.0.0.1", - 11436, - "managed", - 123, - None, - None, - None, - ); - record.status = "unreachable".to_owned(); - record.write()?; - // Each of these matches the restart check on all but one field, so - // none of them may suppress a new restart for `svc-unhealthy`. - for (watcher_id, service_id, status) in [ - ("server-recover", "svc-unhealthy", "approved"), - ("server-recover", "svc-other", "pending"), - ("another-watcher", "svc-unhealthy", "pending"), - ] { - append_automation_proposal( - &paths, - &AutomationProposalRecord { - at_unix_ms: unix_time_millis(), - proposal_id: String::new(), - watcher_id: watcher_id.to_owned(), - action: "queue_restart_proposal".to_owned(), - title: "Restart".to_owned(), - message: "unrelated".to_owned(), - status: status.to_owned(), - service_id: Some(service_id.to_owned()), - tool: Some("restart_server".to_owned()), - arguments: json!({}), - reviewed_at_unix_ms: None, - }, - )?; - } - let mut state = test_runtime_state(vec![test_watcher_snapshot( - "server-recover", - WatcherMode::Propose, - None, - )]); - let event = AutomationTriggerEvent { - at_unix_ms: 1, - kind: "service.manifest_recoverable".to_owned(), - source: "managed_service".to_owned(), - watcher_hint: Some("server-recover".to_owned()), - service_id: Some(record.service_id.clone()), - reason: Some("manifest_status_failed".to_owned()), - payload: json!({}), - }; - - handle_server_recover_event_with_record( - &paths, - WatcherMode::Propose, - &mut state, - &event, - &mut record, - )?; - let queued = rocm_core::load_automation_proposals(&paths)? - .into_iter() - .filter(|proposal| { - proposal.watcher_id == "server-recover" - && proposal.status == "pending" - && proposal.service_id.as_deref() == Some("svc-unhealthy") - }) - .count(); - fs::remove_dir_all(root).ok(); - - assert_eq!(queued, 1); - Ok(()) - } - - #[test] - fn server_recover_reports_an_unreadable_proposals_store_instead_of_failing() -> Result<()> { - let (root, paths) = temp_app_paths("server-recover-corrupt-store"); - paths.ensure()?; - fs::write(paths.automation_proposals_path(), "not a proposal\n")?; - let mut record = ManagedServiceRecord::new( - &paths, - "svc-unhealthy", - "vllm", - "tiny", - "Tiny/Test", - "127.0.0.1", - 11436, - "managed", - 123, - None, - None, - None, - ); - record.status = "unreachable".to_owned(); - record.write()?; - let mut state = test_runtime_state(vec![test_watcher_snapshot( - "server-recover", - WatcherMode::Propose, - None, - )]); - let event = AutomationTriggerEvent { - at_unix_ms: 1, - kind: "service.manifest_recoverable".to_owned(), - source: "managed_service".to_owned(), - watcher_hint: Some("server-recover".to_owned()), - service_id: Some(record.service_id.clone()), - reason: Some("manifest_status_failed".to_owned()), - payload: json!({}), - }; - - let outcome = handle_server_recover_event_with_record( - &paths, - WatcherMode::Propose, - &mut state, - &event, - &mut record, - ); - let events = rocm_core::load_recent_automation_events(&paths, 1)?; - let stored = fs::read_to_string(paths.automation_proposals_path())?; - fs::remove_dir_all(root).ok(); - - // An error out of a watcher ends the daemon loop, so a corrupt store - // must surface as an event and a skipped tick, not as `Err`. - outcome?; - assert_eq!(stored, "not a proposal\n", "nothing may be queued"); - assert!(events.iter().any(|event| { - event.level == "error" && event.action == "restart_proposal_check_failed" - })); - Ok(()) - } - - #[test] - fn local_webhook_gpu_metrics_event_uses_existing_read_only_policy() -> Result<()> { - let (root, paths) = temp_app_paths("local-webhook-gpu-metrics"); - paths.ensure()?; - let mut state = test_runtime_state(vec![test_watcher_snapshot( - "gpu-metrics", - WatcherMode::Contained, - None, - )]); - let mut config = RocmCliConfig::default(); - let watcher = config.watcher_config_mut("gpu-metrics"); - watcher.enabled = true; - watcher.mode = Some(WatcherMode::Contained); - let event = crate::webhook::local_webhook_event_from_request( - crate::webhook::LocalWebhookEventRequest { - watcher_hint: "gpu-metrics".to_owned(), - kind: "gpu.metrics".to_owned(), - service_id: None, - reason: Some("manual smoke".to_owned()), - payload: json!({ - "summary": "manual webhook probe", - "action": "restart_server", - "mode": "contained", - }), - }, - )?; - - evaluate_watchers_for_events(&paths, &config, &mut state, &[event])?; - let event_text = fs::read_to_string(paths.automation_events_path())?; - let event = serde_json::from_str::(event_text.trim())?; - let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; - fs::remove_dir_all(root).ok(); - - assert_eq!(event.watcher_id, "gpu-metrics"); - assert_eq!(event.action, "record_gpu_metrics"); - assert!(event.message.contains("from local webhook")); - assert!(event.message.contains("manual smoke")); - assert!(event.message.contains("no mutating action was taken")); - assert!(proposals.is_empty()); - Ok(()) - } - - #[test] - fn cache_warm_propose_mode_queues_prefetch_proposal() -> Result<()> { - let (root, paths) = temp_app_paths("cache-warm-propose"); - paths.ensure()?; - let mut state = test_runtime_state(vec![test_watcher_snapshot( - "cache-warm", - WatcherMode::Propose, - None, - )]); - let event = crate::webhook::local_webhook_event_from_request( - crate::webhook::LocalWebhookEventRequest { - watcher_hint: "cache-warm".to_owned(), - kind: "cache.warm".to_owned(), - service_id: None, - reason: Some("idle window".to_owned()), - payload: json!({ - "artifact_ref": "Qwen/Test-1B#hf-main", - "tool": "restart_server", - "allow_artifact_download": true, - "artifact_max_bytes": 1024, - }), - }, - )?; - - handle_cache_warm_event_with_resolver( - &paths, - WatcherMode::Propose, - &mut state, - &event, - |_| Ok(true), - )?; - let event_text = fs::read_to_string(paths.automation_events_path())?; - let event = serde_json::from_str::(event_text.trim())?; - let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; - fs::remove_dir_all(root).ok(); - - assert_eq!(event.watcher_id, "cache-warm"); - assert_eq!(event.action, "queue_prefetch_proposal"); - assert_eq!(proposals.len(), 1); - assert_eq!(proposals[0].watcher_id, "cache-warm"); - assert_eq!(proposals[0].tool.as_deref(), Some("prefetch_artifact")); - assert_eq!( - proposals[0] - .arguments - .get("artifact_ref") - .and_then(Value::as_str), - Some("Qwen/Test-1B#hf-main") - ); - assert!( - proposals[0].arguments.get("tool").is_none(), - "webhook payload must not grant arbitrary tool choice" - ); - assert!( - proposals[0] - .arguments - .get("allow_artifact_download") - .is_none(), - "webhook payload must not grant source-policy approval" - ); - assert!( - proposals[0].arguments.get("artifact_max_bytes").is_none(), - "webhook payload must not grant download byte-limit approval" - ); - Ok(()) - } - - #[test] - fn cache_warm_unknown_artifact_does_not_queue_proposal() -> Result<()> { - let (root, paths) = temp_app_paths("cache-warm-unknown"); - paths.ensure()?; - let mut state = test_runtime_state(vec![test_watcher_snapshot( - "cache-warm", - WatcherMode::Propose, - None, - )]); - let event = AutomationTriggerEvent { - at_unix_ms: 1, - kind: "cache.warm".to_owned(), - source: "test".to_owned(), - watcher_hint: Some("cache-warm".to_owned()), - service_id: None, - reason: Some("idle window".to_owned()), - payload: json!({ - "artifact_ref": "missing#artifact", - }), - }; - - handle_cache_warm_event_with_resolver( - &paths, - WatcherMode::Propose, - &mut state, - &event, - |_| Ok(false), - )?; - let event_text = fs::read_to_string(paths.automation_events_path())?; - let event = serde_json::from_str::(event_text.trim())?; - let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; - fs::remove_dir_all(root).ok(); - - assert_eq!(event.action, "cache_warm_unknown_artifact"); - assert!(event.message.contains("unknown registry artifact")); - assert!(proposals.is_empty()); - Ok(()) - } - - #[test] - fn cache_warm_contained_mode_still_requires_reviewed_source_policy() -> Result<()> { - let (root, paths) = temp_app_paths("cache-warm-contained"); - paths.ensure()?; - let mut state = test_runtime_state(vec![test_watcher_snapshot( - "cache-warm", - WatcherMode::Contained, - None, - )]); - let event = AutomationTriggerEvent { - at_unix_ms: 1, - kind: "cache.warm".to_owned(), - source: "test".to_owned(), - watcher_hint: Some("cache-warm".to_owned()), - service_id: None, - reason: Some("idle window".to_owned()), - payload: json!({ - "artifact_ref": "Qwen/Test-1B#hf-main", - }), - }; - - handle_cache_warm_event_with_resolver( - &paths, - WatcherMode::Contained, - &mut state, - &event, - |_| Ok(true), - )?; - let event_text = fs::read_to_string(paths.automation_events_path())?; - let event = serde_json::from_str::(event_text.trim())?; - let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; - fs::remove_dir_all(root).ok(); - - assert_eq!(event.action, "queue_prefetch_proposal"); - assert!(event.message.contains("explicit source-policy approval")); - assert_eq!(proposals.len(), 1); - assert_eq!(proposals[0].tool.as_deref(), Some("prefetch_artifact")); - Ok(()) - } - - #[test] - fn driver_upgrade_propose_mode_queues_driver_plan_proposal() -> Result<()> { - let (root, paths) = temp_app_paths("driver-upgrade-propose"); - paths.ensure()?; - let mut state = test_runtime_state(vec![test_watcher_snapshot( - "driver-upgrade", - WatcherMode::Propose, - None, - )]); - let event = crate::webhook::local_webhook_event_from_request( - crate::webhook::LocalWebhookEventRequest { - watcher_hint: "driver-upgrade".to_owned(), - kind: "update.available".to_owned(), - service_id: None, - reason: Some("driver version is newer".to_owned()), - payload: json!({ - "component": "driver", - "tool": "restart_server", - }), - }, - )?; - - handle_driver_upgrade_event(&paths, WatcherMode::Propose, &mut state, &event)?; - let event_text = fs::read_to_string(paths.automation_events_path())?; - let event = serde_json::from_str::(event_text.trim())?; - let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; - fs::remove_dir_all(root).ok(); - - assert_eq!(event.watcher_id, "driver-upgrade"); - assert_eq!(event.action, "prepare_driver_plan"); - assert_eq!(proposals.len(), 1); - assert_eq!(proposals[0].watcher_id, "driver-upgrade"); - assert_eq!(proposals[0].tool.as_deref(), Some("driver_plan")); - assert!( - proposals[0].arguments.get("tool").is_none(), - "webhook payload must not grant arbitrary tool choice" - ); - Ok(()) - } - - #[test] - fn driver_upgrade_contained_mode_runs_restricted_driver_plan() -> Result<()> { - let (root, paths) = temp_app_paths("driver-upgrade-contained"); - paths.ensure()?; - let mut state = test_runtime_state(vec![test_watcher_snapshot( - "driver-upgrade", - WatcherMode::Contained, - None, - )]); - let event = AutomationTriggerEvent { - at_unix_ms: 1, - kind: "update.available".to_owned(), - source: "test".to_owned(), - watcher_hint: Some("driver-upgrade".to_owned()), - service_id: None, - reason: Some("driver version is newer".to_owned()), - payload: json!({ - "component": "driver", - }), - }; - - handle_driver_upgrade_event_with_runner( - &paths, - WatcherMode::Contained, - &mut state, - &event, - |_paths| { - Ok(crate::sandbox::sandbox_driver_plan_value( - rocm_core::CommandCapture { - argv: vec![ - "rocm".to_owned(), - "install".to_owned(), - "driver".to_owned(), - "--dkms".to_owned(), - "--dry-run".to_owned(), - ], - exit_status: 0, - stdout: "driver install plan\n supported: true\n".to_owned(), - stderr: String::new(), - }, - )) - }, - )?; - let event_text = fs::read_to_string(paths.automation_events_path())?; - let event = serde_json::from_str::(event_text.trim())?; - let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; - fs::remove_dir_all(root).ok(); - - assert_eq!(event.action, "run_driver_plan"); - assert!( - event - .message - .contains("contained restricted driver_plan status=planned") - ); - assert!(event.message.contains("no driver commands were executed")); - assert!(proposals.is_empty()); - Ok(()) - } - - #[test] - fn driver_upgrade_contained_mode_requires_restricted_driver_plan_tool() -> Result<()> { - let (root, paths) = temp_app_paths("driver-upgrade-contained-tool"); - paths.ensure()?; - let mut state = test_runtime_state(vec![test_watcher_snapshot( - "driver-upgrade", - WatcherMode::Contained, - None, - )]); - let event = AutomationTriggerEvent { - at_unix_ms: 1, - kind: "update.available".to_owned(), - source: "test".to_owned(), - watcher_hint: Some("driver-upgrade".to_owned()), - service_id: None, - reason: Some("driver version is newer".to_owned()), - payload: json!({ - "component": "driver", - }), - }; - - handle_driver_upgrade_event_with_runner( - &paths, - WatcherMode::Contained, - &mut state, - &event, - |_paths| { - Ok(json!({ - "tool": "check_updates", - "status": "checked", - "mutating": false, - })) - }, - )?; - let event_text = fs::read_to_string(paths.automation_events_path())?; - let event = serde_json::from_str::(event_text.trim())?; - let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; - fs::remove_dir_all(root).ok(); - - assert_eq!(event.action, "driver_plan_failed"); - assert!(event.message.contains("expected `driver_plan`")); - assert!(event.message.contains("no driver commands were executed")); - assert!(proposals.is_empty()); - Ok(()) - } - - #[test] - fn driver_upgrade_observe_mode_records_without_proposal() -> Result<()> { - let (root, paths) = temp_app_paths("driver-upgrade-observe"); - paths.ensure()?; - let mut state = test_runtime_state(vec![test_watcher_snapshot( - "driver-upgrade", - WatcherMode::Observe, - None, - )]); - let event = AutomationTriggerEvent { - at_unix_ms: 1, - kind: "update.available".to_owned(), - source: "test".to_owned(), - watcher_hint: Some("driver-upgrade".to_owned()), - service_id: None, - reason: Some("driver version is newer".to_owned()), - payload: json!({ - "component": "driver", - }), - }; - - handle_driver_upgrade_event(&paths, WatcherMode::Observe, &mut state, &event)?; - let event_text = fs::read_to_string(paths.automation_events_path())?; - let event = serde_json::from_str::(event_text.trim())?; - let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; - fs::remove_dir_all(root).ok(); - - assert_eq!(event.action, "observe_driver_update"); - assert!(event.message.contains("does not queue or run")); - assert!(proposals.is_empty()); - Ok(()) - } - - #[test] - fn driver_upgrade_ignores_non_driver_component_without_proposal() -> Result<()> { - let (root, paths) = temp_app_paths("driver-upgrade-wrong-component"); - paths.ensure()?; - let mut state = test_runtime_state(vec![test_watcher_snapshot( - "driver-upgrade", - WatcherMode::Propose, - None, - )]); - let event = AutomationTriggerEvent { - at_unix_ms: 1, - kind: "update.available".to_owned(), - source: "test".to_owned(), - watcher_hint: Some("driver-upgrade".to_owned()), - service_id: None, - reason: Some("runtime version is newer".to_owned()), - payload: json!({ - "component": "runtime", - }), - }; - - handle_driver_upgrade_event(&paths, WatcherMode::Propose, &mut state, &event)?; - let event_text = fs::read_to_string(paths.automation_events_path())?; - let event = serde_json::from_str::(event_text.trim())?; - let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; - fs::remove_dir_all(root).ok(); - - assert_eq!(event.action, "driver_upgrade_ignored_component"); - assert!(event.message.contains("payload.component=driver")); - assert!(proposals.is_empty()); + Some("hotspot_temperature_threshold") + ); + assert!( + event + .payload + .get("summary") + .and_then(Value::as_str) + .is_some_and(|summary| summary.contains("GPU 0 hotspot temperature is 96 C")) + ); Ok(()) } #[test] - fn event_dispatcher_preserves_server_recover_proposal_behavior() -> Result<()> { - let (root, paths) = temp_app_paths("event-bus-dispatch"); - paths.ensure()?; - let mut failed = ManagedServiceRecord::new( - &paths, - "svc-failed", - "vllm", - "qwen", - "Qwen/Qwen3.5", - "127.0.0.1", - 11435, - "managed", - 123, - None, - None, - None, - ); - failed.status = "failed".to_owned(); - failed.write()?; - let mut state = test_runtime_state(vec![test_watcher_snapshot( - "server-recover", + fn event_collector_skips_gpu_pressure_below_thresholds() -> Result<()> { + let (root, paths) = temp_app_paths("event-bus-gpu-pressure-cool"); + let state = test_runtime_state(vec![test_watcher_snapshot( + "gpu-thermal-protect", WatcherMode::Propose, None, )]); - let mut config = RocmCliConfig::default(); - let watcher = config.watcher_config_mut("server-recover"); - watcher.enabled = true; - watcher.mode = Some(WatcherMode::Propose); - let events = vec![AutomationTriggerEvent { - at_unix_ms: 1, - kind: "service.manifest_recoverable".to_owned(), - source: "managed_service".to_owned(), - watcher_hint: Some("server-recover".to_owned()), - service_id: Some("svc-failed".to_owned()), - reason: Some("manifest_status_failed".to_owned()), - payload: json!({}), - }]; - evaluate_watchers_for_events(&paths, &config, &mut state, &events)?; - let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; + let events = collect_automation_events_with_gpu_snapshot(&paths, &state, || { + CodexBridgeGpuSnapshot { + amd_smi_available: true, + static_snapshot: None, + monitor_snapshot: Some(json!([ + { + "gpu": 0, + "hotspot_temperature": 80.0, + "memory_temperature": 82.0, + "vram_percent": 50.0 + } + ])), + note: None, + } + })?; fs::remove_dir_all(root).ok(); - assert_eq!(proposals.len(), 1); - assert_eq!(proposals[0].watcher_id, "server-recover"); - assert_eq!(proposals[0].service_id.as_deref(), Some("svc-failed")); - assert_eq!(proposals[0].tool.as_deref(), Some("restart_server")); - assert!(proposals[0].message.contains("manifest reports failed")); - assert!(!proposals[0].message.contains("manifest_status_failed")); + assert!( + !events + .iter() + .any(|event| event.watcher_hint.as_deref() == Some("gpu-thermal-protect")) + ); Ok(()) } #[test] - fn server_recover_local_webhook_does_not_restart_healthy_service() -> Result<()> { - let (root, paths) = temp_app_paths("server-recover-healthy-webhook"); + fn gpu_metrics_event_records_read_only_status_without_proposal() -> Result<()> { + let (root, paths) = temp_app_paths("gpu-metrics-record"); paths.ensure()?; - let mut healthy = ManagedServiceRecord::new( - &paths, - "svc-healthy", - "vllm", - "qwen", - "Qwen/Qwen3.5", - "127.0.0.1", - 11435, - "managed", - 123, - None, - None, - None, - ); - healthy.status = "ready".to_owned(); - healthy.write()?; let mut state = test_runtime_state(vec![test_watcher_snapshot( - "server-recover", - WatcherMode::Propose, + "gpu-metrics", + WatcherMode::Contained, None, )]); let mut config = RocmCliConfig::default(); - let watcher = config.watcher_config_mut("server-recover"); + let watcher = config.watcher_config_mut("gpu-metrics"); watcher.enabled = true; - watcher.mode = Some(WatcherMode::Propose); - let event = crate::webhook::local_webhook_event_from_request( - crate::webhook::LocalWebhookEventRequest { - watcher_hint: "server-recover".to_owned(), - kind: "service.manifest_recoverable".to_owned(), - service_id: Some("svc-healthy".to_owned()), - reason: Some("manual recovery smoke".to_owned()), - payload: json!({}), - }, - )?; - - evaluate_watchers_for_events(&paths, &config, &mut state, &[event])?; - let event_text = fs::read_to_string(paths.automation_events_path())?; - let event = serde_json::from_str::(event_text.trim())?; - let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; - let reloaded = load_service_record(&paths, "svc-healthy")?; - fs::remove_dir_all(root).ok(); - - assert_eq!(event.action, "ignore_nonrecoverable_service"); - assert!(event.message.contains("does not currently need recovery")); - assert!(proposals.is_empty()); - assert_eq!(reloaded.status, "ready"); - Ok(()) - } - - #[test] - fn recovery_reason_display_avoids_raw_status_tokens() { - assert_eq!( - display_recovery_reason("manifest_status_starting_stale"), - "service has been starting for too long" - ); - assert_eq!( - display_recovery_reason("healthcheck_status_unreachable"), - "engine healthcheck reports unreachable" - ); - assert_eq!( - display_recovery_reason("endpoint_status_unreachable"), - "endpoint port is unreachable" - ); - } - #[test] - fn manifest_recovery_policy_covers_terminal_and_stale_transient_states() { - let (root, paths) = temp_app_paths("manifest-recovery-policy"); - let mut record = ManagedServiceRecord::new( - &paths, - "svc-stale", - "vllm", - "qwen", - "Qwen/Qwen3.5", - "127.0.0.1", - 11435, - "managed", - 123, - None, - None, - None, - ); - record.created_at_unix_ms = 1_000; - - record.status = "exited".to_owned(); - assert_eq!( - manifest_service_recovery_reason(&record, 1_001).as_deref(), - Some("manifest_status_exited") - ); - - record.status = "unreachable".to_owned(); - assert_eq!( - manifest_service_recovery_reason(&record, 1_001).as_deref(), - Some("manifest_status_unreachable") - ); - - record.status = "starting".to_owned(); - assert_eq!(manifest_service_recovery_reason(&record, 2_000), None); - assert_eq!( - manifest_service_recovery_reason(&record, 1_000 + SERVER_TRANSIENT_STALE_MS).as_deref(), - Some("manifest_status_starting_stale") - ); - - record.status = "recovering".to_owned(); - record.last_restart_unix_ms = Some(5_000); - assert_eq!(manifest_service_recovery_reason(&record, 6_000), None); - assert_eq!( - manifest_service_recovery_reason(&record, 5_000 + SERVER_TRANSIENT_STALE_MS).as_deref(), - Some("manifest_status_recovering_stale") - ); - fs::remove_dir_all(root).ok(); - } - - #[test] - fn find_recoverable_service_prefers_failed_managed_manifest() -> Result<()> { - let (root, paths) = temp_app_paths("recoverable-service"); - paths.ensure()?; - let mut failed = ManagedServiceRecord::new( - &paths, - "svc-failed", - "vllm", - "qwen", - "Qwen/Qwen3.5", - "127.0.0.1", - 11435, - "managed", - 123, - None, - None, - None, - ); - failed.status = "failed".to_owned(); - failed.write()?; - - let found = find_recoverable_service(&paths)?.expect("failed service should be found"); - fs::remove_dir_all(root).ok(); - assert_eq!(found.0.service_id, "svc-failed"); - assert_eq!(found.1, "manifest_status_failed"); - Ok(()) - } - - #[test] - fn find_recoverable_service_detects_stale_starting_manifest() -> Result<()> { - let (root, paths) = temp_app_paths("recoverable-stale-starting"); - paths.ensure()?; - let mut stale = ManagedServiceRecord::new( - &paths, - "svc-starting", - "vllm", - "qwen", - "Qwen/Qwen3.5", - "127.0.0.1", - 11435, - "managed", - 123, - None, - None, - None, - ); - stale.status = "starting".to_owned(); - stale.created_at_unix_ms = 0; - stale.write()?; - - let found = - find_recoverable_service(&paths)?.expect("stale starting service should recover"); - fs::remove_dir_all(root).ok(); - assert_eq!(found.0.service_id, "svc-starting"); - assert_eq!(found.1, "manifest_status_starting_stale"); - Ok(()) - } - - #[test] - fn server_recover_propose_mode_queues_restart_proposal() -> Result<()> { - let (root, paths) = temp_app_paths("server-recover-proposal"); - paths.ensure()?; - let mut record = ManagedServiceRecord::new( - &paths, - "svc-1", - "vllm", - "qwen", - "Qwen/Qwen3.5", - "127.0.0.1", - 11435, - "managed", - 123, - None, - None, - None, - ); - record.status = "failed".to_owned(); - record.write()?; - let mut state = AutomationRuntimeState { - running: true, - automations_enabled: true, - daemon_pid: 1, - started_at_unix_ms: 1, - last_tick_unix_ms: 1, - local_webhook_endpoint: None, - active_watchers: vec![WatcherRuntimeSnapshot { - id: "server-recover".to_owned(), - enabled: true, - mode: WatcherMode::Propose, - summary: "recover".to_owned(), - last_event: None, - last_event_unix_ms: None, - }], - }; - - evaluate_server_recover(&paths, WatcherMode::Propose, &mut state)?; - let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; - fs::remove_dir_all(root).ok(); - - assert_eq!(proposals.len(), 1); - assert_eq!(proposals[0].watcher_id, "server-recover"); - assert_eq!(proposals[0].action, "queue_restart_proposal"); - assert_eq!(proposals[0].service_id.as_deref(), Some("svc-1")); - assert_eq!(proposals[0].status, "pending"); - Ok(()) - } - - #[test] - fn therock_update_contained_mode_runs_read_only_check_without_queueing() -> Result<()> { - let (root, paths) = temp_app_paths("therock-update-contained"); - paths.ensure()?; - let mut state = AutomationRuntimeState { - running: true, - automations_enabled: true, - daemon_pid: 1, - started_at_unix_ms: 1, - last_tick_unix_ms: 1, - local_webhook_endpoint: None, - active_watchers: vec![WatcherRuntimeSnapshot { - id: "therock-update".to_owned(), - enabled: true, - mode: WatcherMode::Contained, - summary: "check updates".to_owned(), - last_event: None, - last_event_unix_ms: None, - }], - }; - let event = AutomationTriggerEvent { - at_unix_ms: 42, - kind: "schedule.tick".to_owned(), - source: "scheduler".to_owned(), - watcher_hint: Some("therock-update".to_owned()), - service_id: None, - reason: Some("therock_update_interval_due".to_owned()), - payload: json!({ "interval_ms": THEROCK_UPDATE_INTERVAL_MS }), - }; - - handle_therock_update_event_with_runner( - &paths, - WatcherMode::Contained, - &mut state, - &event, - |_paths| { - Ok(crate::sandbox::sandbox_check_updates_value( - rocm_core::CommandCapture { - argv: vec!["rocm".to_owned(), "update".to_owned()], - exit_status: 0, - stdout: "update\n managed runtimes: none\n".to_owned(), - stderr: String::new(), - }, - )) - }, - )?; + watcher.mode = Some(WatcherMode::Contained); + let events = vec![AutomationTriggerEvent { + at_unix_ms: 1, + kind: "gpu.metrics_unavailable".to_owned(), + source: "gpu_telemetry".to_owned(), + watcher_hint: Some("gpu-metrics".to_owned()), + service_id: None, + reason: Some("amd-smi missing".to_owned()), + payload: json!({ + "summary": "amd_smi_available=false static_snapshot=missing monitor_snapshot=missing", + }), + }]; + evaluate_watchers_for_events(&paths, &config, &mut state, &events)?; let event_text = fs::read_to_string(paths.automation_events_path())?; let event = serde_json::from_str::(event_text.trim())?; let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; fs::remove_dir_all(root).ok(); - assert_eq!(event.watcher_id, "therock-update"); - assert_eq!(event.action, "run_update_check"); - assert!(event.message.contains("contained read-only execution")); - assert!( - event - .message - .contains("restricted check_updates status=checked") - ); - assert!(event.message.contains("no updates were applied")); - assert!(!event.message.contains("fallback")); + assert_eq!(event.watcher_id, "gpu-metrics"); + assert_eq!(event.action, "record_gpu_metrics"); + assert!(event.message.contains("telemetry is recorded only")); + assert!(event.message.contains("amd-smi missing")); + assert!(event.message.contains("no mutating action was taken")); assert!(proposals.is_empty()); Ok(()) } #[test] - fn therock_update_contained_mode_records_update_available_without_applying() -> Result<()> { - let (root, paths) = temp_app_paths("therock-update-contained-available"); + fn gpu_thermal_protect_propose_queues_reviewed_stop_for_one_running_service() -> Result<()> { + let (root, paths) = temp_app_paths("gpu-thermal-protect-propose"); paths.ensure()?; - let mut state = AutomationRuntimeState { - running: true, - automations_enabled: true, - daemon_pid: 1, - started_at_unix_ms: 1, - last_tick_unix_ms: 1, - local_webhook_endpoint: None, - active_watchers: vec![WatcherRuntimeSnapshot { - id: "therock-update".to_owned(), - enabled: true, - mode: WatcherMode::Contained, - summary: "check updates".to_owned(), - last_event: None, - last_event_unix_ms: None, - }], - }; + let mut record = ManagedServiceRecord::new( + &paths, + "svc-hot", + "vllm", + "tiny", + "Tiny/Test", + "127.0.0.1", + 11435, + "managed", + 123, + None, + None, + None, + ); + record.status = "ready".to_owned(); + record.write()?; + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "gpu-thermal-protect", + WatcherMode::Propose, + None, + )]); + let mut config = RocmCliConfig::default(); + let watcher = config.watcher_config_mut("gpu-thermal-protect"); + watcher.enabled = true; + watcher.mode = Some(WatcherMode::Propose); let event = AutomationTriggerEvent { - at_unix_ms: 42, - kind: "schedule.tick".to_owned(), - source: "scheduler".to_owned(), - watcher_hint: Some("therock-update".to_owned()), + at_unix_ms: 1, + kind: "gpu.thermal_pressure".to_owned(), + source: "gpu_telemetry".to_owned(), + watcher_hint: Some("gpu-thermal-protect".to_owned()), service_id: None, - reason: Some("therock_update_interval_due".to_owned()), - payload: json!({ "interval_ms": THEROCK_UPDATE_INTERVAL_MS }), + reason: Some("hotspot_temperature_threshold".to_owned()), + payload: json!({ + "gpu": 0, + "summary": "GPU 0 hotspot temperature is 96 C (limit 95 C)", + "hotspot_temperature_c": 96.0, + }), }; - handle_therock_update_event_with_runner( - &paths, - WatcherMode::Contained, - &mut state, - &event, - |_paths| { - Ok(crate::sandbox::sandbox_check_updates_value(rocm_core::CommandCapture { - argv: vec!["rocm".to_owned(), "update".to_owned()], - exit_status: 0, - stdout: "update\n runtime release-pip-gfx120x-all status=update_available installed=7.13.0 latest=7.14.0\n".to_owned(), - stderr: String::new(), - })) - }, - )?; - + evaluate_watchers_for_events(&paths, &config, &mut state, &[event])?; let event_text = fs::read_to_string(paths.automation_events_path())?; - let events = event_text - .lines() - .map(serde_json::from_str::) - .collect::, _>>()?; - let audit_text = fs::read_to_string(paths.audit_events_path())?; + let event = serde_json::from_str::(event_text.trim())?; let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; + let saved = load_service_record(&paths, "svc-hot")?; fs::remove_dir_all(root).ok(); - let update_check = events - .iter() - .find(|event| event.action == "run_update_check") - .expect("update check event should be recorded"); - assert_eq!(update_check.watcher_id, "therock-update"); - assert!( - update_check - .message - .contains("restricted check_updates status=update_available") - ); - assert!( - update_check - .message - .contains("a ROCm runtime update is available") - ); - assert!(update_check.message.contains("no updates were applied")); - assert!(!update_check.message.contains("fallback")); - let notification = events - .iter() - .find(|event| event.action == "notify_if_newer") - .expect("notify-if-newer event should be recorded"); - assert_eq!(notification.watcher_id, "therock-update"); - assert!( - notification - .message - .contains("ROCm runtime update is available") + assert_eq!(event.watcher_id, "gpu-thermal-protect"); + assert_eq!(event.action, "queue_stop_server_proposal"); + assert_eq!(event.service_id.as_deref(), Some("svc-hot")); + assert!(event.message.contains("asking before stopping anything")); + assert_eq!(saved.status, "ready"); + assert_eq!(proposals.len(), 1); + assert_eq!(proposals[0].tool.as_deref(), Some("stop_server")); + assert_eq!(proposals[0].service_id.as_deref(), Some("svc-hot")); + assert_eq!( + proposals[0] + .arguments + .get("pressure_reason") + .and_then(Value::as_str), + Some("hotspot_temperature_threshold") ); - assert!(notification.message.contains("No updates were applied")); - assert!(audit_text.contains("\"category\":\"notification\"")); - assert!(audit_text.contains("\"action\":\"notify_if_newer\"")); - assert!(audit_text.contains("ROCm runtime update is available")); - assert!(proposals.is_empty()); Ok(()) } #[test] - fn therock_update_contained_mode_uses_restricted_check_updates_tool() -> Result<()> { - let (root, paths) = temp_app_paths("therock-update-contained-tool"); + fn local_webhook_gpu_metrics_event_uses_existing_read_only_policy() -> Result<()> { + let (root, paths) = temp_app_paths("local-webhook-gpu-metrics"); paths.ensure()?; - let mut state = AutomationRuntimeState { - running: true, - automations_enabled: true, - daemon_pid: 1, - started_at_unix_ms: 1, - last_tick_unix_ms: 1, - local_webhook_endpoint: None, - active_watchers: vec![WatcherRuntimeSnapshot { - id: "therock-update".to_owned(), - enabled: true, - mode: WatcherMode::Contained, - summary: "check updates".to_owned(), - last_event: None, - last_event_unix_ms: None, - }], - }; - let event = AutomationTriggerEvent { - at_unix_ms: 42, - kind: "schedule.tick".to_owned(), - source: "scheduler".to_owned(), - watcher_hint: Some("therock-update".to_owned()), - service_id: None, - reason: Some("therock_update_interval_due".to_owned()), - payload: json!({ "interval_ms": THEROCK_UPDATE_INTERVAL_MS }), - }; - - handle_therock_update_event_with_runner( - &paths, + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "gpu-metrics", WatcherMode::Contained, - &mut state, - &event, - |_paths| { - Ok(json!({ - "tool": "examine_snapshot", - "status": "captured", - "mutating": false, - })) + None, + )]); + let mut config = RocmCliConfig::default(); + let watcher = config.watcher_config_mut("gpu-metrics"); + watcher.enabled = true; + watcher.mode = Some(WatcherMode::Contained); + let event = crate::webhook::local_webhook_event_from_request( + crate::webhook::LocalWebhookEventRequest { + watcher_hint: "gpu-metrics".to_owned(), + kind: "gpu.metrics".to_owned(), + service_id: None, + reason: Some("manual smoke".to_owned()), + payload: json!({ + "summary": "manual webhook probe", + "action": "restart_server", + "mode": "contained", + }), }, )?; + evaluate_watchers_for_events(&paths, &config, &mut state, &[event])?; let event_text = fs::read_to_string(paths.automation_events_path())?; let event = serde_json::from_str::(event_text.trim())?; + let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; fs::remove_dir_all(root).ok(); - assert_eq!(event.watcher_id, "therock-update"); - assert_eq!(event.action, "update_check_failed"); - assert!(event.message.contains("expected `check_updates`")); - assert!(event.message.contains("no updates were applied")); + assert_eq!(event.watcher_id, "gpu-metrics"); + assert_eq!(event.action, "record_gpu_metrics"); + assert!(event.message.contains("from local webhook")); + assert!(event.message.contains("manual smoke")); + assert!(event.message.contains("no mutating action was taken")); + assert!(proposals.is_empty()); Ok(()) } #[test] - fn therock_update_notify_if_newer_uses_restricted_notification_contract() -> Result<()> { - let (root, paths) = temp_app_paths("therock-update-notify-contract"); + fn event_dispatcher_preserves_server_recover_proposal_behavior() -> Result<()> { + let (root, paths) = temp_app_paths("event-bus-dispatch"); paths.ensure()?; - let mut state = AutomationRuntimeState { - running: true, - automations_enabled: true, - daemon_pid: 1, - started_at_unix_ms: 1, - last_tick_unix_ms: 1, - local_webhook_endpoint: None, - active_watchers: vec![WatcherRuntimeSnapshot { - id: "therock-update".to_owned(), - enabled: true, - mode: WatcherMode::Contained, - summary: "check updates".to_owned(), - last_event: None, - last_event_unix_ms: None, - }], - }; - - record_update_available_notification(&paths, &mut state, "update_available")?; - - let audit_text = fs::read_to_string(paths.audit_events_path())?; - let audit = audit_text - .lines() - .map(serde_json::from_str::) - .collect::, _>>()?; - fs::remove_dir_all(root).ok(); - - let notification = audit - .iter() - .find(|event| event.category == "notification" && event.action == "notify_if_newer") - .expect("notify_if_newer audit should be recorded"); - assert_eq!(notification.category, "notification"); - assert_eq!(notification.actor, "watcher:therock-update"); - assert_eq!(notification.watcher_id.as_deref(), Some("therock-update")); - assert!( - notification - .message - .contains("ROCm runtime update is available") - ); - assert!(notification.message.contains("No updates were applied")); - Ok(()) - } - - #[test] - fn watcher_policy_maps_modes_to_decisions() { - assert_eq!( - watcher_policy_action("server-recover", WatcherMode::Observe), - WatcherPolicyAction::Observe - ); - assert_eq!( - watcher_policy_action("server-recover", WatcherMode::Propose), - WatcherPolicyAction::QueueProposal - ); - assert_eq!( - watcher_policy_action("server-recover", WatcherMode::Contained), - WatcherPolicyAction::RunContained - ); - assert_eq!( - watcher_policy_action("therock-update", WatcherMode::Contained), - WatcherPolicyAction::RunContained + let mut failed = ManagedServiceRecord::new( + &paths, + "svc-failed", + "vllm", + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + 11435, + "managed", + 123, + None, + None, + None, ); - } - - /// An unconfirmed stop leaves the status at `ready` on purpose, which is - /// exactly what the daemon's recovery scan looks for. The stop request it - /// records is the only thing telling it apart from a crash, so recovery - /// must honour it — otherwise the `server-recover` watcher, which acts by - /// default, restarts the service the operator just asked to stop. - #[test] - fn recovery_does_not_restart_a_service_whose_stop_is_unconfirmed() -> Result<()> { - let (root, paths) = temp_app_paths("recover-skips-unconfirmed-stop"); - paths.ensure()?; - let service_id = "svc-recover-skips-unconfirmed-stop"; - seed_keyed_service(&paths, service_id, 11450)?; + failed.status = "failed".to_owned(); + failed.write()?; + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "server-recover", + WatcherMode::Propose, + None, + )]); + let mut config = RocmCliConfig::default(); + let watcher = config.watcher_config_mut("server-recover"); + watcher.enabled = true; + watcher.mode = Some(WatcherMode::Propose); + let events = vec![AutomationTriggerEvent { + at_unix_ms: 1, + kind: "service.manifest_recoverable".to_owned(), + source: "managed_service".to_owned(), + watcher_hint: Some("server-recover".to_owned()), + service_id: Some("svc-failed".to_owned()), + reason: Some("manifest_status_failed".to_owned()), + payload: json!({}), + }]; - let stopped = - stop_with_outcome(&paths, service_id, rocm_core::TerminationOutcome::TimedOut); - let found = find_recoverable_service(&paths); - let reloaded = load_service_record(&paths, service_id); + evaluate_watchers_for_events(&paths, &config, &mut state, &events)?; + let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; fs::remove_dir_all(root).ok(); - assert_eq!( - stopped?.get("stopped").and_then(Value::as_bool), - Some(false), - "precondition: the stop must be unconfirmed" - ); - let reloaded = reloaded?; - assert_eq!( - reloaded.status, "ready", - "precondition: status left as it was" - ); - assert!( - found?.is_none(), - "a service with a standing stop request must not be recovered" - ); + assert_eq!(proposals.len(), 1); + assert_eq!(proposals[0].watcher_id, "server-recover"); + assert_eq!(proposals[0].service_id.as_deref(), Some("svc-failed")); + assert_eq!(proposals[0].tool.as_deref(), Some("restart_server")); + assert!(proposals[0].message.contains("manifest reports failed")); + assert!(!proposals[0].message.contains("manifest_status_failed")); Ok(()) } - /// The same gate on a record recovery would otherwise pick up from its - /// status alone, without any endpoint probe: with a stop request it is - /// skipped, and the identical record without one is still recovered. #[test] - fn recovery_skips_a_failed_service_only_while_its_stop_request_stands() -> Result<()> { - let (root, paths) = temp_app_paths("recover-honours-stop-request"); + fn server_recover_local_webhook_does_not_restart_healthy_service() -> Result<()> { + let (root, paths) = temp_app_paths("server-recover-healthy-webhook"); paths.ensure()?; - let mut record = ManagedServiceRecord::new( + let mut healthy = ManagedServiceRecord::new( &paths, - "svc-recover-honours-stop-request", + "svc-healthy", "vllm", "qwen", "Qwen/Qwen3.5", "127.0.0.1", - 11451, + 11435, "managed", - UNCONFIRMED_STOP_PID, + 123, None, None, None, ); - record.status = "failed".to_owned(); - record.stop_requested_unix_ms = Some(1); - record.write()?; - let event = AutomationTriggerEvent { - at_unix_ms: unix_time_millis(), - kind: "service.manifest_recoverable".to_owned(), - source: "managed_service".to_owned(), - watcher_hint: Some("server-recover".to_owned()), - service_id: Some(record.service_id.clone()), - reason: Some("manifest_status_failed".to_owned()), - payload: json!({}), - }; - let with_request = find_recoverable_service(&paths); - let event_with_request = service_record_matches_recovery_event(&paths, &record, &event); + healthy.status = "ready".to_owned(); + healthy.write()?; + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "server-recover", + WatcherMode::Propose, + None, + )]); + let mut config = RocmCliConfig::default(); + let watcher = config.watcher_config_mut("server-recover"); + watcher.enabled = true; + watcher.mode = Some(WatcherMode::Propose); + let event = crate::webhook::local_webhook_event_from_request( + crate::webhook::LocalWebhookEventRequest { + watcher_hint: "server-recover".to_owned(), + kind: "service.manifest_recoverable".to_owned(), + service_id: Some("svc-healthy".to_owned()), + reason: Some("manual recovery smoke".to_owned()), + payload: json!({}), + }, + )?; - record.stop_requested_unix_ms = None; - record.write()?; - let without_request = find_recoverable_service(&paths); - let event_without_request = service_record_matches_recovery_event(&paths, &record, &event); + evaluate_watchers_for_events(&paths, &config, &mut state, &[event])?; + let event_text = fs::read_to_string(paths.automation_events_path())?; + let event = serde_json::from_str::(event_text.trim())?; + let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; + let reloaded = load_service_record(&paths, "svc-healthy")?; fs::remove_dir_all(root).ok(); - assert!( - with_request?.is_none(), - "a standing stop request must block recovery" - ); - assert!( - !event_with_request, - "a queued recovery event must not act on a service since asked to stop" - ); - assert_eq!( - without_request?.map(|(found, reason)| (found.service_id, reason)), - Some(( - "svc-recover-honours-stop-request".to_owned(), - "manifest_status_failed".to_owned() - )), - "without a stop request the same record must still be recovered" - ); - assert!(event_without_request); + assert_eq!(event.action, "ignore_nonrecoverable_service"); + assert!(event.message.contains("does not currently need recovery")); + assert!(proposals.is_empty()); + assert_eq!(reloaded.status, "ready"); Ok(()) } } diff --git a/apps/rocmd/src/watchers/cache_warm.rs b/apps/rocmd/src/watchers/cache_warm.rs new file mode 100644 index 0000000000..a2c38d4863 --- /dev/null +++ b/apps/rocmd/src/watchers/cache_warm.rs @@ -0,0 +1,277 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The `cache-warm` watcher: checking that a recipe artifact is known and +//! queueing a reviewed prefetch proposal for it. + +use super::policy::queue_proposal_with_arguments; +use crate::common::payload_string; +use anyhow::Result; +use rocm_core::{ + AppPaths, AutomationRuntimeState, AutomationTriggerEvent, WatcherMode, + resolve_model_recipe_artifact, +}; +use serde_json::json; + +pub(super) fn handle_cache_warm_event( + paths: &AppPaths, + mode: WatcherMode, + state: &mut AutomationRuntimeState, + event: &AutomationTriggerEvent, +) -> Result<()> { + handle_cache_warm_event_with_resolver(paths, mode, state, event, |artifact_ref| { + resolve_model_recipe_artifact(artifact_ref).map(|resolved| resolved.is_some()) + }) +} + +fn handle_cache_warm_event_with_resolver( + paths: &AppPaths, + mode: WatcherMode, + state: &mut AutomationRuntimeState, + event: &AutomationTriggerEvent, + mut artifact_exists: F, +) -> Result<()> +where + F: FnMut(&str) -> Result, +{ + let Some(artifact_ref) = payload_string(&event.payload, "artifact_ref") else { + return crate::persistence::record_event( + paths, + state, + "cache-warm", + "warn", + "cache_warm_missing_artifact", + "cache warm event did not include artifact_ref; no prefetch proposal was queued", + None, + ); + }; + match artifact_exists(&artifact_ref) { + Ok(true) => {} + Ok(false) => { + return crate::persistence::record_event( + paths, + state, + "cache-warm", + "warn", + "cache_warm_unknown_artifact", + &format!( + "cache warm requested unknown registry artifact {artifact_ref}; no prefetch proposal was queued" + ), + None, + ); + } + Err(error) => { + return crate::persistence::record_event( + paths, + state, + "cache-warm", + "error", + "cache_warm_registry_error", + &format!( + "cache warm could not verify registry artifact {artifact_ref}: {error}; no prefetch proposal was queued" + ), + None, + ); + } + } + match mode { + WatcherMode::Observe => crate::persistence::record_event( + paths, + state, + "cache-warm", + "info", + "observe_cache_warm_request", + &format!( + "observed cache warm request for {artifact_ref}; observe mode does not queue or download artifacts" + ), + None, + ), + WatcherMode::Propose | WatcherMode::Contained => { + let action = "queue_prefetch_proposal"; + let message = if matches!(mode, WatcherMode::Contained) { + format!( + "cache warm requested for {artifact_ref}; contained mode still queues a review because artifact downloads require explicit source-policy approval" + ) + } else { + format!( + "cache warm requested for {artifact_ref}; queueing a reviewed prefetch proposal" + ) + }; + crate::persistence::record_event( + paths, + state, + "cache-warm", + "info", + action, + &message, + None, + )?; + queue_proposal_with_arguments( + paths, + "cache-warm", + action, + "Prefetch model artifact", + &message, + None, + json!({ + "artifact_ref": artifact_ref, + }), + ) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::{temp_app_paths, test_runtime_state, test_watcher_snapshot}; + use rocm_core::AutomationEventRecord; + use serde_json::Value; + use std::fs; + + #[test] + fn cache_warm_propose_mode_queues_prefetch_proposal() -> Result<()> { + let (root, paths) = temp_app_paths("cache-warm-propose"); + paths.ensure()?; + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "cache-warm", + WatcherMode::Propose, + None, + )]); + let event = crate::webhook::local_webhook_event_from_request( + crate::webhook::LocalWebhookEventRequest { + watcher_hint: "cache-warm".to_owned(), + kind: "cache.warm".to_owned(), + service_id: None, + reason: Some("idle window".to_owned()), + payload: json!({ + "artifact_ref": "Qwen/Test-1B#hf-main", + "tool": "restart_server", + "allow_artifact_download": true, + "artifact_max_bytes": 1024, + }), + }, + )?; + + handle_cache_warm_event_with_resolver( + &paths, + WatcherMode::Propose, + &mut state, + &event, + |_| Ok(true), + )?; + let event_text = fs::read_to_string(paths.automation_events_path())?; + let event = serde_json::from_str::(event_text.trim())?; + let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; + fs::remove_dir_all(root).ok(); + + assert_eq!(event.watcher_id, "cache-warm"); + assert_eq!(event.action, "queue_prefetch_proposal"); + assert_eq!(proposals.len(), 1); + assert_eq!(proposals[0].watcher_id, "cache-warm"); + assert_eq!(proposals[0].tool.as_deref(), Some("prefetch_artifact")); + assert_eq!( + proposals[0] + .arguments + .get("artifact_ref") + .and_then(Value::as_str), + Some("Qwen/Test-1B#hf-main") + ); + assert!( + proposals[0].arguments.get("tool").is_none(), + "webhook payload must not grant arbitrary tool choice" + ); + assert!( + proposals[0] + .arguments + .get("allow_artifact_download") + .is_none(), + "webhook payload must not grant source-policy approval" + ); + assert!( + proposals[0].arguments.get("artifact_max_bytes").is_none(), + "webhook payload must not grant download byte-limit approval" + ); + Ok(()) + } + + #[test] + fn cache_warm_unknown_artifact_does_not_queue_proposal() -> Result<()> { + let (root, paths) = temp_app_paths("cache-warm-unknown"); + paths.ensure()?; + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "cache-warm", + WatcherMode::Propose, + None, + )]); + let event = AutomationTriggerEvent { + at_unix_ms: 1, + kind: "cache.warm".to_owned(), + source: "test".to_owned(), + watcher_hint: Some("cache-warm".to_owned()), + service_id: None, + reason: Some("idle window".to_owned()), + payload: json!({ + "artifact_ref": "missing#artifact", + }), + }; + + handle_cache_warm_event_with_resolver( + &paths, + WatcherMode::Propose, + &mut state, + &event, + |_| Ok(false), + )?; + let event_text = fs::read_to_string(paths.automation_events_path())?; + let event = serde_json::from_str::(event_text.trim())?; + let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; + fs::remove_dir_all(root).ok(); + + assert_eq!(event.action, "cache_warm_unknown_artifact"); + assert!(event.message.contains("unknown registry artifact")); + assert!(proposals.is_empty()); + Ok(()) + } + + #[test] + fn cache_warm_contained_mode_still_requires_reviewed_source_policy() -> Result<()> { + let (root, paths) = temp_app_paths("cache-warm-contained"); + paths.ensure()?; + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "cache-warm", + WatcherMode::Contained, + None, + )]); + let event = AutomationTriggerEvent { + at_unix_ms: 1, + kind: "cache.warm".to_owned(), + source: "test".to_owned(), + watcher_hint: Some("cache-warm".to_owned()), + service_id: None, + reason: Some("idle window".to_owned()), + payload: json!({ + "artifact_ref": "Qwen/Test-1B#hf-main", + }), + }; + + handle_cache_warm_event_with_resolver( + &paths, + WatcherMode::Contained, + &mut state, + &event, + |_| Ok(true), + )?; + let event_text = fs::read_to_string(paths.automation_events_path())?; + let event = serde_json::from_str::(event_text.trim())?; + let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; + fs::remove_dir_all(root).ok(); + + assert_eq!(event.action, "queue_prefetch_proposal"); + assert!(event.message.contains("explicit source-policy approval")); + assert_eq!(proposals.len(), 1); + assert_eq!(proposals[0].tool.as_deref(), Some("prefetch_artifact")); + Ok(()) + } +} diff --git a/apps/rocmd/src/watchers/driver_upgrade.rs b/apps/rocmd/src/watchers/driver_upgrade.rs new file mode 100644 index 0000000000..84967dd6d2 --- /dev/null +++ b/apps/rocmd/src/watchers/driver_upgrade.rs @@ -0,0 +1,376 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The `driver-upgrade` watcher: turning an available driver update into a +//! reviewed driver-plan proposal, or a read-only plan in contained mode. + +use super::policy::queue_proposal; +use crate::common::payload_string; +use anyhow::{Context, Result, bail}; +use rocm_core::{AppPaths, AutomationRuntimeState, AutomationTriggerEvent, WatcherMode}; +use serde_json::Value; + +pub(super) fn handle_driver_upgrade_event( + paths: &AppPaths, + mode: WatcherMode, + state: &mut AutomationRuntimeState, + event: &AutomationTriggerEvent, +) -> Result<()> { + handle_driver_upgrade_event_with_runner(paths, mode, state, event, |paths| { + crate::sandbox::run_sandbox_tool( + paths, + crate::sandbox_tool::SandboxToolArg::DriverPlan, + None, + None, + None, + crate::sandbox_tool::SandboxToolPolicy::default(), + ) + }) +} + +fn handle_driver_upgrade_event_with_runner( + paths: &AppPaths, + mode: WatcherMode, + state: &mut AutomationRuntimeState, + event: &AutomationTriggerEvent, + driver_plan_runner: F, +) -> Result<()> +where + F: FnOnce(&AppPaths) -> Result, +{ + if payload_string(&event.payload, "component").as_deref() != Some("driver") { + return crate::persistence::record_event( + paths, + state, + "driver-upgrade", + "warn", + "driver_upgrade_ignored_component", + "driver-upgrade event did not include payload.component=driver; no driver plan proposal was queued", + None, + ); + } + + match mode { + WatcherMode::Observe => crate::persistence::record_event( + paths, + state, + "driver-upgrade", + "info", + "observe_driver_update", + "observed local driver update signal; observe mode does not queue or run a driver plan", + None, + ), + WatcherMode::Propose => { + let action = "prepare_driver_plan"; + let message = + "local driver update signal received; queueing a reviewed read-only driver plan"; + crate::persistence::record_event( + paths, + state, + "driver-upgrade", + "info", + action, + message, + None, + )?; + queue_proposal( + paths, + "driver-upgrade", + action, + "Review driver install plan", + message, + None, + ) + } + WatcherMode::Contained => match driver_plan_runner(paths) { + Ok(output) => match restricted_driver_plan_result(&output) { + Ok(result) => crate::persistence::record_event( + paths, + state, + "driver-upgrade", + if result.exit_status == 0 { + "info" + } else { + "error" + }, + "run_driver_plan", + &format!( + "local driver update signal received; contained restricted driver_plan status={}; no driver commands were executed", + result.status + ), + None, + ), + Err(error) => crate::persistence::record_event( + paths, + state, + "driver-upgrade", + "error", + "driver_plan_failed", + &format!( + "local driver update signal received, but contained restricted driver_plan failed: {error}; no driver commands were executed" + ), + None, + ), + }, + Err(error) => crate::persistence::record_event( + paths, + state, + "driver-upgrade", + "error", + "driver_plan_failed", + &format!( + "local driver update signal received, but contained restricted driver_plan failed: {error}; no driver commands were executed" + ), + None, + ), + }, + } +} + +struct RestrictedDriverPlanResult<'a> { + status: &'a str, + exit_status: i64, +} + +fn restricted_driver_plan_result(value: &Value) -> Result> { + let tool = value + .get("tool") + .and_then(Value::as_str) + .context("restricted driver plan did not report a tool name")?; + if tool != crate::sandbox_tool::SandboxToolArg::DriverPlan.as_cli_value() { + bail!("restricted driver plan returned `{tool}`, expected `driver_plan`"); + } + let status = value + .get("status") + .and_then(Value::as_str) + .unwrap_or("planned"); + let exit_status = value + .get("exit_status") + .and_then(Value::as_i64) + .unwrap_or_else(|| i64::from(status == "error")); + Ok(RestrictedDriverPlanResult { + status, + exit_status, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::{temp_app_paths, test_runtime_state, test_watcher_snapshot}; + use rocm_core::AutomationEventRecord; + use serde_json::json; + use std::fs; + + #[test] + fn driver_upgrade_propose_mode_queues_driver_plan_proposal() -> Result<()> { + let (root, paths) = temp_app_paths("driver-upgrade-propose"); + paths.ensure()?; + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "driver-upgrade", + WatcherMode::Propose, + None, + )]); + let event = crate::webhook::local_webhook_event_from_request( + crate::webhook::LocalWebhookEventRequest { + watcher_hint: "driver-upgrade".to_owned(), + kind: "update.available".to_owned(), + service_id: None, + reason: Some("driver version is newer".to_owned()), + payload: json!({ + "component": "driver", + "tool": "restart_server", + }), + }, + )?; + + handle_driver_upgrade_event(&paths, WatcherMode::Propose, &mut state, &event)?; + let event_text = fs::read_to_string(paths.automation_events_path())?; + let event = serde_json::from_str::(event_text.trim())?; + let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; + fs::remove_dir_all(root).ok(); + + assert_eq!(event.watcher_id, "driver-upgrade"); + assert_eq!(event.action, "prepare_driver_plan"); + assert_eq!(proposals.len(), 1); + assert_eq!(proposals[0].watcher_id, "driver-upgrade"); + assert_eq!(proposals[0].tool.as_deref(), Some("driver_plan")); + assert!( + proposals[0].arguments.get("tool").is_none(), + "webhook payload must not grant arbitrary tool choice" + ); + Ok(()) + } + + #[test] + fn driver_upgrade_contained_mode_runs_restricted_driver_plan() -> Result<()> { + let (root, paths) = temp_app_paths("driver-upgrade-contained"); + paths.ensure()?; + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "driver-upgrade", + WatcherMode::Contained, + None, + )]); + let event = AutomationTriggerEvent { + at_unix_ms: 1, + kind: "update.available".to_owned(), + source: "test".to_owned(), + watcher_hint: Some("driver-upgrade".to_owned()), + service_id: None, + reason: Some("driver version is newer".to_owned()), + payload: json!({ + "component": "driver", + }), + }; + + handle_driver_upgrade_event_with_runner( + &paths, + WatcherMode::Contained, + &mut state, + &event, + |_paths| { + Ok(crate::sandbox::sandbox_driver_plan_value( + rocm_core::CommandCapture { + argv: vec![ + "rocm".to_owned(), + "install".to_owned(), + "driver".to_owned(), + "--dkms".to_owned(), + "--dry-run".to_owned(), + ], + exit_status: 0, + stdout: "driver install plan\n supported: true\n".to_owned(), + stderr: String::new(), + }, + )) + }, + )?; + let event_text = fs::read_to_string(paths.automation_events_path())?; + let event = serde_json::from_str::(event_text.trim())?; + let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; + fs::remove_dir_all(root).ok(); + + assert_eq!(event.action, "run_driver_plan"); + assert!( + event + .message + .contains("contained restricted driver_plan status=planned") + ); + assert!(event.message.contains("no driver commands were executed")); + assert!(proposals.is_empty()); + Ok(()) + } + + #[test] + fn driver_upgrade_contained_mode_requires_restricted_driver_plan_tool() -> Result<()> { + let (root, paths) = temp_app_paths("driver-upgrade-contained-tool"); + paths.ensure()?; + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "driver-upgrade", + WatcherMode::Contained, + None, + )]); + let event = AutomationTriggerEvent { + at_unix_ms: 1, + kind: "update.available".to_owned(), + source: "test".to_owned(), + watcher_hint: Some("driver-upgrade".to_owned()), + service_id: None, + reason: Some("driver version is newer".to_owned()), + payload: json!({ + "component": "driver", + }), + }; + + handle_driver_upgrade_event_with_runner( + &paths, + WatcherMode::Contained, + &mut state, + &event, + |_paths| { + Ok(json!({ + "tool": "check_updates", + "status": "checked", + "mutating": false, + })) + }, + )?; + let event_text = fs::read_to_string(paths.automation_events_path())?; + let event = serde_json::from_str::(event_text.trim())?; + let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; + fs::remove_dir_all(root).ok(); + + assert_eq!(event.action, "driver_plan_failed"); + assert!(event.message.contains("expected `driver_plan`")); + assert!(event.message.contains("no driver commands were executed")); + assert!(proposals.is_empty()); + Ok(()) + } + + #[test] + fn driver_upgrade_observe_mode_records_without_proposal() -> Result<()> { + let (root, paths) = temp_app_paths("driver-upgrade-observe"); + paths.ensure()?; + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "driver-upgrade", + WatcherMode::Observe, + None, + )]); + let event = AutomationTriggerEvent { + at_unix_ms: 1, + kind: "update.available".to_owned(), + source: "test".to_owned(), + watcher_hint: Some("driver-upgrade".to_owned()), + service_id: None, + reason: Some("driver version is newer".to_owned()), + payload: json!({ + "component": "driver", + }), + }; + + handle_driver_upgrade_event(&paths, WatcherMode::Observe, &mut state, &event)?; + let event_text = fs::read_to_string(paths.automation_events_path())?; + let event = serde_json::from_str::(event_text.trim())?; + let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; + fs::remove_dir_all(root).ok(); + + assert_eq!(event.action, "observe_driver_update"); + assert!(event.message.contains("does not queue or run")); + assert!(proposals.is_empty()); + Ok(()) + } + + #[test] + fn driver_upgrade_ignores_non_driver_component_without_proposal() -> Result<()> { + let (root, paths) = temp_app_paths("driver-upgrade-wrong-component"); + paths.ensure()?; + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "driver-upgrade", + WatcherMode::Propose, + None, + )]); + let event = AutomationTriggerEvent { + at_unix_ms: 1, + kind: "update.available".to_owned(), + source: "test".to_owned(), + watcher_hint: Some("driver-upgrade".to_owned()), + service_id: None, + reason: Some("runtime version is newer".to_owned()), + payload: json!({ + "component": "runtime", + }), + }; + + handle_driver_upgrade_event(&paths, WatcherMode::Propose, &mut state, &event)?; + let event_text = fs::read_to_string(paths.automation_events_path())?; + let event = serde_json::from_str::(event_text.trim())?; + let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; + fs::remove_dir_all(root).ok(); + + assert_eq!(event.action, "driver_upgrade_ignored_component"); + assert!(event.message.contains("payload.component=driver")); + assert!(proposals.is_empty()); + Ok(()) + } +} diff --git a/apps/rocmd/src/watchers/gpu_metrics.rs b/apps/rocmd/src/watchers/gpu_metrics.rs new file mode 100644 index 0000000000..be0c91ce00 --- /dev/null +++ b/apps/rocmd/src/watchers/gpu_metrics.rs @@ -0,0 +1,300 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The `gpu-metrics` watcher: turning an amd-smi snapshot into telemetry and +//! GPU-pressure events, and recording the telemetry events. +//! +//! This module has no test module of its own: its tests drive it through the +//! collector and the dispatcher in `watchers.rs` +//! (`event_collector_emits_gpu_metrics_event_when_enabled`, +//! `event_collector_emits_gpu_thermal_pressure_event_when_enabled`, +//! `event_collector_skips_gpu_pressure_below_thresholds` and +//! `gpu_metrics_event_records_read_only_status_without_proposal`), and a child +//! cannot import those without a cycle with the root. + +use anyhow::Result; +use rocm_core::{ + AppPaths, AutomationRuntimeState, AutomationTriggerEvent, CodexBridgeGpuSnapshot, WatcherMode, +}; +use serde_json::{Value, json}; + +pub(super) const GPU_METRICS_INTERVAL_MS: u128 = 60 * 1000; +pub(super) const GPU_THERMAL_HOTSPOT_PRESSURE_C: f64 = 95.0; +pub(super) const GPU_THERMAL_MEMORY_PRESSURE_C: f64 = 95.0; +pub(super) const GPU_MEMORY_VRAM_PRESSURE_PERCENT: f64 = 95.0; + +pub(super) fn gpu_metrics_due(state: &AutomationRuntimeState, now: u128) -> bool { + let Some(snapshot) = state + .active_watchers + .iter() + .find(|watcher| watcher.id == "gpu-metrics" && watcher.enabled) + else { + return false; + }; + snapshot + .last_event_unix_ms + .is_none_or(|last_event| now.saturating_sub(last_event) >= GPU_METRICS_INTERVAL_MS) +} + +pub(super) fn handle_gpu_metrics_event( + paths: &AppPaths, + mode: WatcherMode, + state: &mut AutomationRuntimeState, + event: &AutomationTriggerEvent, +) -> Result<()> { + let summary = event + .payload + .get("summary") + .and_then(Value::as_str) + .unwrap_or("summary unavailable"); + let level = if event.kind == "gpu.metrics" { + "info" + } else { + "warn" + }; + let mode_note = match mode { + WatcherMode::Observe => "observe mode records telemetry only", + WatcherMode::Propose => { + "propose mode has no GPU mutation policy yet, so telemetry is recorded only" + } + WatcherMode::Contained => { + "contained mode has no GPU mutation policy yet, so telemetry is recorded only" + } + }; + let reason = event + .reason + .as_deref() + .filter(|value| !value.trim().is_empty()) + .unwrap_or("no detail"); + let source = match event.source.as_str() { + "gpu_telemetry" => "local amd-smi telemetry", + "local_webhook" => "local webhook", + other => other, + }; + + crate::persistence::record_event( + paths, + state, + "gpu-metrics", + level, + "record_gpu_metrics", + &format!( + "GPU metrics event from {source}: {summary}; reason={reason}; {mode_note}; no mutating action was taken" + ), + None, + ) +} + +pub(super) fn gpu_snapshot_summary(snapshot: &CodexBridgeGpuSnapshot) -> String { + let mut parts = Vec::new(); + parts.push(format!("amd_smi_available={}", snapshot.amd_smi_available)); + parts.push(format!( + "static_snapshot={}", + if snapshot.static_snapshot.is_some() { + "available" + } else { + "missing" + } + )); + parts.push(format!( + "monitor_snapshot={}", + if snapshot.monitor_snapshot.is_some() { + "available" + } else { + "missing" + } + )); + if let Some(count) = snapshot.static_snapshot.as_ref().and_then(gpu_data_count) { + parts.push(format!("gpu_count={count}")); + } + if let Some(note) = snapshot.note.as_deref() + && !note.trim().is_empty() + { + parts.push(format!("note={note}")); + } + parts.join(" ") +} + +fn gpu_data_count(value: &Value) -> Option { + value + .get("gpu_data") + .and_then(Value::as_array) + .map(Vec::len) +} + +#[derive(Debug, Clone, Copy)] +struct GpuPressureReading { + gpu_index: Option, + hotspot_temperature_c: Option, + memory_temperature_c: Option, + vram_percent: Option, +} + +pub(super) fn gpu_pressure_events( + now: u128, + snapshot: &CodexBridgeGpuSnapshot, +) -> Vec { + let Some(monitor_snapshot) = snapshot.monitor_snapshot.as_ref() else { + return Vec::new(); + }; + monitor_entries(monitor_snapshot) + .into_iter() + .filter_map(gpu_pressure_reading) + .filter_map(|reading| gpu_pressure_event(now, reading)) + .collect() +} + +fn gpu_pressure_event(now: u128, reading: GpuPressureReading) -> Option { + let (kind, reason, metric_label, value, threshold) = if let Some(value) = + reading.hotspot_temperature_c + && value >= GPU_THERMAL_HOTSPOT_PRESSURE_C + { + ( + "gpu.thermal_pressure", + "hotspot_temperature_threshold", + "hotspot temperature", + value, + GPU_THERMAL_HOTSPOT_PRESSURE_C, + ) + } else if let Some(value) = reading.memory_temperature_c + && value >= GPU_THERMAL_MEMORY_PRESSURE_C + { + ( + "gpu.thermal_pressure", + "memory_temperature_threshold", + "memory temperature", + value, + GPU_THERMAL_MEMORY_PRESSURE_C, + ) + } else if let Some(value) = reading.vram_percent + && value >= GPU_MEMORY_VRAM_PRESSURE_PERCENT + { + ( + "gpu.memory_pressure", + "vram_pressure_threshold", + "VRAM use", + value, + GPU_MEMORY_VRAM_PRESSURE_PERCENT, + ) + } else { + return None; + }; + let gpu_label = reading + .gpu_index + .map_or_else(|| "the GPU".to_owned(), |gpu| format!("GPU {gpu}")); + let unit = if metric_label == "VRAM use" { + "%" + } else { + " C" + }; + let summary = format!( + "{gpu_label} {metric_label} is {}{} (limit {}{})", + display_metric(value), + unit, + display_metric(threshold), + unit + ); + Some(AutomationTriggerEvent { + at_unix_ms: now, + kind: kind.to_owned(), + source: "gpu_telemetry".to_owned(), + watcher_hint: Some("gpu-thermal-protect".to_owned()), + service_id: None, + reason: Some(reason.to_owned()), + payload: json!({ + "gpu": reading.gpu_index, + "hotspot_temperature_c": reading.hotspot_temperature_c, + "memory_temperature_c": reading.memory_temperature_c, + "vram_percent": reading.vram_percent, + "hotspot_threshold_c": GPU_THERMAL_HOTSPOT_PRESSURE_C, + "memory_temperature_threshold_c": GPU_THERMAL_MEMORY_PRESSURE_C, + "vram_threshold_percent": GPU_MEMORY_VRAM_PRESSURE_PERCENT, + "recommended_action": "stop_serving_load", + "summary": summary, + }), + }) +} + +fn monitor_entries(value: &Value) -> Vec<&Value> { + if let Some(entries) = value.as_array() { + return entries.iter().collect(); + } + value + .get("gpu_data") + .and_then(Value::as_array) + .map(|entries| entries.iter().collect()) + .unwrap_or_default() +} + +fn gpu_pressure_reading(entry: &Value) -> Option { + let reading = GpuPressureReading { + gpu_index: metric_u64(entry, &["gpu", "gpu_id", "gpu_index"]), + hotspot_temperature_c: metric_f64( + entry, + &[ + "hotspot_temperature", + "hotspot_temperature_c", + "temperature_hotspot", + ], + ), + memory_temperature_c: metric_f64( + entry, + &[ + "memory_temperature", + "memory_temperature_c", + "temperature_memory", + ], + ), + vram_percent: metric_f64( + entry, + &["vram_percent", "vram_usage_percent", "vram_used_percent"], + ), + }; + (reading.hotspot_temperature_c.is_some() + || reading.memory_temperature_c.is_some() + || reading.vram_percent.is_some()) + .then_some(reading) +} + +fn metric_f64(entry: &Value, keys: &[&str]) -> Option { + keys.iter() + .find_map(|key| entry.get(*key).and_then(value_as_metric_f64)) +} + +fn metric_u64(entry: &Value, keys: &[&str]) -> Option { + keys.iter() + .find_map(|key| entry.get(*key).and_then(value_as_metric_u64)) +} + +fn value_as_metric_f64(value: &Value) -> Option { + match value { + Value::Number(number) => number.as_f64(), + Value::String(text) => text.trim().parse::().ok(), + Value::Object(map) => map + .get("value") + .or_else(|| map.get("val")) + .and_then(value_as_metric_f64), + _ => None, + } +} + +fn value_as_metric_u64(value: &Value) -> Option { + match value { + Value::Number(number) => number.as_u64(), + Value::String(text) => text.trim().parse::().ok(), + Value::Object(map) => map + .get("value") + .or_else(|| map.get("val")) + .and_then(value_as_metric_u64), + _ => None, + } +} + +fn display_metric(value: f64) -> String { + if value.fract().abs() < f64::EPSILON { + format!("{value:.0}") + } else { + format!("{value:.1}") + } +} diff --git a/apps/rocmd/src/watchers/gpu_thermal_protect.rs b/apps/rocmd/src/watchers/gpu_thermal_protect.rs new file mode 100644 index 0000000000..0cf2e72661 --- /dev/null +++ b/apps/rocmd/src/watchers/gpu_thermal_protect.rs @@ -0,0 +1,445 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The `gpu-thermal-protect` watcher: choosing the one running managed +//! service to relieve when the GPU is under thermal or memory pressure, and +//! queueing a reviewed stop request for it. + +use super::gpu_metrics::{ + GPU_MEMORY_VRAM_PRESSURE_PERCENT, GPU_METRICS_INTERVAL_MS, GPU_THERMAL_HOTSPOT_PRESSURE_C, + GPU_THERMAL_MEMORY_PRESSURE_C, +}; +use super::policy::{pending_proposal_exists, queue_proposal_with_arguments}; +use crate::common::payload_string; +use crate::persistence::load_service_record; +use anyhow::Result; +use rocm_core::{ + AppPaths, AutomationRuntimeState, AutomationTriggerEvent, ManagedServiceRecord, WatcherMode, +}; +use serde_json::{Value, json}; + +pub(super) fn gpu_thermal_protect_due(state: &AutomationRuntimeState, now: u128) -> bool { + let Some(snapshot) = state + .active_watchers + .iter() + .find(|watcher| watcher.id == "gpu-thermal-protect" && watcher.enabled) + else { + return false; + }; + snapshot + .last_event_unix_ms + .is_none_or(|last_event| now.saturating_sub(last_event) >= GPU_METRICS_INTERVAL_MS) +} + +pub(super) fn handle_gpu_thermal_protect_event( + paths: &AppPaths, + mode: WatcherMode, + state: &mut AutomationRuntimeState, + event: &AutomationTriggerEvent, +) -> Result<()> { + let summary = payload_string(&event.payload, "summary") + .unwrap_or_else(|| "GPU pressure is high".to_owned()); + let reason = event.reason.as_deref().unwrap_or("gpu_pressure_threshold"); + + if matches!(mode, WatcherMode::Observe) { + return crate::persistence::record_event( + paths, + state, + "gpu-thermal-protect", + "warn", + "observe_gpu_pressure", + &format!( + "{summary}; observe mode records this only and does not stop any model server" + ), + event.service_id.clone(), + ); + } + + let Some(record) = resolve_gpu_pressure_service_target(paths, event)? else { + return crate::persistence::record_event( + paths, + state, + "gpu-thermal-protect", + "warn", + "gpu_pressure_no_clear_target", + &format!( + "{summary}; rocm-cli did not choose a model server to stop because there was no single clear running managed server" + ), + None, + ); + }; + + if pending_stop_proposal_exists(paths, &record.service_id)? { + return crate::persistence::record_event( + paths, + state, + "gpu-thermal-protect", + "info", + "stop_proposal_already_pending", + &format!( + "{summary}; a reviewed stop request is already waiting for {}", + record.service_id + ), + Some(record.service_id), + ); + } + + let action = "queue_stop_server_proposal"; + let mode_note = if matches!(mode, WatcherMode::Contained) { + "contained mode still asks before stopping anything" + } else { + "asking before stopping anything" + }; + let message = format!( + "{summary}; {mode_note}; selected managed server {} ({})", + record.service_id, record.endpoint_url + ); + crate::persistence::record_event( + paths, + state, + "gpu-thermal-protect", + "warn", + action, + &message, + Some(record.service_id.clone()), + )?; + queue_proposal_with_arguments( + paths, + "gpu-thermal-protect", + action, + "Review GPU pressure stop", + &message, + Some(record.service_id.clone()), + json!({ + "service_id": record.service_id, + "model_ref": record.model_ref, + "canonical_model_id": record.canonical_model_id, + "endpoint_url": record.endpoint_url, + "engine": record.engine, + "pressure_kind": event.kind, + "pressure_reason": reason, + "pressure_summary": summary, + "gpu": event.payload.get("gpu").cloned().unwrap_or(Value::Null), + "hotspot_temperature_c": event.payload.get("hotspot_temperature_c").cloned().unwrap_or(Value::Null), + "memory_temperature_c": event.payload.get("memory_temperature_c").cloned().unwrap_or(Value::Null), + "vram_percent": event.payload.get("vram_percent").cloned().unwrap_or(Value::Null), + "hotspot_threshold_c": GPU_THERMAL_HOTSPOT_PRESSURE_C, + "memory_temperature_threshold_c": GPU_THERMAL_MEMORY_PRESSURE_C, + "vram_threshold_percent": GPU_MEMORY_VRAM_PRESSURE_PERCENT, + }), + ) +} + +fn resolve_gpu_pressure_service_target( + paths: &AppPaths, + event: &AutomationTriggerEvent, +) -> Result> { + if let Some(service_id) = event + .service_id + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + { + let record = load_service_record(paths, service_id)?; + return Ok(active_pressure_target(record)); + } + + let active = crate::persistence::load_managed_services(paths)? + .into_iter() + .filter_map(active_pressure_target) + .collect::>(); + if active.len() == 1 { + Ok(active.into_iter().next()) + } else { + Ok(None) + } +} + +fn active_pressure_target(record: ManagedServiceRecord) -> Option { + (record.mode == "managed" && matches!(record.status.as_str(), "ready" | "running")) + .then_some(record) +} + +fn pending_stop_proposal_exists(paths: &AppPaths, service_id: &str) -> Result { + pending_proposal_exists( + paths, + "gpu-thermal-protect", + service_id, + "queue_stop_server_proposal", + "stop_server", + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::{ + proposals_beyond_a_recent_window, temp_app_paths, test_runtime_state, test_watcher_snapshot, + }; + use rocm_core::AutomationEventRecord; + use std::fs; + + #[test] + fn gpu_thermal_protect_contained_still_queues_reviewed_stop() -> Result<()> { + let (root, paths) = temp_app_paths("gpu-thermal-protect-contained"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "svc-hot", + "vllm", + "qwen", + "Qwen/Test", + "127.0.0.1", + 11436, + "managed", + 123, + None, + None, + None, + ); + record.status = "running".to_owned(); + record.write()?; + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "gpu-thermal-protect", + WatcherMode::Contained, + None, + )]); + let event = AutomationTriggerEvent { + at_unix_ms: 1, + kind: "gpu.memory_pressure".to_owned(), + source: "gpu_telemetry".to_owned(), + watcher_hint: Some("gpu-thermal-protect".to_owned()), + service_id: Some("svc-hot".to_owned()), + reason: Some("vram_pressure_threshold".to_owned()), + payload: json!({ + "summary": "GPU 0 VRAM use is 96% (limit 95%)", + "vram_percent": 96.0, + }), + }; + + handle_gpu_thermal_protect_event(&paths, WatcherMode::Contained, &mut state, &event)?; + let event_text = fs::read_to_string(paths.automation_events_path())?; + let event = serde_json::from_str::(event_text.trim())?; + let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; + let saved = load_service_record(&paths, "svc-hot")?; + fs::remove_dir_all(root).ok(); + + assert_eq!(event.action, "queue_stop_server_proposal"); + assert!(event.message.contains("contained mode still asks")); + assert_eq!(saved.status, "running"); + assert_eq!(proposals.len(), 1); + assert_eq!(proposals[0].tool.as_deref(), Some("stop_server")); + Ok(()) + } + + #[test] + fn gpu_thermal_protect_observe_records_without_proposal() -> Result<()> { + let (root, paths) = temp_app_paths("gpu-thermal-protect-observe"); + paths.ensure()?; + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "gpu-thermal-protect", + WatcherMode::Observe, + None, + )]); + let event = AutomationTriggerEvent { + at_unix_ms: 1, + kind: "gpu.thermal_pressure".to_owned(), + source: "gpu_telemetry".to_owned(), + watcher_hint: Some("gpu-thermal-protect".to_owned()), + service_id: None, + reason: Some("memory_temperature_threshold".to_owned()), + payload: json!({ + "summary": "GPU memory temperature is high", + }), + }; + + handle_gpu_thermal_protect_event(&paths, WatcherMode::Observe, &mut state, &event)?; + let event_text = fs::read_to_string(paths.automation_events_path())?; + let event = serde_json::from_str::(event_text.trim())?; + let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; + fs::remove_dir_all(root).ok(); + + assert_eq!(event.action, "observe_gpu_pressure"); + assert!(event.message.contains("does not stop any model server")); + assert!(proposals.is_empty()); + Ok(()) + } + + #[test] + fn gpu_thermal_protect_ambiguous_services_records_no_action() -> Result<()> { + let (root, paths) = temp_app_paths("gpu-thermal-protect-ambiguous"); + paths.ensure()?; + for service_id in ["svc-a", "svc-b"] { + let mut record = ManagedServiceRecord::new( + &paths, + service_id, + "vllm", + "qwen", + "Qwen/Test", + "127.0.0.1", + 11435, + "managed", + 123, + None, + None, + None, + ); + record.status = "ready".to_owned(); + record.write()?; + } + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "gpu-thermal-protect", + WatcherMode::Propose, + None, + )]); + let event = AutomationTriggerEvent { + at_unix_ms: 1, + kind: "gpu.thermal_pressure".to_owned(), + source: "gpu_telemetry".to_owned(), + watcher_hint: Some("gpu-thermal-protect".to_owned()), + service_id: None, + reason: Some("hotspot_temperature_threshold".to_owned()), + payload: json!({ + "summary": "GPU 0 hotspot temperature is 96 C (limit 95 C)", + }), + }; + + handle_gpu_thermal_protect_event(&paths, WatcherMode::Propose, &mut state, &event)?; + let event_text = fs::read_to_string(paths.automation_events_path())?; + let event = serde_json::from_str::(event_text.trim())?; + let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; + fs::remove_dir_all(root).ok(); + + assert_eq!(event.action, "gpu_pressure_no_clear_target"); + assert!(event.message.contains("did not choose a model server")); + assert!(proposals.is_empty()); + Ok(()) + } + + #[test] + fn gpu_thermal_protect_does_not_duplicate_pending_stop_proposals() -> Result<()> { + let (root, paths) = temp_app_paths("gpu-thermal-protect-dedupe"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "svc-hot", + "vllm", + "tiny", + "Tiny/Test", + "127.0.0.1", + 11435, + "managed", + 123, + None, + None, + None, + ); + record.status = "ready".to_owned(); + record.write()?; + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "gpu-thermal-protect", + WatcherMode::Propose, + None, + )]); + let event = AutomationTriggerEvent { + at_unix_ms: 1, + kind: "gpu.thermal_pressure".to_owned(), + source: "gpu_telemetry".to_owned(), + watcher_hint: Some("gpu-thermal-protect".to_owned()), + service_id: Some("svc-hot".to_owned()), + reason: Some("hotspot_temperature_threshold".to_owned()), + payload: json!({ + "summary": "GPU 0 hotspot temperature is 96 C (limit 95 C)", + }), + }; + + handle_gpu_thermal_protect_event(&paths, WatcherMode::Propose, &mut state, &event)?; + handle_gpu_thermal_protect_event(&paths, WatcherMode::Propose, &mut state, &event)?; + let events = rocm_core::load_recent_automation_events(&paths, 2)?; + let proposals = rocm_core::load_recent_automation_proposals(&paths, 10)?; + fs::remove_dir_all(root).ok(); + + assert_eq!(proposals.len(), 1); + assert_eq!(proposals[0].tool.as_deref(), Some("stop_server")); + assert!( + events + .iter() + .any(|event| event.action == "stop_proposal_already_pending") + ); + Ok(()) + } + + #[test] + fn gpu_thermal_protect_dedupe_sees_a_pending_stop_older_than_the_recent_window() -> Result<()> { + let (root, paths) = temp_app_paths("gpu-thermal-protect-dedupe-old"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-hot", + "vllm", + "tiny", + "Tiny/Test", + "127.0.0.1", + 11435, + "managed", + 123, + None, + None, + None, + ); + record.status = "ready".to_owned(); + record.write()?; + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "gpu-thermal-protect", + WatcherMode::Propose, + None, + )]); + let event = AutomationTriggerEvent { + at_unix_ms: 1, + kind: "gpu.thermal_pressure".to_owned(), + source: "gpu_telemetry".to_owned(), + watcher_hint: Some("gpu-thermal-protect".to_owned()), + service_id: Some("svc-hot".to_owned()), + reason: Some("hotspot_temperature_threshold".to_owned()), + payload: json!({ + "summary": "GPU 0 hotspot temperature is 96 C (limit 95 C)", + }), + }; + + handle_gpu_thermal_protect_event(&paths, WatcherMode::Propose, &mut state, &event)?; + // More newer proposals than the store keeps reviewed ones (all pending + // here, so none is compacted away), which puts the stop proposal beyond + // a recent-window read the size of that cap. A pending proposal is + // never dropped from the store, so it is still waiting for review. + for index in 0..proposals_beyond_a_recent_window() { + queue_proposal_with_arguments( + &paths, + "gpu-thermal-protect", + "queue_stop_server_proposal", + "Stop another server", + "unrelated proposal", + Some(format!("svc-other-{index}")), + json!({}), + )?; + } + handle_gpu_thermal_protect_event(&paths, WatcherMode::Propose, &mut state, &event)?; + let stops_for_hot_server = rocm_core::load_automation_proposals(&paths)? + .into_iter() + .filter(|proposal| { + proposal.tool.as_deref() == Some("stop_server") + && proposal.service_id.as_deref() == Some("svc-hot") + && proposal.status == "pending" + }) + .count(); + let events = rocm_core::load_recent_automation_events(&paths, 2)?; + fs::remove_dir_all(root).ok(); + + assert_eq!(stops_for_hot_server, 1); + assert!( + events + .iter() + .any(|event| event.action == "stop_proposal_already_pending") + ); + Ok(()) + } +} diff --git a/apps/rocmd/src/watchers/policy.rs b/apps/rocmd/src/watchers/policy.rs new file mode 100644 index 0000000000..a59cb2da9f --- /dev/null +++ b/apps/rocmd/src/watchers/policy.rs @@ -0,0 +1,151 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The watcher policy: what a watcher does in each mode, and the automation +//! proposals it queues for review. + +use anyhow::Result; +use rocm_core::{ + AppPaths, AutomationProposalRecord, WatcherMode, append_automation_proposal, unix_time_millis, +}; +use serde_json::{Value, json}; + +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub(super) enum WatcherPolicyAction { + Observe, + QueueProposal, + RunContained, +} + +pub(super) const fn watcher_policy_action( + watcher_id: &str, + mode: WatcherMode, +) -> WatcherPolicyAction { + match (watcher_id, mode) { + (_, WatcherMode::Observe) => WatcherPolicyAction::Observe, + (_, WatcherMode::Propose) => WatcherPolicyAction::QueueProposal, + (_, WatcherMode::Contained) => WatcherPolicyAction::RunContained, + } +} + +pub(super) fn queue_proposal( + paths: &AppPaths, + watcher_id: &str, + action: &str, + title: &str, + message: &str, + service_id: Option, +) -> Result<()> { + queue_proposal_with_arguments( + paths, + watcher_id, + action, + title, + message, + service_id.clone(), + proposal_arguments_for_action(action, service_id.as_deref()), + ) +} + +pub(super) fn queue_proposal_with_arguments( + paths: &AppPaths, + watcher_id: &str, + action: &str, + title: &str, + message: &str, + service_id: Option, + arguments: Value, +) -> Result<()> { + append_automation_proposal( + paths, + &AutomationProposalRecord { + at_unix_ms: unix_time_millis(), + proposal_id: String::new(), + watcher_id: watcher_id.to_owned(), + action: action.to_owned(), + title: title.to_owned(), + message: message.to_owned(), + status: "pending".to_owned(), + service_id, + tool: proposal_tool_for_action(action).map(str::to_owned), + arguments, + reviewed_at_unix_ms: None, + }, + ) +} + +fn proposal_tool_for_action(action: &str) -> Option<&'static str> { + match action { + "queue_restart_proposal" => Some("restart_server"), + "queue_stop_server_proposal" => Some("stop_server"), + "queue_update_proposal" => Some("check_updates"), + "queue_prefetch_proposal" => Some("prefetch_artifact"), + "prepare_driver_plan" => Some("driver_plan"), + _ => None, + } +} + +fn proposal_arguments_for_action(action: &str, service_id: Option<&str>) -> Value { + match action { + "queue_restart_proposal" => json!({ + "service_id": service_id, + }), + "queue_stop_server_proposal" => json!({ + "service_id": service_id, + }), + "queue_update_proposal" => json!({}), + "prepare_driver_plan" => json!({}), + _ => Value::Null, + } +} + +/// Whether `watcher_id` already has a pending proposal for `service_id`, either +/// queued under `action` or carrying `tool`. +/// +/// Reads the whole proposals store rather than a recent window: a pending +/// proposal is never dropped, so it can be arbitrarily old, and a window that +/// missed it would let the watcher queue a duplicate once the original aged +/// out of that window. Compaction bounds the reviewed proposals to a fixed +/// count; pending ones are kept. +pub(super) fn pending_proposal_exists( + paths: &AppPaths, + watcher_id: &str, + service_id: &str, + action: &str, + tool: &str, +) -> Result { + Ok(rocm_core::load_automation_proposals(paths)? + .into_iter() + .any(|proposal| { + proposal.status == "pending" + && proposal.watcher_id == watcher_id + && proposal.service_id.as_deref() == Some(service_id) + && (proposal.action == action || proposal.tool.as_deref() == Some(tool)) + })) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn watcher_policy_maps_modes_to_decisions() { + assert_eq!( + watcher_policy_action("server-recover", WatcherMode::Observe), + WatcherPolicyAction::Observe + ); + assert_eq!( + watcher_policy_action("server-recover", WatcherMode::Propose), + WatcherPolicyAction::QueueProposal + ); + assert_eq!( + watcher_policy_action("server-recover", WatcherMode::Contained), + WatcherPolicyAction::RunContained + ); + assert_eq!( + watcher_policy_action("therock-update", WatcherMode::Contained), + WatcherPolicyAction::RunContained + ); + } +} diff --git a/apps/rocmd/src/watchers/server_recover.rs b/apps/rocmd/src/watchers/server_recover.rs new file mode 100644 index 0000000000..b82a8c5338 --- /dev/null +++ b/apps/rocmd/src/watchers/server_recover.rs @@ -0,0 +1,934 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The `server-recover` watcher: finding a managed service that needs +//! recovery, and restarting it or queueing a restart proposal. + +use super::policy::{ + WatcherPolicyAction, pending_proposal_exists, queue_proposal, watcher_policy_action, +}; +use crate::persistence::load_service_record; +use crate::supervise::restart_managed_service; +use anyhow::{Context, Result}; +use rocm_core::{ + AppPaths, AutomationRuntimeState, AutomationTriggerEvent, ManagedServiceRecord, WatcherMode, + format_host_port, unix_time_millis, +}; +#[cfg(test)] +use serde_json::json; +use std::net::{SocketAddr, TcpStream}; +use std::thread; +use std::time::Duration; + +const SERVER_RECOVER_BACKOFF_MS: u128 = 30_000; +const SERVER_TRANSIENT_STALE_MS: u128 = 5 * 60 * 1_000; +const ENDPOINT_HEALTH_TIMEOUT: Duration = Duration::from_millis(250); + +#[cfg(test)] +fn evaluate_server_recover( + paths: &AppPaths, + mode: WatcherMode, + state: &mut AutomationRuntimeState, +) -> Result<()> { + let now = unix_time_millis(); + if !server_recover_due(state, now) { + return Ok(()); + } + + let Some((mut record, recovery_reason)) = find_recoverable_service(paths)? else { + return Ok(()); + }; + let kind = service_recovery_event_kind(&recovery_reason); + let event = AutomationTriggerEvent { + at_unix_ms: now, + kind: kind.to_owned(), + source: "managed_service".to_owned(), + watcher_hint: Some("server-recover".to_owned()), + service_id: Some(record.service_id.clone()), + reason: Some(recovery_reason), + payload: json!({ + "engine": record.engine.clone(), + "status": record.status.clone(), + "endpoint": record.endpoint_url.clone(), + }), + }; + handle_server_recover_event_with_record(paths, mode, state, &event, &mut record) +} + +pub(super) fn handle_server_recover_event( + paths: &AppPaths, + mode: WatcherMode, + state: &mut AutomationRuntimeState, + event: &AutomationTriggerEvent, +) -> Result<()> { + let service_id = event + .service_id + .as_deref() + .context("server-recover event is missing service_id")?; + let mut record = load_service_record(paths, service_id)?; + if !service_record_matches_recovery_event(paths, &record, event) { + crate::persistence::record_event( + paths, + state, + "server-recover", + "info", + "ignore_nonrecoverable_service", + &format!( + "managed service {} does not currently need recovery; restart not attempted", + record.service_id + ), + Some(record.service_id.clone()), + )?; + return Ok(()); + } + handle_server_recover_event_with_record(paths, mode, state, event, &mut record) +} + +fn service_record_matches_recovery_event( + paths: &AppPaths, + record: &ManagedServiceRecord, + event: &AutomationTriggerEvent, +) -> bool { + // Re-checked here as well as in the scan: a stop can be requested between + // the event being raised and it being handled. + if stop_requested(record) { + return false; + } + match event.kind.as_str() { + "service.manifest_recoverable" => { + manifest_service_recovery_reason(record, unix_time_millis()).is_some() + } + "service.endpoint_recoverable" => endpoint_service_recovery_reason(record).is_some(), + "service.healthcheck_recoverable" => crate::engine_ipc::engine_healthcheck_response( + paths, + &record.engine, + &record.service_id, + ) + .is_ok_and(|healthcheck| crate::engine_ipc::healthcheck_response_recoverable(&healthcheck)), + _ => false, + } +} + +fn handle_server_recover_event_with_record( + paths: &AppPaths, + mode: WatcherMode, + state: &mut AutomationRuntimeState, + event: &AutomationTriggerEvent, + record: &mut ManagedServiceRecord, +) -> Result<()> { + let now = unix_time_millis(); + let recovery_reason = event.reason.as_deref().unwrap_or("recoverable_event"); + let recovery_reason_display = display_recovery_reason(recovery_reason); + + match watcher_policy_action("server-recover", mode) { + WatcherPolicyAction::Observe => crate::persistence::record_event( + paths, + state, + "server-recover", + "warn", + "observe_failure", + &format!( + "observed managed service {} needing recovery ({recovery_reason_display}); restart not attempted in observe mode", + record.service_id, + ), + Some(record.service_id.clone()), + ), + WatcherPolicyAction::QueueProposal => { + // An unreadable proposals store is reported and skipped, never + // propagated: the daemon loop exits on an error out of + // `evaluate_watchers`, so a store that stays corrupt would take it + // down on every recovery tick. + match pending_restart_proposal_exists(paths, &record.service_id) { + Ok(true) => { + return crate::persistence::record_event( + paths, + state, + "server-recover", + "info", + "restart_proposal_already_pending", + &format!( + "managed service {} needs recovery ({recovery_reason_display}); a restart request awaiting review is already queued", + record.service_id, + ), + Some(record.service_id.clone()), + ); + } + Ok(false) => {} + Err(error) => { + return crate::persistence::record_event( + paths, + state, + "server-recover", + "error", + "restart_proposal_check_failed", + &format!( + "managed service {} needs recovery ({recovery_reason_display}); \ + not queueing a restart request because the proposals store could \ + not be checked for one already pending: {error:#}", + record.service_id, + ), + Some(record.service_id.clone()), + ); + } + } + let message = format!( + "managed service {} needs recovery ({recovery_reason_display}); queueing restart proposal", + record.service_id, + ); + crate::persistence::record_event( + paths, + state, + "server-recover", + "warn", + "queue_restart_proposal", + &message, + Some(record.service_id.clone()), + )?; + queue_proposal( + paths, + "server-recover", + "queue_restart_proposal", + "Restart managed service", + &message, + Some(record.service_id.clone()), + ) + } + WatcherPolicyAction::RunContained => { + if let Some(last_restart) = record.last_restart_unix_ms + && now.saturating_sub(last_restart) < SERVER_RECOVER_BACKOFF_MS + { + return Ok(()); + } + // A public service whose endpoint key is gone can never be recovered: + // the respawn guard in `supervise_service` refuses it by design. + // Report it and stop, rather than letting a permanent failure + // propagate out of `evaluate_watchers` and take the whole daemon — + // and every other watcher — down on each 30s recovery tick. + if let Err(error) = crate::engine_ipc::ensure_public_service_has_endpoint_key( + &record.host, + rocm_engine_protocol::endpoint_key_file_if_present(paths, &record.service_id) + .and_then(|path| rocm_engine_protocol::endpoint_api_key_file_if_valid(&path)) + .is_some(), + record.requires_api_key, + ) { + return crate::persistence::record_event( + paths, + state, + "server-recover", + "error", + "restart_managed_service_refused", + &format!( + "cannot recover managed service {} on {}:{} after \ + {recovery_reason_display}: {error}", + record.service_id, record.host, record.port + ), + Some(record.service_id.clone()), + ); + } + restart_managed_service(paths, &mut *record)?; + crate::persistence::record_event( + paths, + state, + "server-recover", + "info", + "restart_managed_service", + &format!( + "restarted managed service {} on {}:{} after {recovery_reason_display}", + record.service_id, record.host, record.port + ), + Some(record.service_id.clone()), + ) + } + } +} + +fn display_recovery_reason(reason: &str) -> String { + match reason { + "manifest_status_failed" => "manifest reports failed".to_owned(), + "manifest_status_exited" => "manifest reports exited".to_owned(), + "manifest_status_unreachable" => "manifest reports unreachable".to_owned(), + "manifest_status_starting_stale" => "service has been starting for too long".to_owned(), + "manifest_status_recovering_stale" => "service has been recovering for too long".to_owned(), + "endpoint_status_unreachable" => "endpoint port is unreachable".to_owned(), + other if other.starts_with("healthcheck_status_") => format!( + "engine healthcheck reports {}", + other.trim_start_matches("healthcheck_status_") + ), + other => other.replace('_', " "), + } +} + +pub(super) fn service_recovery_event_kind(recovery_reason: &str) -> &'static str { + if recovery_reason.starts_with("healthcheck_status_") { + "service.healthcheck_recoverable" + } else if recovery_reason.starts_with("endpoint_status_") { + "service.endpoint_recoverable" + } else { + "service.manifest_recoverable" + } +} + +pub(super) fn server_recover_due(state: &AutomationRuntimeState, now: u128) -> bool { + let Some(snapshot) = state + .active_watchers + .iter() + .find(|watcher| watcher.id == "server-recover" && watcher.enabled) + else { + return false; + }; + snapshot + .last_event_unix_ms + .is_none_or(|last_event| now.saturating_sub(last_event) >= SERVER_RECOVER_BACKOFF_MS) +} + +pub(crate) fn find_recoverable_service( + paths: &AppPaths, +) -> Result> { + let now = unix_time_millis(); + for record in crate::persistence::load_managed_services(paths)? { + if record.mode != "managed" || stop_requested(&record) { + continue; + } + if let Some(reason) = manifest_service_recovery_reason(&record, now) { + return Ok(Some((record, reason))); + } + if matches!(record.status.as_str(), "ready" | "running") { + let Ok(healthcheck) = crate::engine_ipc::engine_healthcheck_response( + paths, + &record.engine, + &record.service_id, + ) else { + if let Some(reason) = endpoint_service_recovery_reason(&record) { + return Ok(Some((record, reason))); + } + continue; + }; + if crate::engine_ipc::healthcheck_response_recoverable(&healthcheck) { + return Ok(Some(( + record, + format!("healthcheck_status_{}", healthcheck.status), + ))); + } + if let Some(reason) = endpoint_service_recovery_reason(&record) { + return Ok(Some((record, reason))); + } + } + } + Ok(None) +} + +/// Whether an operator asked for this service to stop and that stop is still +/// standing. +/// +/// Such a service is never a recovery candidate, whatever its status says. A +/// stop that could not confirm every recorded process gone deliberately leaves +/// the status at `ready`/`running` — it did not happen, so it must not be +/// claimed — and a stop still in flight has not reached its verdict yet. Either +/// way the endpoint has stopped answering, which on its own is exactly what +/// recovery restarts. `stop_requested_unix_ms` is what tells the operator's +/// stop apart from a crash; a confirmed stop, and any fresh launch, clear it. +pub(crate) const fn stop_requested(record: &ManagedServiceRecord) -> bool { + record.stop_requested_unix_ms.is_some() +} + +fn endpoint_service_recovery_reason(record: &ManagedServiceRecord) -> Option { + (!wait_for_port(&record.host, record.port, ENDPOINT_HEALTH_TIMEOUT)) + .then(|| "endpoint_status_unreachable".to_owned()) +} + +fn wait_for_port(host: &str, port: u16, timeout: Duration) -> bool { + let address: SocketAddr = match format_host_port(host, port).parse() { + Ok(value) => value, + Err(_) => return false, + }; + + let start = std::time::Instant::now(); + while start.elapsed() < timeout { + if TcpStream::connect_timeout(&address, Duration::from_millis(200)).is_ok() { + return true; + } + thread::sleep(Duration::from_millis(200)); + } + false +} + +fn manifest_service_recovery_reason( + record: &ManagedServiceRecord, + now_unix_ms: u128, +) -> Option { + match record.status.as_str() { + "failed" | "exited" | "unreachable" => Some(format!("manifest_status_{}", record.status)), + "starting" | "recovering" => { + let started_at = record + .last_restart_unix_ms + .unwrap_or(record.created_at_unix_ms); + (now_unix_ms.saturating_sub(started_at) >= SERVER_TRANSIENT_STALE_MS) + .then(|| format!("manifest_status_{}_stale", record.status)) + } + _ => None, + } +} + +fn pending_restart_proposal_exists(paths: &AppPaths, service_id: &str) -> Result { + pending_proposal_exists( + paths, + "server-recover", + service_id, + "queue_restart_proposal", + "restart_server", + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::{ + UNCONFIRMED_STOP_PID, proposals_beyond_a_recent_window, seed_keyed_service, + stop_with_outcome, temp_app_paths, test_runtime_state, test_watcher_snapshot, + }; + use crate::watchers::policy::queue_proposal_with_arguments; + use rocm_core::{AutomationProposalRecord, WatcherRuntimeSnapshot, append_automation_proposal}; + use serde_json::Value; + use std::fs; + + #[test] + fn server_recover_does_not_duplicate_pending_restart_proposals() -> Result<()> { + let (root, paths) = temp_app_paths("server-recover-dedupe"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-unhealthy", + "vllm", + "tiny", + "Tiny/Test", + "127.0.0.1", + 11436, + "managed", + 123, + None, + None, + None, + ); + record.status = "unreachable".to_owned(); + record.write()?; + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "server-recover", + WatcherMode::Propose, + None, + )]); + let event = AutomationTriggerEvent { + at_unix_ms: 1, + kind: "service.manifest_recoverable".to_owned(), + source: "managed_service".to_owned(), + watcher_hint: Some("server-recover".to_owned()), + service_id: Some(record.service_id.clone()), + reason: Some("manifest_status_failed".to_owned()), + payload: json!({}), + }; + + handle_server_recover_event_with_record( + &paths, + WatcherMode::Propose, + &mut state, + &event, + &mut record, + )?; + handle_server_recover_event_with_record( + &paths, + WatcherMode::Propose, + &mut state, + &event, + &mut record, + )?; + let events = rocm_core::load_recent_automation_events(&paths, 2)?; + let proposals = rocm_core::load_recent_automation_proposals(&paths, 10)?; + fs::remove_dir_all(root).ok(); + + assert_eq!(proposals.len(), 1); + assert_eq!(proposals[0].tool.as_deref(), Some("restart_server")); + // The wording describes a proposal still awaiting review; the single + // pending proposal asserted here is what makes that true. + assert_eq!(proposals[0].status, "pending"); + assert!(events.iter().any(|event| { + event.action == "restart_proposal_already_pending" + && event.message.contains("a restart request awaiting review") + })); + Ok(()) + } + + #[test] + fn server_recover_dedupe_sees_a_pending_restart_older_than_the_recent_window() -> Result<()> { + let (root, paths) = temp_app_paths("server-recover-dedupe-old"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-unhealthy", + "vllm", + "tiny", + "Tiny/Test", + "127.0.0.1", + 11436, + "managed", + 123, + None, + None, + None, + ); + record.status = "unreachable".to_owned(); + record.write()?; + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "server-recover", + WatcherMode::Propose, + None, + )]); + let event = AutomationTriggerEvent { + at_unix_ms: 1, + kind: "service.manifest_recoverable".to_owned(), + source: "managed_service".to_owned(), + watcher_hint: Some("server-recover".to_owned()), + service_id: Some(record.service_id.clone()), + reason: Some("manifest_status_failed".to_owned()), + payload: json!({}), + }; + + handle_server_recover_event_with_record( + &paths, + WatcherMode::Propose, + &mut state, + &event, + &mut record, + )?; + // More newer proposals than the store keeps reviewed ones (all pending + // here, so none is compacted away), which puts the restart proposal + // beyond a recent-window read the size of that cap. + for index in 0..proposals_beyond_a_recent_window() { + queue_proposal_with_arguments( + &paths, + "gpu-thermal-protect", + "queue_stop_server_proposal", + "Stop another server", + "unrelated proposal", + Some(format!("svc-other-{index}")), + json!({}), + )?; + } + handle_server_recover_event_with_record( + &paths, + WatcherMode::Propose, + &mut state, + &event, + &mut record, + )?; + let restarts = rocm_core::load_automation_proposals(&paths)? + .into_iter() + .filter(|proposal| proposal.tool.as_deref() == Some("restart_server")) + .count(); + fs::remove_dir_all(root).ok(); + + assert_eq!(restarts, 1); + Ok(()) + } + + #[test] + fn server_recover_dedupe_ignores_proposals_that_do_not_block_a_restart() -> Result<()> { + let (root, paths) = temp_app_paths("server-recover-dedupe-filters"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-unhealthy", + "vllm", + "tiny", + "Tiny/Test", + "127.0.0.1", + 11436, + "managed", + 123, + None, + None, + None, + ); + record.status = "unreachable".to_owned(); + record.write()?; + // Each of these matches the restart check on all but one field, so + // none of them may suppress a new restart for `svc-unhealthy`. + for (watcher_id, service_id, status) in [ + ("server-recover", "svc-unhealthy", "approved"), + ("server-recover", "svc-other", "pending"), + ("another-watcher", "svc-unhealthy", "pending"), + ] { + append_automation_proposal( + &paths, + &AutomationProposalRecord { + at_unix_ms: unix_time_millis(), + proposal_id: String::new(), + watcher_id: watcher_id.to_owned(), + action: "queue_restart_proposal".to_owned(), + title: "Restart".to_owned(), + message: "unrelated".to_owned(), + status: status.to_owned(), + service_id: Some(service_id.to_owned()), + tool: Some("restart_server".to_owned()), + arguments: json!({}), + reviewed_at_unix_ms: None, + }, + )?; + } + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "server-recover", + WatcherMode::Propose, + None, + )]); + let event = AutomationTriggerEvent { + at_unix_ms: 1, + kind: "service.manifest_recoverable".to_owned(), + source: "managed_service".to_owned(), + watcher_hint: Some("server-recover".to_owned()), + service_id: Some(record.service_id.clone()), + reason: Some("manifest_status_failed".to_owned()), + payload: json!({}), + }; + + handle_server_recover_event_with_record( + &paths, + WatcherMode::Propose, + &mut state, + &event, + &mut record, + )?; + let queued = rocm_core::load_automation_proposals(&paths)? + .into_iter() + .filter(|proposal| { + proposal.watcher_id == "server-recover" + && proposal.status == "pending" + && proposal.service_id.as_deref() == Some("svc-unhealthy") + }) + .count(); + fs::remove_dir_all(root).ok(); + + assert_eq!(queued, 1); + Ok(()) + } + + #[test] + fn server_recover_reports_an_unreadable_proposals_store_instead_of_failing() -> Result<()> { + let (root, paths) = temp_app_paths("server-recover-corrupt-store"); + paths.ensure()?; + fs::write(paths.automation_proposals_path(), "not a proposal\n")?; + let mut record = ManagedServiceRecord::new( + &paths, + "svc-unhealthy", + "vllm", + "tiny", + "Tiny/Test", + "127.0.0.1", + 11436, + "managed", + 123, + None, + None, + None, + ); + record.status = "unreachable".to_owned(); + record.write()?; + let mut state = test_runtime_state(vec![test_watcher_snapshot( + "server-recover", + WatcherMode::Propose, + None, + )]); + let event = AutomationTriggerEvent { + at_unix_ms: 1, + kind: "service.manifest_recoverable".to_owned(), + source: "managed_service".to_owned(), + watcher_hint: Some("server-recover".to_owned()), + service_id: Some(record.service_id.clone()), + reason: Some("manifest_status_failed".to_owned()), + payload: json!({}), + }; + + let outcome = handle_server_recover_event_with_record( + &paths, + WatcherMode::Propose, + &mut state, + &event, + &mut record, + ); + let events = rocm_core::load_recent_automation_events(&paths, 1)?; + let stored = fs::read_to_string(paths.automation_proposals_path())?; + fs::remove_dir_all(root).ok(); + + // An error out of a watcher ends the daemon loop, so a corrupt store + // must surface as an event and a skipped tick, not as `Err`. + outcome?; + assert_eq!(stored, "not a proposal\n", "nothing may be queued"); + assert!(events.iter().any(|event| { + event.level == "error" && event.action == "restart_proposal_check_failed" + })); + Ok(()) + } + + #[test] + fn recovery_reason_display_avoids_raw_status_tokens() { + assert_eq!( + display_recovery_reason("manifest_status_starting_stale"), + "service has been starting for too long" + ); + assert_eq!( + display_recovery_reason("healthcheck_status_unreachable"), + "engine healthcheck reports unreachable" + ); + assert_eq!( + display_recovery_reason("endpoint_status_unreachable"), + "endpoint port is unreachable" + ); + } + + #[test] + fn manifest_recovery_policy_covers_terminal_and_stale_transient_states() { + let (root, paths) = temp_app_paths("manifest-recovery-policy"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-stale", + "vllm", + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + 11435, + "managed", + 123, + None, + None, + None, + ); + record.created_at_unix_ms = 1_000; + + record.status = "exited".to_owned(); + assert_eq!( + manifest_service_recovery_reason(&record, 1_001).as_deref(), + Some("manifest_status_exited") + ); + + record.status = "unreachable".to_owned(); + assert_eq!( + manifest_service_recovery_reason(&record, 1_001).as_deref(), + Some("manifest_status_unreachable") + ); + + record.status = "starting".to_owned(); + assert_eq!(manifest_service_recovery_reason(&record, 2_000), None); + assert_eq!( + manifest_service_recovery_reason(&record, 1_000 + SERVER_TRANSIENT_STALE_MS).as_deref(), + Some("manifest_status_starting_stale") + ); + + record.status = "recovering".to_owned(); + record.last_restart_unix_ms = Some(5_000); + assert_eq!(manifest_service_recovery_reason(&record, 6_000), None); + assert_eq!( + manifest_service_recovery_reason(&record, 5_000 + SERVER_TRANSIENT_STALE_MS).as_deref(), + Some("manifest_status_recovering_stale") + ); + fs::remove_dir_all(root).ok(); + } + + #[test] + fn find_recoverable_service_prefers_failed_managed_manifest() -> Result<()> { + let (root, paths) = temp_app_paths("recoverable-service"); + paths.ensure()?; + let mut failed = ManagedServiceRecord::new( + &paths, + "svc-failed", + "vllm", + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + 11435, + "managed", + 123, + None, + None, + None, + ); + failed.status = "failed".to_owned(); + failed.write()?; + + let found = find_recoverable_service(&paths)?.expect("failed service should be found"); + fs::remove_dir_all(root).ok(); + assert_eq!(found.0.service_id, "svc-failed"); + assert_eq!(found.1, "manifest_status_failed"); + Ok(()) + } + + #[test] + fn find_recoverable_service_detects_stale_starting_manifest() -> Result<()> { + let (root, paths) = temp_app_paths("recoverable-stale-starting"); + paths.ensure()?; + let mut stale = ManagedServiceRecord::new( + &paths, + "svc-starting", + "vllm", + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + 11435, + "managed", + 123, + None, + None, + None, + ); + stale.status = "starting".to_owned(); + stale.created_at_unix_ms = 0; + stale.write()?; + + let found = + find_recoverable_service(&paths)?.expect("stale starting service should recover"); + fs::remove_dir_all(root).ok(); + assert_eq!(found.0.service_id, "svc-starting"); + assert_eq!(found.1, "manifest_status_starting_stale"); + Ok(()) + } + + #[test] + fn server_recover_propose_mode_queues_restart_proposal() -> Result<()> { + let (root, paths) = temp_app_paths("server-recover-proposal"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "svc-1", + "vllm", + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + 11435, + "managed", + 123, + None, + None, + None, + ); + record.status = "failed".to_owned(); + record.write()?; + let mut state = AutomationRuntimeState { + running: true, + automations_enabled: true, + daemon_pid: 1, + started_at_unix_ms: 1, + last_tick_unix_ms: 1, + local_webhook_endpoint: None, + active_watchers: vec![WatcherRuntimeSnapshot { + id: "server-recover".to_owned(), + enabled: true, + mode: WatcherMode::Propose, + summary: "recover".to_owned(), + last_event: None, + last_event_unix_ms: None, + }], + }; + + evaluate_server_recover(&paths, WatcherMode::Propose, &mut state)?; + let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; + fs::remove_dir_all(root).ok(); + + assert_eq!(proposals.len(), 1); + assert_eq!(proposals[0].watcher_id, "server-recover"); + assert_eq!(proposals[0].action, "queue_restart_proposal"); + assert_eq!(proposals[0].service_id.as_deref(), Some("svc-1")); + assert_eq!(proposals[0].status, "pending"); + Ok(()) + } + + /// An unconfirmed stop leaves the status at `ready` on purpose, which is + /// exactly what the daemon's recovery scan looks for. The stop request it + /// records is the only thing telling it apart from a crash, so recovery + /// must honour it — otherwise the `server-recover` watcher, which acts by + /// default, restarts the service the operator just asked to stop. + #[test] + fn recovery_does_not_restart_a_service_whose_stop_is_unconfirmed() -> Result<()> { + let (root, paths) = temp_app_paths("recover-skips-unconfirmed-stop"); + paths.ensure()?; + let service_id = "svc-recover-skips-unconfirmed-stop"; + seed_keyed_service(&paths, service_id, 11450)?; + + let stopped = + stop_with_outcome(&paths, service_id, rocm_core::TerminationOutcome::TimedOut); + let found = find_recoverable_service(&paths); + let reloaded = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + assert_eq!( + stopped?.get("stopped").and_then(Value::as_bool), + Some(false), + "precondition: the stop must be unconfirmed" + ); + let reloaded = reloaded?; + assert_eq!( + reloaded.status, "ready", + "precondition: status left as it was" + ); + assert!( + found?.is_none(), + "a service with a standing stop request must not be recovered" + ); + Ok(()) + } + + /// The same gate on a record recovery would otherwise pick up from its + /// status alone, without any endpoint probe: with a stop request it is + /// skipped, and the identical record without one is still recovered. + #[test] + fn recovery_skips_a_failed_service_only_while_its_stop_request_stands() -> Result<()> { + let (root, paths) = temp_app_paths("recover-honours-stop-request"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "svc-recover-honours-stop-request", + "vllm", + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + 11451, + "managed", + UNCONFIRMED_STOP_PID, + None, + None, + None, + ); + record.status = "failed".to_owned(); + record.stop_requested_unix_ms = Some(1); + record.write()?; + let event = AutomationTriggerEvent { + at_unix_ms: unix_time_millis(), + kind: "service.manifest_recoverable".to_owned(), + source: "managed_service".to_owned(), + watcher_hint: Some("server-recover".to_owned()), + service_id: Some(record.service_id.clone()), + reason: Some("manifest_status_failed".to_owned()), + payload: json!({}), + }; + let with_request = find_recoverable_service(&paths); + let event_with_request = service_record_matches_recovery_event(&paths, &record, &event); + + record.stop_requested_unix_ms = None; + record.write()?; + let without_request = find_recoverable_service(&paths); + let event_without_request = service_record_matches_recovery_event(&paths, &record, &event); + fs::remove_dir_all(root).ok(); + + assert!( + with_request?.is_none(), + "a standing stop request must block recovery" + ); + assert!( + !event_with_request, + "a queued recovery event must not act on a service since asked to stop" + ); + assert_eq!( + without_request?.map(|(found, reason)| (found.service_id, reason)), + Some(( + "svc-recover-honours-stop-request".to_owned(), + "manifest_status_failed".to_owned() + )), + "without a stop request the same record must still be recovered" + ); + assert!(event_without_request); + Ok(()) + } +} diff --git a/apps/rocmd/src/watchers/therock_update.rs b/apps/rocmd/src/watchers/therock_update.rs new file mode 100644 index 0000000000..e364c923d7 --- /dev/null +++ b/apps/rocmd/src/watchers/therock_update.rs @@ -0,0 +1,475 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The `therock-update` watcher: the scheduled update check, run read-only in +//! contained mode, and the notification it records when an update is available. + +use super::policy::{WatcherPolicyAction, queue_proposal, watcher_policy_action}; +use anyhow::{Context, Result, bail}; +use rocm_core::{AppPaths, AutomationRuntimeState, AutomationTriggerEvent, WatcherMode}; +use serde_json::Value; + +pub(super) const THEROCK_UPDATE_INTERVAL_MS: u128 = 6 * 60 * 60 * 1000; + +pub(super) fn handle_therock_update_event( + paths: &AppPaths, + mode: WatcherMode, + state: &mut AutomationRuntimeState, + event: &AutomationTriggerEvent, +) -> Result<()> { + handle_therock_update_event_with_runner(paths, mode, state, event, |paths| { + crate::sandbox::run_sandbox_tool( + paths, + crate::sandbox_tool::SandboxToolArg::CheckUpdates, + None, + None, + None, + crate::sandbox_tool::SandboxToolPolicy::default(), + ) + }) +} + +fn handle_therock_update_event_with_runner( + paths: &AppPaths, + mode: WatcherMode, + state: &mut AutomationRuntimeState, + _event: &AutomationTriggerEvent, + update_runner: F, +) -> Result<()> +where + F: FnOnce(&AppPaths) -> Result, +{ + let policy = watcher_policy_action("therock-update", mode); + let action = match policy { + WatcherPolicyAction::Observe => "observe_schedule", + WatcherPolicyAction::QueueProposal => "queue_update_proposal", + WatcherPolicyAction::RunContained => "run_update_check", + }; + let message = match policy { + WatcherPolicyAction::Observe => { + "scheduled TheRock update check reminder emitted; run `rocm update` to inspect the selected channel" + } + WatcherPolicyAction::QueueProposal => { + "scheduled TheRock update check reminder emitted; queueing read-only update-check proposal for review" + } + WatcherPolicyAction::RunContained => { + "scheduled TheRock update check is approved for contained read-only execution" + } + }; + match policy { + WatcherPolicyAction::Observe | WatcherPolicyAction::QueueProposal => { + crate::persistence::record_event( + paths, + state, + "therock-update", + "info", + action, + message, + None, + )?; + if policy == WatcherPolicyAction::QueueProposal { + queue_proposal( + paths, + "therock-update", + action, + "Check TheRock updates", + "Run `rocm update` to inspect available CLI, runtime, engine, and recipe updates before applying changes.", + None, + )?; + } + } + WatcherPolicyAction::RunContained => match update_runner(paths) { + Ok(output) => match restricted_check_updates_result(&output) { + Ok(result) => { + crate::persistence::record_event( + paths, + state, + "therock-update", + if result.exit_status == 0 { + "info" + } else { + "error" + }, + action, + &format!( + "{message}; restricted check_updates status={}; {}", + result.status, + crate::common::update_check_message(result.status) + ), + None, + )?; + if result.update_available { + record_update_available_notification(paths, state, result.status)?; + } + } + Err(error) => { + crate::persistence::record_event( + paths, + state, + "therock-update", + "error", + "update_check_failed", + &format!( + "scheduled TheRock update check failed during contained restricted execution: {error}; no updates were applied" + ), + None, + )?; + } + }, + Err(error) => { + crate::persistence::record_event( + paths, + state, + "therock-update", + "error", + "update_check_failed", + &format!( + "scheduled TheRock update check failed during contained read-only execution: {error}; no updates were applied" + ), + None, + )?; + } + }, + } + Ok(()) +} + +struct RestrictedCheckUpdatesResult<'a> { + status: &'a str, + update_available: bool, + exit_status: i64, +} + +fn restricted_check_updates_result(value: &Value) -> Result> { + let tool = value + .get("tool") + .and_then(Value::as_str) + .context("restricted update check did not report a tool name")?; + if tool != crate::sandbox_tool::SandboxToolArg::CheckUpdates.as_cli_value() { + bail!("restricted update check returned `{tool}`, expected `check_updates`"); + } + let status = value + .get("status") + .and_then(Value::as_str) + .unwrap_or("checked"); + let update_available = value + .get("update_available") + .and_then(Value::as_bool) + .unwrap_or(matches!(status, "update_available" | "repair_available")); + let exit_status = value + .get("exit_status") + .and_then(Value::as_i64) + .unwrap_or_else(|| i64::from(status == "error")); + Ok(RestrictedCheckUpdatesResult { + status, + update_available, + exit_status, + }) +} + +fn record_update_available_notification( + paths: &AppPaths, + state: &mut AutomationRuntimeState, + status: &str, +) -> Result<()> { + let message = if status == "repair_available" { + "A ROCm runtime repair is available because its package composition changed. Preview it before applying. No updates were applied." + } else { + "A ROCm runtime update is available. Preview it before applying. No updates were applied." + }; + crate::persistence::record_event( + paths, + state, + "therock-update", + "info", + "notify_if_newer", + message, + None, + )?; + crate::sandbox::record_notification_audit( + paths, + "watcher:therock-update", + "notify_if_newer", + Some("therock-update"), + message, + ) +} + +pub(super) fn therock_update_due(state: &AutomationRuntimeState, now: u128) -> bool { + let Some(snapshot) = state + .active_watchers + .iter() + .find(|watcher| watcher.id == "therock-update" && watcher.enabled) + else { + return false; + }; + snapshot + .last_event_unix_ms + .is_none_or(|last_event| now.saturating_sub(last_event) >= THEROCK_UPDATE_INTERVAL_MS) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::temp_app_paths; + use rocm_core::{AuditEventRecord, AutomationEventRecord, WatcherRuntimeSnapshot}; + use serde_json::json; + use std::fs; + + #[test] + fn therock_update_contained_mode_runs_read_only_check_without_queueing() -> Result<()> { + let (root, paths) = temp_app_paths("therock-update-contained"); + paths.ensure()?; + let mut state = AutomationRuntimeState { + running: true, + automations_enabled: true, + daemon_pid: 1, + started_at_unix_ms: 1, + last_tick_unix_ms: 1, + local_webhook_endpoint: None, + active_watchers: vec![WatcherRuntimeSnapshot { + id: "therock-update".to_owned(), + enabled: true, + mode: WatcherMode::Contained, + summary: "check updates".to_owned(), + last_event: None, + last_event_unix_ms: None, + }], + }; + let event = AutomationTriggerEvent { + at_unix_ms: 42, + kind: "schedule.tick".to_owned(), + source: "scheduler".to_owned(), + watcher_hint: Some("therock-update".to_owned()), + service_id: None, + reason: Some("therock_update_interval_due".to_owned()), + payload: json!({ "interval_ms": THEROCK_UPDATE_INTERVAL_MS }), + }; + + handle_therock_update_event_with_runner( + &paths, + WatcherMode::Contained, + &mut state, + &event, + |_paths| { + Ok(crate::sandbox::sandbox_check_updates_value( + rocm_core::CommandCapture { + argv: vec!["rocm".to_owned(), "update".to_owned()], + exit_status: 0, + stdout: "update\n managed runtimes: none\n".to_owned(), + stderr: String::new(), + }, + )) + }, + )?; + + let event_text = fs::read_to_string(paths.automation_events_path())?; + let event = serde_json::from_str::(event_text.trim())?; + let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; + fs::remove_dir_all(root).ok(); + + assert_eq!(event.watcher_id, "therock-update"); + assert_eq!(event.action, "run_update_check"); + assert!(event.message.contains("contained read-only execution")); + assert!( + event + .message + .contains("restricted check_updates status=checked") + ); + assert!(event.message.contains("no updates were applied")); + assert!(!event.message.contains("fallback")); + assert!(proposals.is_empty()); + Ok(()) + } + + #[test] + fn therock_update_contained_mode_records_update_available_without_applying() -> Result<()> { + let (root, paths) = temp_app_paths("therock-update-contained-available"); + paths.ensure()?; + let mut state = AutomationRuntimeState { + running: true, + automations_enabled: true, + daemon_pid: 1, + started_at_unix_ms: 1, + last_tick_unix_ms: 1, + local_webhook_endpoint: None, + active_watchers: vec![WatcherRuntimeSnapshot { + id: "therock-update".to_owned(), + enabled: true, + mode: WatcherMode::Contained, + summary: "check updates".to_owned(), + last_event: None, + last_event_unix_ms: None, + }], + }; + let event = AutomationTriggerEvent { + at_unix_ms: 42, + kind: "schedule.tick".to_owned(), + source: "scheduler".to_owned(), + watcher_hint: Some("therock-update".to_owned()), + service_id: None, + reason: Some("therock_update_interval_due".to_owned()), + payload: json!({ "interval_ms": THEROCK_UPDATE_INTERVAL_MS }), + }; + + handle_therock_update_event_with_runner( + &paths, + WatcherMode::Contained, + &mut state, + &event, + |_paths| { + Ok(crate::sandbox::sandbox_check_updates_value(rocm_core::CommandCapture { + argv: vec!["rocm".to_owned(), "update".to_owned()], + exit_status: 0, + stdout: "update\n runtime release-pip-gfx120x-all status=update_available installed=7.13.0 latest=7.14.0\n".to_owned(), + stderr: String::new(), + })) + }, + )?; + + let event_text = fs::read_to_string(paths.automation_events_path())?; + let events = event_text + .lines() + .map(serde_json::from_str::) + .collect::, _>>()?; + let audit_text = fs::read_to_string(paths.audit_events_path())?; + let proposals = rocm_core::load_recent_automation_proposals(&paths, 1)?; + fs::remove_dir_all(root).ok(); + + let update_check = events + .iter() + .find(|event| event.action == "run_update_check") + .expect("update check event should be recorded"); + assert_eq!(update_check.watcher_id, "therock-update"); + assert!( + update_check + .message + .contains("restricted check_updates status=update_available") + ); + assert!( + update_check + .message + .contains("a ROCm runtime update is available") + ); + assert!(update_check.message.contains("no updates were applied")); + assert!(!update_check.message.contains("fallback")); + let notification = events + .iter() + .find(|event| event.action == "notify_if_newer") + .expect("notify-if-newer event should be recorded"); + assert_eq!(notification.watcher_id, "therock-update"); + assert!( + notification + .message + .contains("ROCm runtime update is available") + ); + assert!(notification.message.contains("No updates were applied")); + assert!(audit_text.contains("\"category\":\"notification\"")); + assert!(audit_text.contains("\"action\":\"notify_if_newer\"")); + assert!(audit_text.contains("ROCm runtime update is available")); + assert!(proposals.is_empty()); + Ok(()) + } + + #[test] + fn therock_update_contained_mode_uses_restricted_check_updates_tool() -> Result<()> { + let (root, paths) = temp_app_paths("therock-update-contained-tool"); + paths.ensure()?; + let mut state = AutomationRuntimeState { + running: true, + automations_enabled: true, + daemon_pid: 1, + started_at_unix_ms: 1, + last_tick_unix_ms: 1, + local_webhook_endpoint: None, + active_watchers: vec![WatcherRuntimeSnapshot { + id: "therock-update".to_owned(), + enabled: true, + mode: WatcherMode::Contained, + summary: "check updates".to_owned(), + last_event: None, + last_event_unix_ms: None, + }], + }; + let event = AutomationTriggerEvent { + at_unix_ms: 42, + kind: "schedule.tick".to_owned(), + source: "scheduler".to_owned(), + watcher_hint: Some("therock-update".to_owned()), + service_id: None, + reason: Some("therock_update_interval_due".to_owned()), + payload: json!({ "interval_ms": THEROCK_UPDATE_INTERVAL_MS }), + }; + + handle_therock_update_event_with_runner( + &paths, + WatcherMode::Contained, + &mut state, + &event, + |_paths| { + Ok(json!({ + "tool": "examine_snapshot", + "status": "captured", + "mutating": false, + })) + }, + )?; + + let event_text = fs::read_to_string(paths.automation_events_path())?; + let event = serde_json::from_str::(event_text.trim())?; + fs::remove_dir_all(root).ok(); + + assert_eq!(event.watcher_id, "therock-update"); + assert_eq!(event.action, "update_check_failed"); + assert!(event.message.contains("expected `check_updates`")); + assert!(event.message.contains("no updates were applied")); + Ok(()) + } + + #[test] + fn therock_update_notify_if_newer_uses_restricted_notification_contract() -> Result<()> { + let (root, paths) = temp_app_paths("therock-update-notify-contract"); + paths.ensure()?; + let mut state = AutomationRuntimeState { + running: true, + automations_enabled: true, + daemon_pid: 1, + started_at_unix_ms: 1, + last_tick_unix_ms: 1, + local_webhook_endpoint: None, + active_watchers: vec![WatcherRuntimeSnapshot { + id: "therock-update".to_owned(), + enabled: true, + mode: WatcherMode::Contained, + summary: "check updates".to_owned(), + last_event: None, + last_event_unix_ms: None, + }], + }; + + record_update_available_notification(&paths, &mut state, "update_available")?; + + let audit_text = fs::read_to_string(paths.audit_events_path())?; + let audit = audit_text + .lines() + .map(serde_json::from_str::) + .collect::, _>>()?; + fs::remove_dir_all(root).ok(); + + let notification = audit + .iter() + .find(|event| event.category == "notification" && event.action == "notify_if_newer") + .expect("notify_if_newer audit should be recorded"); + assert_eq!(notification.category, "notification"); + assert_eq!(notification.actor, "watcher:therock-update"); + assert_eq!(notification.watcher_id.as_deref(), Some("therock-update")); + assert!( + notification + .message + .contains("ROCm runtime update is available") + ); + assert!(notification.message.contains("No updates were applied")); + Ok(()) + } +} diff --git a/docs/architecture.md b/docs/architecture.md index 4dd3148e6d..5a78ebb2db 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -117,17 +117,24 @@ Further `apps/rocm` modules, not part of the `main.rs` split: Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): - [`cli.rs`](../apps/rocmd/src/cli.rs) — the `Cli`/`Command` clap definitions, the bridge-snapshot diagnostic printer, the pretty-JSON printer for the subcommands that print a result (`print_json`), and the top-level dispatch in `run_cli`/`run_bin_cli`/`run_from_args`. -- [`common.rs`](../apps/rocmd/src/common.rs) — helpers the other modules call into: the read-only `rocm update` message, the `rocm`-subprocess capture wrapper (`run_rocm_capture_for_paths`) the sandbox and MCP clusters both call (it only chooses the child's `ROCM_CLI_*_DIR` environment; `CommandCapture` and the spawn itself live in [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), shared with `rocm`'s chat tool bridge, and the bounded-execution half in [`process.rs`](../crates/rocm-core/src/process.rs)), and small arg/JSON-payload utilities (`payload_string`, for example, is read by both [`watchers.rs`](../apps/rocmd/src/watchers.rs) and [`webhook.rs`](../apps/rocmd/src/webhook.rs)). +- [`common.rs`](../apps/rocmd/src/common.rs) — helpers the other modules call into: the read-only `rocm update` message, the `rocm`-subprocess capture wrapper (`run_rocm_capture_for_paths`) the sandbox and MCP clusters both call (it only chooses the child's `ROCM_CLI_*_DIR` environment; `CommandCapture` and the spawn itself live in [`command_capture.rs`](../crates/rocm-core/src/command_capture.rs), shared with `rocm`'s chat tool bridge, and the bounded-execution half in [`process.rs`](../crates/rocm-core/src/process.rs)), and small arg/JSON-payload utilities (`payload_string`, for example, is read by the [`watchers/cache_warm.rs`](../apps/rocmd/src/watchers/cache_warm.rs), [`watchers/driver_upgrade.rs`](../apps/rocmd/src/watchers/driver_upgrade.rs) and [`watchers/gpu_thermal_protect.rs`](../apps/rocmd/src/watchers/gpu_thermal_protect.rs) handlers and by [`webhook.rs`](../apps/rocmd/src/webhook.rs)). - [`daemon.rs`](../apps/rocmd/src/daemon.rs) — the foreground `rocmd run` loop (`run_daemon`: evaluate the watchers each tick, dispatch local webhook events, write the runtime state, stop on a signal; with no watcher enabled it exits by itself, deciding that under the autostart lock and publishing "stopped" and clearing its spawn claim before releasing it — `record_daemon_stop`, `release_own_autostart_claim`) and `rocmd status` (`print_status`), with the watcher tick interval, which `LoopHooks` makes injectable for tests. It uses [`watchers.rs`](../apps/rocmd/src/watchers.rs), [`webhook.rs`](../apps/rocmd/src/webhook.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs); [`cli.rs`](../apps/rocmd/src/cli.rs) calls it. -- [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) — talking to an engine and guarding its endpoint key: `engine_request` (the stdio request to an engine process), the healthcheck call and its readiness and recoverability checks, and the endpoint-key helpers `apply_endpoint_key_env` and `ensure_public_service_has_endpoint_key`. It uses no other module of the crate; [`service.rs`](../apps/rocmd/src/service.rs), [`supervise.rs`](../apps/rocmd/src/supervise.rs) and [`watchers.rs`](../apps/rocmd/src/watchers.rs) call it. +- [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) — talking to an engine and guarding its endpoint key: `engine_request` (the stdio request to an engine process), the healthcheck call and its readiness and recoverability checks, and the endpoint-key helpers `apply_endpoint_key_env` and `ensure_public_service_has_endpoint_key`. It uses no other module of the crate; [`service.rs`](../apps/rocmd/src/service.rs), [`supervise.rs`](../apps/rocmd/src/supervise.rs) and [`watchers/server_recover.rs`](../apps/rocmd/src/watchers/server_recover.rs) call it. - [`host_snapshot.rs`](../apps/rocmd/src/host_snapshot.rs) — the host snapshot: `build_bridge_snapshot` (the bridge snapshot printed by `rocmd bridge-snapshot` and returned by the `bridge_snapshot` MCP tool, assembled from the examine summary, the GPU snapshot, the automation runtime state and recent events, the engine inventory and the managed services), the amd-smi GPU snapshot (`gather_gpu_snapshot_for_config`, which honours the telemetry setting, over time-bounded amd-smi probes) and `bridge_engine_inventory`. It uses only [`persistence.rs`](../apps/rocmd/src/persistence.rs); [`cli.rs`](../apps/rocmd/src/cli.rs), [`mcp.rs`](../apps/rocmd/src/mcp.rs) and [`watchers.rs`](../apps/rocmd/src/watchers.rs) call it. - [`mcp.rs`](../apps/rocmd/src/mcp.rs) — the MCP stdio server, tool schema table, tool dispatch, and the argv-building and result-shaping helpers behind the MCP tools (the `rocm` subprocess itself is spawned through [`common.rs`](../apps/rocmd/src/common.rs), via `run_rocm_capture` here). - [`persistence.rs`](../apps/rocmd/src/persistence.rs) — `record_event`/`load_managed_services`/`load_service_record`, the automation-event/audit-log and managed-service-registry I/O shared across the daemon's sandbox, MCP, service-lifecycle, and watcher code. - [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) — bubblewrap/native sandbox execution, atomic-write helpers, artifact prefetch policy gating, and the sandbox-tool result shaping for `check_updates`/`driver_plan`/`stop_server`. - [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs) — the sandbox-tool vocabulary: the `SandboxToolArg` value enum naming each sandbox tool, and `SandboxToolPolicy`, the artifact-download policy built from the `--allow-*` flags (including the Hugging Face token read from the environment when authenticated downloads are allowed). It uses no other module of the crate, so modules that need the vocabulary do not import [`cli.rs`](../apps/rocmd/src/cli.rs). - [`service.rs`](../apps/rocmd/src/service.rs) — managed-service stop: terminating the recorded processes of a service after verifying their identity, and the stop report (`stop_managed_service`, `stop_report_confirmed`) that the sandbox and MCP tools read. -- [`supervise.rs`](../apps/rocmd/src/supervise.rs) — launching and re-launching a managed service's supervisor: `supervise_service` (the `rocmd supervise` entry point: launch the engine child, record its identity, wait for readiness, record the exit) with the serve-log startup-phase polling and readiness wait it uses, and `restart_managed_service` (the recovery respawn: re-launch `rocmd supervise` detached for a service and record the new supervisor). Writes after a supervisor's first go through the private `update_supervised_record`, which re-reads the record and leaves a stop marker or another supervisor's record alone. Its production code uses only [`common.rs`](../apps/rocmd/src/common.rs), [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs). [`cli.rs`](../apps/rocmd/src/cli.rs) calls `supervise_service`; [`watchers.rs`](../apps/rocmd/src/watchers.rs) (the server-recover handler) and [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) (the `restart_server` tool) call `restart_managed_service`. -- [`watchers.rs`](../apps/rocmd/src/watchers.rs) — event collection/dispatch for all built-in watchers (TheRock update, GPU metrics/thermal-pressure, cache-warm, driver-upgrade, server-recover), plus managed-service recovery classification and automation-proposal queuing. +- [`supervise.rs`](../apps/rocmd/src/supervise.rs) — launching and re-launching a managed service's supervisor: `supervise_service` (the `rocmd supervise` entry point: launch the engine child, record its identity, wait for readiness, record the exit) with the serve-log startup-phase polling and readiness wait it uses, and `restart_managed_service` (the recovery respawn: re-launch `rocmd supervise` detached for a service and record the new supervisor). Writes after a supervisor's first go through the private `update_supervised_record`, which re-reads the record and leaves a stop marker or another supervisor's record alone. Its production code uses only [`common.rs`](../apps/rocmd/src/common.rs), [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs). [`cli.rs`](../apps/rocmd/src/cli.rs) calls `supervise_service`; [`watchers/server_recover.rs`](../apps/rocmd/src/watchers/server_recover.rs) (the server-recover handler) and [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) (the `restart_server` tool) call `restart_managed_service`. +- [`watchers.rs`](../apps/rocmd/src/watchers.rs) — the module root of the built-in watchers: the registry (`reconcile_watcher_snapshots`) and the event collector and dispatcher (`evaluate_watchers`, `collect_automation_events_with_gpu_snapshot`, which takes the GPU snapshot as a closure so tests can inject one, and `evaluate_watchers_for_events`), which route each event to its watcher's handler. `find_recoverable_service` and `stop_requested` are re-exported from here because [`supervise.rs`](../apps/rocmd/src/supervise.rs)'s and [`service.rs`](../apps/rocmd/src/service.rs)'s tests reach them by the `crate::watchers::` path. Each watcher's handler lives in the child below that carries its name, together with the policy they share, and a child reaches only [`policy.rs`](../apps/rocmd/src/watchers/policy.rs) and, for the thermal-protect watcher, [`gpu_metrics.rs`](../apps/rocmd/src/watchers/gpu_metrics.rs) among the others. + - [`watchers/policy.rs`](../apps/rocmd/src/watchers/policy.rs) — what a watcher does in each mode and the proposals it queues: `WatcherPolicyAction` and `watcher_policy_action` (observe, queue a proposal or run contained, chosen by the watcher mode), `queue_proposal` and `queue_proposal_with_arguments` with the action-to-tool and action-to-arguments tables behind them, and `pending_proposal_exists`, the check the thermal-protect and server-recover handlers use so they do not queue a duplicate of a proposal that is still pending. It uses no other watcher module. + - [`watchers/server_recover.rs`](../apps/rocmd/src/watchers/server_recover.rs) — the `server-recover` watcher: finding a managed service that needs recovery (`find_recoverable_service`, from a failed or stale manifest, an unreachable endpoint or a failing engine healthcheck, skipping a service whose stop request still stands via `stop_requested`), the recovery event kinds and plain-language reasons, the backoff that spaces recovery attempts (`server_recover_due`), and the handler that restarts the service or queues a restart proposal by watcher mode. It uses [`policy.rs`](../apps/rocmd/src/watchers/policy.rs), [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs), [`persistence.rs`](../apps/rocmd/src/persistence.rs) and [`supervise.rs`](../apps/rocmd/src/supervise.rs)'s `restart_managed_service`. + - [`watchers/gpu_metrics.rs`](../apps/rocmd/src/watchers/gpu_metrics.rs) — the `gpu-metrics` watcher and the GPU readings it works from: the telemetry-event handler (`handle_gpu_metrics_event`, which records telemetry and takes no action), the interval check (`gpu_metrics_due`), the one-line snapshot summary, and the reading of amd-smi monitor entries into thermal-pressure and memory-pressure events (`gpu_pressure_events`, `gpu_pressure_event`) with the thresholds they use. It uses [`persistence.rs`](../apps/rocmd/src/persistence.rs) and no other watcher module, and [`gpu_thermal_protect.rs`](../apps/rocmd/src/watchers/gpu_thermal_protect.rs) reads its interval and thresholds. + - [`watchers/gpu_thermal_protect.rs`](../apps/rocmd/src/watchers/gpu_thermal_protect.rs) — the `gpu-thermal-protect` watcher: the pressure-event handler (`handle_gpu_thermal_protect_event`), which records the event and, outside observe mode, picks the one running managed service as the target and queues a reviewed stop request unless one is already pending, and the interval check (`gpu_thermal_protect_due`). It uses [`gpu_metrics.rs`](../apps/rocmd/src/watchers/gpu_metrics.rs) for the interval and thresholds, [`policy.rs`](../apps/rocmd/src/watchers/policy.rs), [`common.rs`](../apps/rocmd/src/common.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs). + - [`watchers/therock_update.rs`](../apps/rocmd/src/watchers/therock_update.rs) — the `therock-update` watcher: the schedule check (`therock_update_due`, every six hours) and the handler, which in observe mode records a reminder, in propose mode also queues an update-check proposal, and in contained mode runs the read-only `check_updates` sandbox tool and records a notification when an update or repair is available, never applying one. It uses [`policy.rs`](../apps/rocmd/src/watchers/policy.rs), [`sandbox.rs`](../apps/rocmd/src/sandbox.rs), [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs), [`common.rs`](../apps/rocmd/src/common.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs). + - [`watchers/driver_upgrade.rs`](../apps/rocmd/src/watchers/driver_upgrade.rs) — the `driver-upgrade` watcher: the handler for an available driver update, which ignores non-driver components, records the signal in observe mode, queues a reviewed driver-plan proposal in propose mode, and in contained mode runs the restricted read-only `driver_plan` sandbox tool, executing no driver command. It uses [`policy.rs`](../apps/rocmd/src/watchers/policy.rs), [`sandbox.rs`](../apps/rocmd/src/sandbox.rs), [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs), [`common.rs`](../apps/rocmd/src/common.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs); its tests also use [`webhook.rs`](../apps/rocmd/src/webhook.rs)'s `local_webhook_event_from_request`. + - [`watchers/cache_warm.rs`](../apps/rocmd/src/watchers/cache_warm.rs) — the `cache-warm` watcher: the handler for a request to warm a recipe artifact. It checks the artifact against the model registry (the lookup is injectable for tests), records an unknown or unverifiable artifact without queueing anything, and otherwise records the request and, outside observe mode, queues a reviewed prefetch proposal; contained mode does too, because an artifact download needs explicit source-policy approval. It uses [`policy.rs`](../apps/rocmd/src/watchers/policy.rs), [`common.rs`](../apps/rocmd/src/common.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs); its tests also use [`webhook.rs`](../apps/rocmd/src/webhook.rs)'s `local_webhook_event_from_request`. - [`webhook.rs`](../apps/rocmd/src/webhook.rs) — the local webhook source: its axum routes, request validation, and watcher-kind allow-list. [`lib.rs`](../apps/rocmd/src/lib.rs) itself is now top-level glue: module declarations and the three externally-consumed entry points re-exported from [`cli.rs`](../apps/rocmd/src/cli.rs) (`run_bin_cli`/`run_from_args`, plus `parse_args`, which `apps/rocm`'s test-only [`advised_commands.rs`](../apps/rocm/src/advised_commands.rs) uses to check advised `rocmd` commands).