From fe59db3bdd59abe1db17cfb9fbff358c3878192e Mon Sep 17 00:00:00 2001 From: fredespi Date: Wed, 28 Oct 2026 11:01:30 +0100 Subject: [PATCH 1/4] fix(uninstall): stop managed services before removing the tooling that would stop them rocm uninstall reported success while a managed model server, possibly bound publicly and holding the GPU, kept running, and then deleted the binaries and service records needed to stop it. Uninstall now stops the managed services and the rocmd helper first, and refuses to remove anything when it cannot confirm a stop: a process whose identity it cannot verify, an endpoint that still answers, or a record it cannot parse. The refusal names each recovery step, including the relaunch command for every service it did stop, since stopping drops its endpoint key. Covers EAI-8014. Signed-off-by: fredespi --- Cargo.lock | 2 - README.md | 19 + apps/rocm/Cargo.toml | 1 - apps/rocm/src/chat.rs | 2 +- apps/rocm/src/main.rs | 4454 ++++++++++++++++- apps/rocm/src/service_launch.rs | 49 +- apps/rocm/src/test_support.rs | 17 + apps/rocm/src/therock.rs | 68 +- apps/rocm/src/uninstall.rs | 1094 +++- apps/rocm/src/uninstall_gate.rs | 1387 +++++ apps/rocm/src/uninstall_plan.rs | 133 +- apps/rocmd/Cargo.toml | 3 - apps/rocmd/src/common.rs | 35 +- apps/rocmd/src/persistence.rs | 1 + apps/rocmd/src/sandbox.rs | 64 +- apps/rocmd/src/service.rs | 32 +- apps/rocmd/src/watchers.rs | 6 + crates/rocm-core/src/automation.rs | 53 +- crates/rocm-core/src/config.rs | 4 +- crates/rocm-core/src/lib.rs | 636 ++- crates/rocm-core/src/net.rs | 250 +- crates/rocm-core/src/proc_lifecycle.rs | 230 +- crates/rocm-core/src/service_record.rs | 26 +- crates/rocm-engine-protocol/src/lib.rs | 129 + docs/architecture.md | 5 +- docs/manual-testing.md | 32 + docs/testing.md | 8 +- engines/lemonade/src/backend_alignment.rs | 7 +- engines/lemonade/src/install.rs | 6 +- engines/lemonade/src/state.rs | 8 +- engines/vllm/src/state.rs | 11 +- .../features/install_lifecycle.feature | 158 +- .../e2e-cucumber/tests/e2e/lifecycle_steps.rs | 464 +- 33 files changed, 8885 insertions(+), 509 deletions(-) create mode 100644 apps/rocm/src/uninstall_gate.rs diff --git a/Cargo.lock b/Cargo.lock index bcbd0022c1..ce3da5f602 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3691,7 +3691,6 @@ dependencies = [ "unicode-width", "ureq", "windows-native-keyring-store", - "windows-sys 0.61.2", "zbus-secret-service-keyring-store", ] @@ -3880,7 +3879,6 @@ dependencies = [ "sha2", "tokio", "ureq", - "windows-sys 0.61.2", ] [[package]] diff --git a/README.md b/README.md index 255f985977..0042e260f4 100644 --- a/README.md +++ b/README.md @@ -1223,6 +1223,25 @@ be, nothing is removed, so you can fix the cause and run `rocm uninstall` again. Each step is reported as it finishes, so if a later step fails, the output still shows what was already removed. +`rocm uninstall` stops any managed model server that is still running before it +removes anything, and also stops the background helper (`rocmd`), which is +force-stopped (on Linux its whole process tree, on Windows only the recorded +process) because it would otherwise restart the servers. +If a server cannot be stopped (or its service record cannot be read), the +command exits non-zero, leaves every file in place, and says what to repair or +stop by hand, so the tooling needed to stop it is never deleted out from under +a running server. A live server or helper recorded without a start time (written +by an older version) cannot be proven to be ours, so it is refused rather than +killed: stop it yourself, then re-run. The same refusal applies when the +helper's runtime state file cannot be read. The plan shows the refusal before +you confirm it. The stop pass is not atomic: if it aborts after stopping some +servers (or the helper), those stay stopped and their endpoint keys are gone; +the error names each one and the `rocm serve` command that brings it back. A +run that keeps the binaries and data (`--keep-binaries --keep-data`) leaves +running servers and the helper alone. +If a removal fails partway, the error lists what was removed and what is left, +because the binaries and service records may already be gone. + On Linux, if a config, data, or cache folder is a link to somewhere else (for example a cache moved to a bigger disk), `rocm uninstall` removes the link and leaves the folder it points to alone, whether or not the setting ends in a `/`. diff --git a/apps/rocm/Cargo.toml b/apps/rocm/Cargo.toml index 5db1f4e8d3..e46055e393 100644 --- a/apps/rocm/Cargo.toml +++ b/apps/rocm/Cargo.toml @@ -58,7 +58,6 @@ libc.workspace = true [target.'cfg(windows)'.dependencies] self-replace = "1.5" windows-native-keyring-store = "1.1" -windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } [target.'cfg(target_os = "macos")'.dependencies] apple-native-keyring-store = { version = "1.0", features = ["keychain"] } diff --git a/apps/rocm/src/chat.rs b/apps/rocm/src/chat.rs index ed6a054a98..237c52348e 100644 --- a/apps/rocm/src/chat.rs +++ b/apps/rocm/src/chat.rs @@ -195,7 +195,7 @@ fn chat_tool_call_display_label(call: &providers::ChatToolCall) -> String { } } -fn loopback_tcp_port_is_reachable(host: &str, port: u16) -> bool { +pub(crate) fn loopback_tcp_port_is_reachable(host: &str, port: u16) -> bool { let Ok(addresses) = (host, port).to_socket_addrs() else { return false; }; diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 39e696c8a0..bf46cc62ab 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -44,6 +44,7 @@ mod storage; mod test_support; mod therock; mod uninstall; +mod uninstall_gate; mod uninstall_plan; // Per-command handlers mechanically relocated into modules — fns, plus the @@ -684,6 +685,21 @@ rocm logs --search error timeout")] /// PATH change the installer made (its marked shell-profile block, or the /// user PATH entry it recorded adding on Windows), unless other programs /// share the install folder. + /// + /// Managed model servers that are still running are stopped first, and so + /// is the background helper (`rocmd`), which is force-stopped (on Linux its + /// whole process tree; on Windows only the recorded process, as + /// descendants are not enumerated there) because it would otherwise + /// restart them. If one cannot be + /// stopped, nothing is removed, the command exits non-zero, and it names + /// what to repair or stop by hand. A server or helper that was recorded by + /// an older version, with no start time to prove its pid, is refused rather + /// than killed: stop it yourself and re-run, as when the helper's runtime + /// state cannot be read. If the stop pass aborts after stopping some + /// services (or the helper), those stay stopped with their keys dropped; + /// the error lists the commands that bring them back. A run that keeps the + /// binaries and data (`--keep-binaries --keep-data`) leaves running servers + /// alone. Uninstall { /// Do not ask for interactive confirmation. #[arg(long)] @@ -4920,6 +4936,19 @@ pub(crate) fn render_launch_summary(paths: &AppPaths, config: &RocmCliConfig) -> output } +/// Whether `host` resolves to any address at all. +/// +/// Split out from [`loopback_tcp_port_is_reachable`], which answers `false` both +/// for "resolved, nothing listening" and for "did not resolve". Those mean +/// opposite things to the uninstall gate, and only the caller that has to +/// distinguish them pays for the second lookup. +fn host_port_resolves(host: &str, port: u16) -> bool { + use std::net::ToSocketAddrs as _; + (host, port) + .to_socket_addrs() + .is_ok_and(|mut addresses| addresses.next().is_some()) +} + pub(crate) fn render_examine_text() -> Result { let paths = AppPaths::discover()?; let config = RocmCliConfig::load(&paths).unwrap_or_default(); @@ -6856,24 +6885,8 @@ const MANAGED_STOP_GRACE: Duration = Duration::from_secs(10); /// known token. Returns the PIDs actually signalled and whether every recorded /// process is confirmed no longer running. fn terminate_recorded_service_pids(record: &ManagedServiceRecord) -> (Vec, bool) { - // Build the (pid, own-token) work list, de-duplicating on PID and preferring - // an entry that carries a verifiable start-time. - let mut entries: Vec<(u32, Option)> = Vec::new(); - for (pid, ticks) in [ - (Some(record.supervisor_pid), record.supervisor_start_ticks), - (record.engine_pid, record.engine_start_ticks), - ] { - let Some(pid) = pid.filter(|pid| *pid != 0 && *pid != std::process::id()) else { - continue; - }; - if let Some(existing) = entries.iter_mut().find(|(seen, _)| *seen == pid) { - if existing.1.is_none() { - existing.1 = ticks; - } - } else { - entries.push((pid, ticks)); - } - } + // The same (pid, own-token) list the uninstall gate pre-checks. + let entries = uninstall_gate::recorded_pid_identities(record); let mut signaled_pids = Vec::new(); let mut all_stopped = true; @@ -6910,6 +6923,17 @@ fn stop_internal_managed_service(paths: &AppPaths, service_id: &str) -> Result Result<(serde_json::Value, bool)> { + stop_internal_managed_service_confirmed_with(paths, service_id, terminate_recorded_service_pids) +} + /// [`stop_internal_managed_service`], with the termination of the recorded /// processes supplied by the caller. Production passes /// [`terminate_recorded_service_pids`]; a test passes a stand-in that observes @@ -9721,6 +9745,11 @@ pub(crate) mod tests { } use super::*; + use crate::service_launch::tests::runtime_state; + use crate::uninstall_gate::*; + #[cfg(unix)] + use crate::uninstall_plan::build_uninstall_plan; + use crate::uninstall_plan::{UninstallPlan, UninstallPlanEntry, plan_removes_recovery_tooling}; use serde_json::json; use crate::runtime_install_finalize::tests::finish_sdk_install_with; @@ -10207,7 +10236,7 @@ pub(crate) mod tests { // "guarded" arm wraps the write; the "unguarded" arm does not. if let Some(mode) = std::env::var_os("ROCM_TEST_SIGPIPE_ARM") { reset_sigpipe(); - let mut grandchild = ProcessCommand::new("true") + let mut grandchild = crate::test_support::bare_command("true") .stdin(Stdio::piped()) .stdout(Stdio::null()) .stderr(Stdio::null()) @@ -15791,6 +15820,7 @@ pub(crate) mod tests { running: true, automations_enabled: true, daemon_pid: 123, + daemon_start_ticks: None, started_at_unix_ms: 1, last_tick_unix_ms: 2, local_webhook_endpoint: Some("http://127.0.0.1:19191/automation-events".to_owned()), @@ -16963,20 +16993,66 @@ pub(crate) mod tests { ); } + /// Assert `child` exits within a bounded grace, and reap it either way. + /// + /// NOT `wait()`. These tests spawn a stand-in that exits on its own after a + /// minute, so an unbounded `wait()` blocks until that happens and then finds + /// the process gone — passing whether or not anything killed it. The only + /// symptom of a total regression would be the suite taking a minute. Polling + /// against a deadline asserts it died *now*, which is the actual claim. + /// + /// The grace exceeds the stop path's own per-process wait but stays far + /// below the stand-in's lifetime, or this goes back to measuring nothing. + /// + /// Shared with `crate::uninstall`'s tests rather than duplicated there: two + /// near-identical copies of this, with near-identical comments, is what made + /// it easy to believe both process-based tests were bounded when only one + /// of them was. + #[cfg(unix)] + pub(crate) const STOP_ASSERTION_GRACE: Duration = Duration::from_secs(30); + + #[cfg(unix)] + pub(crate) fn assert_stopped_within_grace(child: &mut std::process::Child, message: &str) { + let deadline = std::time::Instant::now() + STOP_ASSERTION_GRACE; + let exited = loop { + match child.try_wait().expect("poll the stand-in") { + Some(_) => break true, + None if std::time::Instant::now() >= deadline => break false, + None => thread::sleep(Duration::from_millis(50)), + } + }; + if !exited { + let _ = child.kill(); + let _ = child.wait(); + } + assert!(exited, "{message}"); + } + #[cfg(unix)] fn managed_record_for_pid( paths: &AppPaths, pid: u32, start_ticks: Option, + ) -> ManagedServiceRecord { + managed_record_named(paths, "svc-managed-stop", pid, start_ticks) + } + + #[cfg(unix)] + fn managed_record_named( + paths: &AppPaths, + service_id: &str, + pid: u32, + start_ticks: Option, ) -> ManagedServiceRecord { let mut record = ManagedServiceRecord::new( paths, - "svc-managed-stop", + service_id, "vllm", "m", "m", "127.0.0.1", - 9, + // Port 0 is never served, so the gate's probe cannot meet a listener. + 0, "managed", pid, None, @@ -16994,7 +17070,7 @@ pub(crate) mod tests { fn managed_stop_refuses_recycled_pid() { // A live process whose recorded identity does not match: this is exactly // what a recycled PID looks like. The stop must NOT signal it. - let mut child = std::process::Command::new("sleep") + let mut child = crate::test_support::bare_command("sleep") .arg("60") .spawn() .expect("spawn child"); @@ -17022,11 +17098,11 @@ pub(crate) mod tests { fn managed_stop_verifies_distinct_supervisor_and_engine_pids() { // The launcher and the engine server are different processes; each PID // must be verified with its OWN start-time token and terminated. - let sup = std::process::Command::new("sleep") + let sup = crate::test_support::bare_command("sleep") .arg("60") .spawn() .expect("spawn supervisor"); - let eng = std::process::Command::new("sleep") + let eng = crate::test_support::bare_command("sleep") .arg("60") .spawn() .expect("spawn engine"); @@ -17056,7 +17132,7 @@ pub(crate) mod tests { #[test] fn managed_stop_terminates_verified_pid() { // Correct identity: the recorded process is really ours, so it is stopped. - let child = std::process::Command::new("sleep") + let child = crate::test_support::bare_command("sleep") .arg("60") .spawn() .expect("spawn child"); @@ -17069,10 +17145,10 @@ pub(crate) mod tests { assert_eq!(signaled, vec![pid]); assert!(all_stopped); - // Reap our own child (a detached managed process would be reaped by init) - // so the liveness check does not observe a not-yet-reaped zombie. + // Bounded, and our own child, so this pins that the terminate killed it + // rather than that it outlived the test. let mut child = child; - let _ = child.wait(); + assert_stopped_within_grace(&mut child, "the verified process must be terminated"); assert!( !rocm_core::process_is_running(pid), "the verified process must be terminated" @@ -17080,206 +17156,3792 @@ pub(crate) mod tests { let _ = fs::remove_dir_all(root); } - pub(crate) fn test_paths(name: &str) -> (PathBuf, AppPaths) { - let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("..") - .join("..") - .join(".rocm-work") - .join("tests") - .join("main") - .join(format!( - "rocm-cli-main-test-{name}-{}", - crate::test_support::unique_suffix() - )); - let _ = fs::remove_dir_all(&root); - ( - root.clone(), - AppPaths { - config_dir: root.join("config"), - data_dir: root.join("data"), - cache_dir: root.join("cache"), - }, - ) - } - + // Linux-only, not merely unix: these rely on `process_start_ticks` (Some on + // Linux, None elsewhere) and zombie-state detection in `terminate_verified`. + #[cfg(target_os = "linux")] #[test] - fn test_paths_gives_each_call_its_own_root_even_for_one_label() { - crate::test_support::assert_each_call_gets_its_own_root(|| test_paths("same-label").0); - } + fn uninstall_stops_live_managed_service_and_reports_it() { + // EAI-8014: a live managed server must be stopped before uninstall + // removes the tooling that stops it, and reported so the operator knows. + let child = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn managed server"); + let pid = child.id(); + let (root, paths) = test_paths("uninstall-stops-live"); + let real = rocm_core::process_start_ticks(pid).expect("start-ticks"); + let mut record = managed_record_for_pid(&paths, pid, Some(real)); + record.status = "ready".to_owned(); + record.write().expect("write service record"); - // ---- Phase 9: reroute dispatch (bare `rocm` + interactive `rocm chat`) ---- - // - // The interactive branches require a real TTY (`interactive_terminal()`), - // which is unavailable in CI, and the dash visuals are trust-prior. These - // tests instead PROVE the dispatch TARGET changed: the two interactive - // handlers now call `dash::run_chat` and no longer call `tui::run`. We read - // this source file at test time and assert on the handler bodies. - fn main_rs_source() -> String { - let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) - .join("src") - .join("main.rs"); - fs::read_to_string(path).expect("read apps/rocm/src/main.rs") - } + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); - /// Extract the body of `fn launch_default()` (brace-balanced). - fn launch_default_body(src: &str) -> String { - slice_braced_block(src, "fn launch_default() -> Result<()> {") + assert!(report.failed.is_empty(), "nothing should fail: {report:?}"); + assert_eq!(report.stopped, vec![record.service_id]); + let mut child = child; + assert_stopped_within_grace( + &mut child, + "the managed server must be stopped before uninstall proceeds", + ); + assert!( + !rocm_core::process_is_running(pid), + "the managed server must be stopped before uninstall proceeds" + ); + let _ = fs::remove_dir_all(root); } - /// Extract the whole `Some(Command::Chat { .. }) => { .. }` handler arm, - /// brace-balanced from the destructure pattern through the block body, so - /// both the destructured fields (e.g. `chat_mock`) and the dispatch code are - /// visible. Restricts to the production handler (the `=> {` form), not the - /// variant declaration which has no `=>`. - fn command_chat_handler_body(src: &str) -> String { - // Anchor on the handler arm specifically: the destructure that is - // immediately followed (after the closing `}` + `)`) by `=> {`. - let arm = "Some(Command::Chat {"; - let arm_at = src.find(arm).expect("Command::Chat handler present"); - // Balance from the `{` of the destructure to capture the full arm, - // including the `=> { ... }` block that follows. - slice_braced_to_arm_end(&src[arm_at..]) - } + #[cfg(target_os = "linux")] + #[test] + fn uninstall_skips_already_dead_managed_service() { + // A service whose process already crashed is not live; it must neither be + // counted as stopped by us nor fail the abort gate that keeps the tooling. + // + // A live service shares the directory so the assertion cannot be + // satisfied by a gate that simply does nothing: the dead one must be + // skipped WHILE the live one is stopped. + let mut dead = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn"); + let dead_pid = dead.id(); + let _ = dead.kill(); + let _ = dead.wait(); + assert!( + !rocm_core::process_is_running(dead_pid), + "the process must be gone before the record is loaded" + ); + let (root, paths) = test_paths("uninstall-skips-dead"); + let mut crashed = managed_record_for_pid(&paths, dead_pid, None); + crashed.status = "ready".to_owned(); + crashed.write().expect("write service record"); - /// Given a slice beginning at `Some(Command::Chat {`, return the text from - /// that point through the end of the arm's `=> { .. }` block. - fn slice_braced_to_arm_end(src: &str) -> String { - // 1. balance the destructure `{ .. }`. - let pat_open = src.find('{').expect("destructure open brace"); - let bytes = src.as_bytes(); - let mut depth = 0usize; - let mut pat_end = pat_open; - for (i, &b) in bytes.iter().enumerate().skip(pat_open) { - match b { - b'{' => depth += 1, - b'}' => { - depth -= 1; - if depth == 0 { - pat_end = i + 1; - break; - } - } - _ => {} - } - } - // 2. find the `=> {` block opener after the destructure and balance it. - let after = &src[pat_end..]; - let block_rel = after.find("=> {").expect("arm block opener `=> {`"); - let block_body = slice_braced_block(after, &after[block_rel..block_rel + 4]); - // Return destructure + block body together. - format!("{}{}", &src[pat_open..pat_end], block_body) - } + let live = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn live managed server"); + let live_pid = live.id(); + let mut running = managed_record_for_pid(&paths, live_pid, None); + running.service_id = "svc-managed-live".to_owned(); + running.manifest_path = paths.service_manifest_path(&running.service_id); + running.supervisor_start_ticks = rocm_core::process_start_ticks(live_pid); + running.status = "ready".to_owned(); + running.write().expect("write live service record"); - /// Return the brace-balanced block that follows `opener` (which must end in - /// `{`), including the contents but excluding the trailing brace's tail. - fn slice_braced_block(src: &str, opener: &str) -> String { - let start = src.find(opener).unwrap_or_else(|| { - panic!("opener not found: {opener}"); - }); - let brace_start = start + opener.len() - 1; // index of the `{` - let bytes = src.as_bytes(); - let mut depth = 0usize; - let mut end = brace_start; - for (i, &b) in bytes.iter().enumerate().skip(brace_start) { - match b { - b'{' => depth += 1, - b'}' => { - depth -= 1; - if depth == 0 { - end = i + 1; - break; - } - } - _ => {} - } - } - src[brace_start..end].to_string() - } + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); - /// Strip `//` line comments so assertions key on real code, not the prose - /// comments that still mention `tui::run` for documentation. - fn strip_line_comments(block: &str) -> String { - block - .lines() - .map(|line| match line.find("//") { - Some(idx) => &line[..idx], - None => line, - }) - .collect::>() - .join("\n") + assert!( + report.failed.is_empty(), + "a crashed service must not abort uninstall: {report:?}" + ); + assert_eq!( + report.stopped, + vec!["svc-managed-live".to_owned()], + "only the live service is stopped and reported: {report:?}" + ); + let mut live = live; + assert_stopped_within_grace(&mut live, "the live service must really have been stopped"); + let _ = fs::remove_dir_all(root); } + #[cfg(target_os = "linux")] #[test] - fn dash_run_chat_is_the_bare_and_chat_dispatch_target() { - // Symbol exists with the expected signature (bare `rocm` / `rocm chat` - // interactive target). Coercing to the fn pointer fails to compile if - // the entrypoint is removed or its signature drifts; casting the pointer - // to an address gives the binding a real effect (no underscore-bind). - let target: fn(bool, ChatInferenceParams, bool) -> Result<()> = dash::run_chat; - assert_ne!(target as usize, 0); - } + fn uninstall_stops_a_live_background_helper() { + // This pins that the daemon stop still happens at all — the + // feature-removal guard. It does NOT pin identity verification: an + // unrecorded start-time also classifies as `Matches`, so this passes + // against the pre-fix `ProcessIdentity::new(pid, None)` too. The + // verification itself is pinned by + // `uninstall_never_kills_a_daemon_pid_that_was_recycled` and + // `uninstall_never_kills_a_daemon_pid_from_a_state_file_that_predates_start_ticks`, + // both of which fail if the identity check is dropped. + // + // Why the daemon goes first: it restarts a managed service whose + // endpoint stops answering, which is exactly the state the stop pass + // creates before writing the record back. + let child = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn daemon stand-in"); + let pid = child.id(); + let (root, paths) = test_paths("uninstall-stops-verified-daemon"); + let mut state = runtime_state(true, pid); + state.daemon_start_ticks = rocm_core::process_start_ticks(pid); + assert!( + state.daemon_start_ticks.is_some(), + "the recorded identity is the point of this test" + ); + state.write(&paths).expect("write runtime state"); - #[test] - fn launch_default_routes_to_dash_run_chat_not_tui() { - let src = main_rs_source(); - let body = strip_line_comments(&launch_default_body(&src)); + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + assert!(report.failed.is_empty(), "nothing should fail: {report:?}"); + assert_eq!( + report.stopped, + Vec::::new(), + "the helper is not a service: {report:?}" + ); assert!( - body.contains("dash::run_launcher(false)"), - "bare `rocm` interactive branch must route to the launcher; body:\n{body}" + report.helper_stopped, + "a verified daemon is stopped and reported: {report:?}" + ); + let mut child = child; + assert_stopped_within_grace( + &mut child, + "the background helper must be stopped before uninstall proceeds", ); assert!( - !body.contains("tui::run"), - "launch_default must NOT call tui::run after the reroute; body:\n{body}" + !rocm_core::process_is_running(pid), + "the background helper must be stopped before uninstall proceeds" ); + let _ = fs::remove_dir_all(root); } + #[cfg(target_os = "linux")] #[test] - fn command_chat_interactive_routes_to_dash_run_chat_not_tui() { - let src = main_rs_source(); - let body = strip_line_comments(&command_chat_handler_body(&src)); + fn uninstall_never_kills_a_daemon_pid_from_a_state_file_that_predates_start_ticks() { + // The ordinary upgrade path: `runtime-state.json` written by a pre-upgrade + // `rocmd` carries no `daemon_start_ticks`. `identity_state` calls that + // `Matches` (its legacy best-effort arm), so signalling on that verdict + // would force-kill a whole tree at a pid nothing has verified. On a + // platform that CAN read start-times, an unrecorded one means the record + // is stale, not that the pid is ours. + let stranger = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn unrelated process"); + let pid = stranger.id(); + let (root, paths) = test_paths("uninstall-daemon-legacy-record"); + let state = runtime_state(true, pid); assert!( - body.contains("dash::run_chat("), - "interactive `rocm chat` must route to dash::run_chat(...); body:\n{body}" + state.daemon_start_ticks.is_none(), + "this test is about the unrecorded-identity path" ); + state.write(&paths).expect("write runtime state"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + let mut stranger = stranger; assert!( - !body.contains("tui::run"), - "Command::Chat handler must NOT call tui::run after the reroute; body:\n{body}" + stranger + .try_wait() + .expect("poll the unrelated process") + .is_none(), + "an unverifiable pid must never be signalled: uninstall killed an unrelated process" ); - } - - #[test] - fn command_chat_interactive_notes_dropped_provider_flag() { - // Phase-9 polish: --provider on interactive `rocm chat` is no longer - // silently ignored — the handler emits a one-line note when provider is - // set before rerouting to the dash. Proven by reading the handler body - // (the interactive branch requires a TTY, unavailable in CI). - let src = main_rs_source(); - let body = strip_line_comments(&command_chat_handler_body(&src)); assert!( - body.contains("provider.is_some()"), - "handler must gate the note on a set --provider; body:\n{body}" + report.stopped.is_empty() && !report.helper_stopped, + "nothing was confirmed stopped: {report:?}" ); assert!( - body.contains("/provider"), - "the note must point the user at /provider for live switching; body:\n{body}" + report.failed.iter().any(|failure| failure.remedy + == StopFailureRemedy::ConfirmTheDaemonPid + && failure.service_id.contains(&pid.to_string())), + "an unverifiable helper must abort the uninstall and name its pid: {report:?}" ); + let _ = stranger.kill(); + let _ = stranger.wait(); + let _ = fs::remove_dir_all(root); } #[test] - fn command_chat_honors_chat_mock_and_keeps_prompt_passthrough() { - let src = main_rs_source(); - let body = strip_line_comments(&command_chat_handler_body(&src)); - // --chat-mock is forwarded into the dash reroute. + fn uninstall_refuses_when_the_daemon_runtime_state_cannot_be_read() { + // `load` returns `Err` for a corrupt or unreadable state file and + // `Ok(None)` only when there is none. Discarding that `Err` would skip the + // daemon stop silently and remove the tooling while a live `rocmd` can + // still respawn what the service pass just stopped. Reachable in practice: + // the file is rewritten on every tick. + let (root, paths) = test_paths("uninstall-daemon-state-corrupt"); + paths.ensure().expect("create the app directories"); + fs::write(paths.automation_state_path(), b"{ not json") + .expect("seed a corrupt runtime state"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + assert!( - body.contains("chat_mock,") || body.contains("chat_mock\n"), - "Command::Chat must destructure the --chat-mock field; body:\n{body}" + report + .failed + .iter() + .any(|failure| failure.remedy == StopFailureRemedy::RepairTheDaemonState), + "an unreadable runtime state must abort the uninstall: {report:?}" ); + let error = uninstall_removal_gate(&report) + .expect_err("the gate must refuse to remove anything") + .to_string(); + // The abort has to be followable. No pid was ever parsed out of this + // file, so "kill that pid" would be a dead end — uninstall has no + // `--force`, and repairing or deleting the named file is the only way + // out. + let state_path = paths.automation_state_path(); assert!( - body.contains("dash::run_chat(") && body.contains("chat_mock,"), - "--chat-mock must be forwarded to run_chat; body:\n{body}" + error.contains(&state_path.display().to_string()), + "the abort must name the file to repair or delete: {error}" ); assert!( - body.contains("ChatInferenceParams") + error.contains("repair or delete"), + "the abort must say what to do with it: {error}" + ); + assert!( + !error.contains("kill that pid"), + "no pid was ever read from this file, so that advice cannot be followed: {error}" + ); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn uninstall_never_kills_a_daemon_pid_that_was_recycled() { + // `runtime-state.json` survives a crash, OOM-kill or reboot with + // `running` still true, so the recorded pid can belong to a stranger by + // the time uninstall runs. Killing on a bare pid would make uninstall + // destroy something the user never installed — with `KillScope::Tree` + // and `force`, an unrelated process AND all its children. + // + // The stand-in plays the recycled process: a real live pid recorded with + // a start-time that is not its own. It must come through untouched. + let stranger = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn unrelated process"); + let pid = stranger.id(); + let real = rocm_core::process_start_ticks(pid).expect("start-ticks"); + let (root, paths) = test_paths("uninstall-recycled-daemon-pid"); + let mut state = runtime_state(true, pid); + // The daemon that recorded this pid started at a different time; this pid + // has since been reissued to `stranger`. + state.daemon_start_ticks = Some(real.wrapping_add(1)); + state.write(&paths).expect("write runtime state"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + // `try_wait` on our own child, not `process_is_running`: a killed but + // unreaped child is a zombie, which a pid probe still calls running. + let mut stranger = stranger; + assert!( + stranger + .try_wait() + .expect("poll the unrelated process") + .is_none(), + "a recycled pid must never be signalled: uninstall killed an unrelated process" + ); + assert!( + !report.helper_stopped, + "a recycled pid is not a helper this pass stopped: {report:?}" + ); + assert!( + report.stopped.is_empty(), + "nothing of ours was running, so nothing was stopped: {report:?}" + ); + assert!( + report.failed.is_empty(), + "a recycled pid means the daemon is already gone, not that it is stuck: {report:?}" + ); + let _ = stranger.kill(); + let _ = stranger.wait(); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn uninstall_refuses_when_a_service_record_cannot_be_parsed() { + // `load_managed_services` skips a manifest it cannot parse, which is + // right for listing and wrong here: the unreadable record may describe a + // live, GPU-holding server, so uninstall must not proceed blind to it. + let (root, paths) = test_paths("uninstall-corrupt-record"); + let services_dir = paths.services_dir(); + fs::create_dir_all(&services_dir).expect("create services dir"); + fs::write(services_dir.join("svc-corrupt.json"), b"{ not json") + .expect("write corrupt service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + let failed: Vec<&str> = report + .failed + .iter() + .map(|failure| failure.service_id.as_str()) + .collect(); + let corrupt = services_dir.join("svc-corrupt.json"); + assert_eq!( + failed, + vec![corrupt.display().to_string().as_str()], + "an unparseable record must fail the gate: {report:?}" + ); + let error = uninstall_removal_gate(&report) + .expect_err("an unparseable record must abort uninstall") + .to_string(); + // The abort has to be escapable. `rocm services stop` loads the same + // file and fails the same way, so prescribing it here would make every + // retry abort identically — a permanent block with no way out. The only + // remedy is on disk, so the message must name the file and say so. + assert!( + error.contains(&corrupt.display().to_string()), + "names the file to act on: {error}" + ); + assert!( + error.contains("repair or delete the file"), + "states the remedy that can actually clear it: {error}" + ); + assert!( + !error.contains("rocm services stop"), + "must not prescribe a command that fails on the same unparseable file: {error}" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn uninstall_removal_gate_aborts_when_a_service_cannot_be_stopped() { + // The core safety guarantee: a service that could not be confirmed stopped + // aborts uninstall (Err, so the removal loop never runs — nothing removed), + // and the message names the offending service and points at the recovery + // path. Guards against a regression that would delete the tooling while a + // GPU-holding endpoint keeps serving (EAI-8014). + let report = ManagedServiceStopReport { + stopped: vec!["svc-stopped".to_owned()], + failed: vec![FailedManagedServiceStop { + service_id: "svc-stuck".to_owned(), + reason: "still \"ready\" after the stop attempt".to_owned(), + remedy: StopFailureRemedy::StopTheService, + }], + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }; + let error = uninstall_removal_gate(&report) + .expect_err("a non-empty `failed` must abort uninstall") + .to_string(); + assert!( + error.contains("svc-stuck"), + "names the stuck service: {error}" + ); + assert!( + error.contains("still \"ready\" after the stop attempt"), + "says why the stop could not be confirmed: {error}" + ); + assert!( + error.contains("rocm services stop"), + "points at the recovery path: {error}" + ); + assert!( + error.contains("No files were removed"), + "states nothing was deleted: {error}" + ); + } + + #[test] + fn uninstall_abort_says_which_services_it_already_stopped() { + // The stop pass is not atomic: services stopped before the failing one + // stay stopped and have lost their endpoint keys. An abort that only + // said "No files were removed" would read as "nothing happened". + let report = ManagedServiceStopReport { + stopped: vec!["svc-public".to_owned()], + relaunch: vec![format!( + "svc-public: `{}`", + rocm_engine_protocol::relaunch_command("0.0.0.0", true) + )], + failed: vec![FailedManagedServiceStop { + service_id: "svc-stuck".to_owned(), + reason: "still \"ready\" after the stop attempt".to_owned(), + remedy: StopFailureRemedy::StopTheService, + }], + warnings: Vec::new(), + helper_stopped: false, + }; + let error = uninstall_removal_gate(&report) + .expect_err("a non-empty `failed` must abort uninstall") + .to_string(); + assert!( + error.contains("svc-public"), + "names what it already stopped: {error}" + ); + assert!( + error.contains("--allow-public-bind"), + "says how a public service comes back after losing its key: {error}" + ); + } + + #[test] + fn every_failure_class_carries_its_own_recovery_advice() { + // This gate refuses to remove anything while a stop is unconfirmed, so + // the advice it prints is the operator's only way out; a class that + // reaches the abort with no advice turns the refusal into a dead end. + // `StopFailureRemedy::advice` and `advice_rank` are exhaustive matches, + // so a new variant cannot compile without being given text and a + // place in the order — but the compiler cannot check that the text is + // the *right* text, or that the three id-carrying classes still name + // their ids. That is what this asserts, across every class at once. + // Listed in REVERSE `advice_rank` order, and that is load-bearing + // rather than arbitrary. Listing them in rank order makes insertion + // order and rank order coincide, so the ordering assertion below holds + // whether or not the code sorts at all — deleting the sort left the + // whole suite green. Reversed, the assertion can only pass because + // `advice_rank` put them back. + let report = ManagedServiceStopReport { + stopped: Vec::new(), + failed: vec![ + FailedManagedServiceStop { + service_id: "svc-corrupt.json".to_owned(), + reason: "does not parse".to_owned(), + remedy: StopFailureRemedy::RepairTheRecord, + }, + FailedManagedServiceStop { + service_id: "runtime.json".to_owned(), + reason: "does not parse".to_owned(), + remedy: StopFailureRemedy::RepairTheDaemonState, + }, + FailedManagedServiceStop { + service_id: "rocmd (pid 4321)".to_owned(), + reason: "identity unverified".to_owned(), + remedy: StopFailureRemedy::StopTheDaemon, + }, + FailedManagedServiceStop { + service_id: "svc-orphaned".to_owned(), + reason: "endpoint still answers".to_owned(), + remedy: StopFailureRemedy::StopWhatHoldsThePort, + }, + FailedManagedServiceStop { + service_id: "svc-wedged".to_owned(), + reason: "still ready".to_owned(), + remedy: StopFailureRemedy::StopTheService, + }, + FailedManagedServiceStop { + service_id: "svc-oldport".to_owned(), + reason: "an authenticated server answers".to_owned(), + remedy: StopFailureRemedy::RemoveTheStoppedRecord, + }, + FailedManagedServiceStop { + service_id: "svc-tickless".to_owned(), + reason: "no start time".to_owned(), + remedy: StopFailureRemedy::ConfirmTheServicePid, + }, + FailedManagedServiceStop { + service_id: "rocmd (pid 4321; state file /x/runtime-state.json)".to_owned(), + reason: "identity unverified".to_owned(), + remedy: StopFailureRemedy::ConfirmTheDaemonPid, + }, + ], + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }; + + let error = uninstall_removal_gate(&report) + .expect_err("a non-empty `failed` must abort uninstall") + .to_string(); + + // One distinctive phrase per class, none of them supplied by this test. + for expected in [ + "rocm services stop --yes", + "Find what holds that port", + "restarts managed services on its own", + "cannot tell whether the helper is running", + "No `rocm` command can act on an unparseable record", + "rocm services remove --yes", + "No start time was recorded for a live pid", + "delete the runtime state file named there", + ] { + assert!( + error.contains(expected), + "every failure class must carry its own remedy; missing {expected:?} in: {error}" + ); + } + // Most actionable first, hand-repair last: a dead-end-avoidance + // ordering, not cosmetics. + let position = |needle: &str| error.find(needle).expect("asserted present above"); + // The whole `advice_rank` chain, one phrase per class: moving any one + // class out of its place breaks a neighbouring comparison. + let in_rank_order = [ + "rocm services stop --yes", + "Find what holds that port", + "rocm services remove --yes", + "No start time was recorded for a live pid", + "restarts managed services on its own", + "delete the runtime state file named there", + "cannot tell whether the helper is running", + "No `rocm` command can act on an unparseable record", + ]; + for pair in in_rank_order.windows(2) { + assert!( + position(pair[0]) < position(pair[1]), + "remedies must stay ordered most-actionable-first ({:?} before {:?}): {error}", + pair[0], + pair[1] + ); + } + } + + #[cfg(target_os = "linux")] + #[test] + fn uninstall_refuses_while_a_recorded_endpoint_still_accepts_connections() { + // The Windows grandchild case, modelled faithfully: the recorded + // processes really do die, so the stop reports "stopped" — but an engine + // child that outlived them still holds the port and the GPU. Only the + // port probe can catch that; PID bookkeeping alone says success. + let listener = + std::net::TcpListener::bind("127.0.0.1:0").expect("bind the surviving engine's socket"); + let port = listener.local_addr().expect("socket address").port(); + // The recorded supervisor: alive now, so the record stays live through + // the liveness refresh, and killable, so the stop confirms. + let supervisor = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn the recorded supervisor"); + let supervisor_pid = supervisor.id(); + let (root, paths) = test_paths("uninstall-endpoint-still-serving"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-orphaned-engine", + "vllm", + "m", + "m", + "127.0.0.1", + port, + "managed", + supervisor_pid, + None, + None, + None, + ); + record.supervisor_start_ticks = rocm_core::process_start_ticks(supervisor_pid); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + let mut supervisor = supervisor; + assert_stopped_within_grace( + &mut supervisor, + "the recorded process must really have been stopped, so only the port probe \ + can catch the survivor", + ); + assert!( + !rocm_core::process_is_running(supervisor_pid), + "the recorded process must really have been stopped, so only the \ + port probe can catch the survivor" + ); + let failed: Vec<&str> = report + .failed + .iter() + .map(|failure| failure.service_id.as_str()) + .collect(); + assert_eq!( + failed, + vec!["svc-orphaned-engine"], + "a reachable endpoint must fail the gate: {report:?}" + ); + assert!( + !report.stopped.iter().any(|id| id == "svc-orphaned-engine"), + "a service that is still serving must not also be counted stopped: {report:?}" + ); + assert!( + uninstall_removal_gate(&report).is_err(), + "uninstall must not remove the tooling while something still serves" + ); + drop(listener); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_stale_record_whose_old_port_was_reused_does_not_block_uninstall() { + // Nothing prunes a stopped record automatically, so an old one stays on + // disk until the operator runs `rocm services remove --yes`. If the gate judged + // those records by their recorded port, any unrelated process that later + // bound it would fail uninstall deterministically, with no override and + // no recovery — `rocm services stop` cannot help an already-stopped + // record, so every retry would fail identically. + // + // Unlike its neighbours this one does NOT fail if the port probe is + // deleted, and that is deliberate rather than an oversight: it guards + // the opposite direction. The others pin that a live server blocks; this + // pins that a *stranger* does not, so it fails against a naive "any + // listener blocks" gate — the over-strict implementation the rest of + // this file's pressure pushes toward — and passes against no gate at + // all. It is a false-positive guard, so read it as one. + let listener = std::net::TcpListener::bind("127.0.0.1:0") + .expect("bind an unrelated process on a recycled port"); + let port = listener.local_addr().expect("socket address").port(); + let (root, paths) = test_paths("uninstall-stale-record-recycled-port"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-long-stopped", + "vllm", + "m", + "m", + "127.0.0.1", + port, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + assert!( + report.failed.is_empty(), + "a stale record must not be judged by whoever holds its old port now: {report:?}" + ); + assert!( + uninstall_removal_gate(&report).is_ok(), + "uninstall must not be permanently blocked by a recycled port" + ); + drop(listener); + let _ = fs::remove_dir_all(root); + } + + /// A fake OpenAI `/v1/models` endpoint on loopback that shuts itself down. + /// + /// `drop`ping a bare `JoinHandle` only detaches it: the thread stays parked + /// in `accept()` holding an ephemeral port for the life of the test binary, + /// and a failing assertion panics before any manual cleanup line. Owning the + /// shutdown in `Drop` means the port is released even when the test fails, + /// which is when it matters. + struct ServingEndpoint { + bind_host: &'static str, + port: u16, + shutdown: std::sync::Arc, + thread: Option>, + } + + impl ServingEndpoint { + /// Serve `model_id` from `/v1/models` on IPv4 loopback until dropped. + fn serving(model_id: &str) -> Self { + Self::bind(model_id, "127.0.0.1", false).expect("bind the surviving engine on IPv4") + } + + /// Serve `model_id` only to a request carrying an `Authorization` header, + /// answering 401 otherwise — a public service as the retry run meets it. + fn serving_with_authorization(model_id: &str) -> Self { + Self::bind(model_id, "127.0.0.1", true).expect("bind the authenticated engine") + } + + /// Answer `/v1/models` with an empty list — an engine that is up and + /// holding the port but has not populated its models, or is mid-unload. + fn listing_nothing() -> Self { + Self::bind("", "127.0.0.1", false).expect("bind the empty-listing engine") + } + + /// Serve `model_id` on `bind_host`, or `None` when the host's address + /// family is unavailable (IPv6 is absent in some containers, and a test + /// that needs it has to skip rather than fail). + fn bind(model_id: &str, bind_host: &'static str, require_auth: bool) -> Option { + Self::bind_refusing_with(model_id, bind_host, require_auth, 401) + } + + /// As [`Self::bind`], answering `refusal_status` to a request the + /// authorization requirement rejects (401 and 403 are both "guarded"). + fn bind_refusing_with( + model_id: &str, + bind_host: &'static str, + require_auth: bool, + refusal_status: u16, + ) -> Option { + Self::bind_at(model_id, bind_host, 0, require_auth, refusal_status) + } + + /// Both loopback families on ONE port: `v6_model` on `::1` and + /// `v4_model` on `127.0.0.1`, so a test about a wildcard record owns + /// both halves and no concurrent test can grab the v4 side of its port. + /// `None` when this host has no IPv6 loopback. + fn bind_loopback_pair(v6_model: &str, v4_model: &str) -> Option<(Self, Self)> { + for _ in 0..50 { + let v4 = Self::bind(v4_model, "127.0.0.1", false)?; + if let Some(v6) = Self::bind_at(v6_model, "::1", v4.port, false, 401) { + return Some((v6, v4)); + } + // The port was free on v4 but taken on v6: try another. + } + None + } + + fn bind_at( + model_id: &str, + bind_host: &'static str, + port: u16, + require_auth: bool, + refusal_status: u16, + ) -> Option { + use std::sync::atomic::Ordering; + + let listener = std::net::TcpListener::bind((bind_host, port)).ok()?; + let port = listener.local_addr().ok()?.port(); + let shutdown = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let stopping = std::sync::Arc::clone(&shutdown); + // An empty `model_id` means "list nothing at all", not "list a model + // whose id is the empty string" — the two are different answers and + // only the first is the engine-still-loading case. + let body = if model_id.is_empty() { + r#"{"data":[]}"#.to_owned() + } else { + format!(r#"{{"data":[{{"id":"{model_id}"}}]}}"#) + }; + let thread = thread::spawn(move || { + use std::io::{Read, Write}; + while let Ok((mut stream, _)) = listener.accept() { + if stopping.load(Ordering::SeqCst) { + break; + } + stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); + let mut buffer = [0_u8; 1024]; + let Ok(read) = stream.read(&mut buffer) else { + continue; + }; + let authorized = !require_auth + || String::from_utf8_lossy(&buffer[..read]) + .to_ascii_lowercase() + .contains("authorization: bearer "); + if authorized { + let _ = write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ); + } else { + let _ = write!( + stream, + "HTTP/1.1 {refusal_status} Refused\r\nContent-Type: application/json\r\nContent-Length: 2\r\nConnection: close\r\n\r\n{{}}" + ); + } + } + }); + Some(Self { + bind_host, + port, + shutdown, + thread: Some(thread), + }) + } + } + + impl Drop for ServingEndpoint { + fn drop(&mut self) { + self.shutdown + .store(true, std::sync::atomic::Ordering::SeqCst); + // Unblock the parked `accept()` so the thread observes the flag. + let _ = std::net::TcpStream::connect((self.bind_host, self.port)); + if let Some(thread) = self.thread.take() { + let _ = thread.join(); + } + } + } + + #[test] + fn a_stopped_record_still_serving_its_own_model_blocks_uninstall() { + // The retry the abort message asks for must not be the hole. A stop + // persists `status = "stopped"` BEFORE the gate probes the port, so the + // record that failed run 1 reads as not-live on run 2 — and if the gate + // only probed what it had just tried to stop, run 2 would sail through + // and remove the tooling while the survivor kept serving and holding the + // GPU. That is EAI-8014 reached by following the gate's own + // instructions, so the evidence has to survive the retry: an endpoint + // serving this record's own model blocks, whoever it belongs to. + let endpoint = ServingEndpoint::serving("amd/orphaned-model"); + let port = endpoint.port; + + let (root, paths) = test_paths("uninstall-stopped-record-still-serving"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-orphaned-engine", + "vllm", + "amd/orphaned-model", + "amd/orphaned-model", + "127.0.0.1", + port, + "managed", + 0, + None, + None, + None, + ); + // Exactly the state run 1 leaves behind: supervisor killed, record + // written as stopped, engine grandchild still on the port. + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + let failure = report + .failed + .iter() + .find(|failure| failure.service_id == "svc-orphaned-engine") + .unwrap_or_else(|| panic!("a still-serving engine must fail the gate: {report:?}")); + assert_eq!( + failure.remedy, + StopFailureRemedy::StopWhatHoldsThePort, + "the recorded processes are gone, so `rocm services stop` is not the remedy" + ); + // This record was already marked stopped before the run, so this pass + // never attempted a stop on it. The reason must not say the endpoint + // survived one — that describes something that did not happen, on the + // one output its operator has to reason from. Collapsing the two arms + // back into the old single sentence fails here. + assert!( + failure + .reason + .contains("is recorded stopped, but something there is still serving"), + "a record nothing was attempted on must not be reported as surviving a stop: {}", + failure.reason + ); + assert!( + !failure.reason.contains("after the stop"), + "no stop ran for this record, so the reason must not claim one did: {}", + failure.reason + ); + let error = uninstall_removal_gate(&report) + .expect_err("a still-serving engine must abort uninstall") + .to_string(); + assert!( + error.contains("outlived its supervisor"), + "says why `rocm services stop` will not help: {error}" + ); + drop(endpoint); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_wildcard_bound_engine_is_identified_instead_of_waved_through() { + // `probe_host` normalizes a wildcard bind to loopback for the + // reachability check, but the identity probe used to be handed the raw + // record, whose `endpoint_url` still spells the wildcard. That address + // does not resolve, so the probe errored and took the fail-open branch: + // uninstall removed the tooling while a wildcard-bound engine was still + // serving — the exact outcome this gate exists to prevent. + // + // `*` is one of the spellings `probe_host` documents a record can carry, + // and unlike `0.0.0.0` it fails to resolve on every platform, so this + // pins the behaviour rather than a host quirk. + let endpoint = ServingEndpoint::serving("amd/wildcard-model"); + let (root, paths) = test_paths("uninstall-wildcard-bound-engine"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-wildcard-engine", + "vllm", + "amd/wildcard-model", + "amd/wildcard-model", + "*", + endpoint.port, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + let failure = report + .failed + .iter() + .find(|failure| failure.service_id == "svc-wildcard-engine") + .unwrap_or_else(|| { + panic!("a wildcard-bound engine still serving must fail the gate: {report:?}") + }); + assert_eq!( + failure.remedy, + StopFailureRemedy::StopWhatHoldsThePort, + "its recorded processes are gone, so the port holder is the remedy" + ); + assert!( + uninstall_removal_gate(&report).is_err(), + "the gate must refuse to remove anything" + ); + drop(endpoint); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn an_ipv6_only_wildcard_engine_is_not_waved_through() { + // The IPv6 half of the wildcard case. `::` with `IPV6_V6ONLY` — the + // default on Windows, and what `TcpListener::bind("::1", 0)` gives here + // — answers on `::1` and refuses `127.0.0.1`. Probing IPv4 alone reads + // that live, port-holding engine as gone and removes the tooling anyway, + // which is EAI-8014 reached through the address family. + // The v4 half of the same port is held by a listener that names no + // models, so no concurrent test can bind it and answer for the engine. + let Some((endpoint, v4_holder)) = ServingEndpoint::bind_loopback_pair("amd/v6-model", "") + else { + // No IPv6 on this host: the thing under test cannot be staged. Say + // so on stderr rather than returning green and silent — a skip that + // looks identical to a pass is how a lane stops covering something + // without anyone noticing. `every_wildcard_bind_spelling_is_probed_\ + // on_both_loopback_families` still pins both families here, with no + // socket required. + eprintln!( + "SKIPPED an_ipv6_only_wildcard_engine_is_not_waved_through: no IPv6 loopback \ + on this host" + ); + return; + }; + let (root, paths) = test_paths("uninstall-ipv6-wildcard-engine"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-v6-engine", + "vllm", + "amd/v6-model", + "amd/v6-model", + // Recorded as the wildcard it was launched on, which is all the + // record ever says — not which family the listener chose. + "::", + endpoint.port, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + let failure = report + .failed + .iter() + .find(|failure| failure.service_id == "svc-v6-engine") + .unwrap_or_else(|| { + panic!("an IPv6-only engine still serving must fail the gate: {report:?}") + }); + assert_eq!( + failure.remedy, + StopFailureRemedy::StopWhatHoldsThePort, + "its recorded processes are gone, so the port holder is the remedy" + ); + assert!( + uninstall_removal_gate(&report).is_err(), + "the gate must refuse to remove anything" + ); + drop(endpoint); + drop(v4_holder); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_stranger_on_one_loopback_family_does_not_hide_the_record_serving_on_the_other() { + // `probe_records_for_survivors` used to identity-probe only the FIRST + // loopback family that accepted a connection. Here `127.0.0.1` answers + // first with another model (proceed), while the record's own model still + // serves on `::1`: a first-hit probe waved the removal through. + let Some((v6_engine, v4_stranger)) = + ServingEndpoint::bind_loopback_pair("amd/dual-model", "amd/someone-elses-model") + else { + eprintln!( + "SKIPPED a_stranger_on_one_loopback_family_does_not_hide_the_record_serving_on_\ + the_other: no IPv6 loopback on this host" + ); + return; + }; + let (root, paths) = test_paths("uninstall-dual-family"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-dual", + "vllm", + "amd/dual-model", + "amd/dual-model", + "::", + v6_engine.port, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + assert!( + report + .failed + .iter() + .any(|failure| failure.service_id == "svc-dual" + && failure.remedy == StopFailureRemedy::StopWhatHoldsThePort), + "the record's own model still serving on ::1 must block: {report:?}" + ); + assert!( + uninstall_removal_gate(&report).is_err(), + "the gate must refuse to remove anything" + ); + drop(v6_engine); + drop(v4_stranger); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn every_identity_answer_maps_to_exactly_one_gate_outcome() { + // The five arms, asserted directly. Reached through the stop loop each + // one needs a server that answers a particular way plus a reading of + // stderr, which is why two of them were previously pinned by nothing: + // mutating either "proceed" arm into a block, or a block into a proceed, + // left the whole suite green while changing what a destructive command + // does to a live endpoint. + use rocm_core::EndpointIdentity::{ListsNoModels, ServesExpectedModel, ServesOtherModels}; + + assert_eq!( + stopped_record_verdict(Some(ServesExpectedModel), false), + StoppedRecordVerdict::BlockServingOurModel, + "an engine serving this record's own model outlived its supervisor" + ); + // Only this one is allowed to be silent: naming models and not naming + // ours is the single answer that is real evidence of a stranger. + assert_eq!( + stopped_record_verdict(Some(ServesOtherModels), false), + StoppedRecordVerdict::ProceedUnrelated, + "a listener naming other models is somebody else on a recycled port" + ); + // Blocking here would turn any JSON listener that lists nothing into an + // unescapable abort; proceeding silently would remove the tooling from + // under an engine that is merely still loading. Hence a third outcome. + assert_eq!( + stopped_record_verdict(Some(ListsNoModels), false), + StoppedRecordVerdict::ProceedListingNothing, + "an empty listing is not evidence of a stranger, and not silent" + ); + assert_eq!( + stopped_record_verdict(None, true), + StoppedRecordVerdict::BlockAuthRefused, + "a refusal is a live server stating it guards the path" + ); + assert_eq!( + stopped_record_verdict(None, false), + StoppedRecordVerdict::ProceedUnidentified, + "no usable answer is the documented fail-open" + ); + + assert!(stopped_record_verdict(Some(ServesExpectedModel), false).blocks()); + assert!(stopped_record_verdict(None, true).blocks()); + assert!(!stopped_record_verdict(Some(ServesOtherModels), false).blocks()); + assert!(!stopped_record_verdict(Some(ListsNoModels), false).blocks()); + assert!(!stopped_record_verdict(None, false).blocks()); + } + + #[test] + fn an_endpoint_listing_nothing_does_not_block_uninstall() { + // The end-to-end half of the `ListsNoModels` arm: it must not abort. + // Blocking would make any listener that answers `/v1/models` with an + // empty list — including something unrelated on a recycled port — an + // abort with no override, which is the dead end this gate must not + // create. Sensitive to exactly one mutation: turning that arm into a + // block. + // + // What keeps this a tradeoff rather than a silent removal is the + // warning, a value on the report that this test asserts below. The + // verdict mapping is pinned separately, so emptying the warning body or + // flipping the verdict each turn a test red. + let endpoint = ServingEndpoint::listing_nothing(); + let (root, paths) = test_paths("uninstall-endpoint-listing-nothing"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-empty-listing", + "vllm", + "amd/loading-model", + "amd/loading-model", + "127.0.0.1", + endpoint.port, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + assert!( + report.failed.is_empty(), + "an empty model list must not abort uninstall: {report:?}" + ); + assert!( + uninstall_removal_gate(&report).is_ok(), + "the gate must let the removal proceed" + ); + // The disclosure, not just the decision. Proceeding here is only + // defensible because the operator is told the port was never proven + // free, so emptying that message is as much a regression as flipping + // the verdict — and until the warnings became values on the report, + // nothing could say so. + assert!( + report + .warnings + .iter() + .any(|warning| warning.contains("empty model list") + && warning.contains("svc-empty-listing")), + "proceeding past an empty listing must disclose itself: {:?}", + report.warnings + ); + drop(endpoint); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_record_whose_host_no_longer_resolves_says_so_before_proceeding() { + // The third fail-open, and the one that used to pass in silence. A + // refused connect and an unresolvable name both come back `false` from + // `loopback_tcp_port_is_reachable`, but they are opposite evidence: the + // first says the port is free, the second says the question was never + // asked. Proceeding on the second is still the right call — a record + // written on another machine must not brick uninstall — but it has to + // be disclosed, and `.invalid` is reserved by RFC 2606 precisely so it + // never resolves anywhere. + let (root, paths) = test_paths("uninstall-host-unresolvable"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-foreign-host", + "vllm", + "amd/our-model", + "amd/our-model", + "no-such-host.invalid", + 8123, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + assert!( + report.failed.is_empty(), + "an unresolvable host must not abort uninstall: {report:?}" + ); + assert!( + report + .warnings + .iter() + .any(|warning| warning.contains("does not resolve here") + && warning.contains("svc-foreign-host")), + "skipping a record whose host does not resolve must disclose itself: {:?}", + report.warnings + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_listener_naming_another_model_does_not_block_uninstall() { + // The end-to-end half of the `ServesOtherModels` arm, and the only test + // that drives it through the real call site rather than through the + // pure helper. A listener that names its models and does not name ours + // is the one answer that is positive evidence of a stranger on a + // recycled port, so it must proceed — blocking here would let any + // unrelated OpenAI-shaped server on a reused port wedge uninstall with + // no override. Sensitive to exactly one mutation: turning that arm into + // a block. + let endpoint = ServingEndpoint::serving("amd/somebody-elses-model"); + let (root, paths) = test_paths("uninstall-endpoint-other-model"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-recycled-port", + "vllm", + "amd/our-model", + "amd/our-model", + "127.0.0.1", + endpoint.port, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + assert!( + report.failed.is_empty(), + "a listener naming only other models must not abort uninstall: {report:?}" + ); + assert!( + uninstall_removal_gate(&report).is_ok(), + "the gate must let the removal proceed" + ); + // The one proceed-arm that is allowed to be silent, and the assertion + // that keeps it distinguishable from the two that are not: a listener + // that named its models and did not name ours is positive evidence of a + // stranger, so there is nothing to disclose. + assert!( + report.warnings.is_empty(), + "a named stranger is evidence, not a fail-open, so it warns about nothing: {:?}", + report.warnings + ); + drop(endpoint); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_stopped_record_whose_engine_lists_its_served_name_still_blocks_uninstall() { + // An AIM service records its Docker image reference as the canonical id + // and the container lists a different, served name. Comparing the + // listing against the image reference reads the survivor as a stranger + // and lets uninstall proceed over a server that is still holding the GPU. + let endpoint = ServingEndpoint::serving("amd/served-name"); + let (root, paths) = test_paths("uninstall-served-name-survivor"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-aim-survivor", + "aim", + "registry.example/aim/some-image:1", + "registry.example/aim/some-image:1", + "127.0.0.1", + endpoint.port, + "managed", + 0, + None, + None, + None, + ); + record.served_model_id = Some("amd/served-name".to_owned()); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + let failed: Vec<&str> = report + .failed + .iter() + .map(|failure| failure.service_id.as_str()) + .collect(); + assert_eq!( + failed, + ["svc-aim-survivor"], + "a listener naming this service's served model must block: {report:?}" + ); + assert!( + uninstall_removal_gate(&report).is_err(), + "the gate must refuse the removal" + ); + drop(endpoint); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn an_authenticated_survivor_blocks_the_retry_instead_of_failing_open() { + // The retry run is where a public service loses its own evidence: + // stopping the recorded processes clears the stored endpoint key, so run + // 2 probes the still-serving endpoint with no credentials and is + // answered 401. `managed_service_endpoint_model_ready` turns any non-200 + // into `Err`, which used to take the fail-open branch — deleting the + // tooling while a publicly reachable, GPU-holding server kept answering. + // + // A refusal is not a failure to reach: it is a live HTTP server saying + // it guards this path, and that has to block. + let endpoint = ServingEndpoint::serving_with_authorization("amd/guarded-model"); + let (root, paths) = test_paths("uninstall-authenticated-survivor"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-guarded-engine", + "vllm", + "amd/guarded-model", + "amd/guarded-model", + "127.0.0.1", + endpoint.port, + "managed", + 0, + None, + None, + None, + ); + // Exactly run 2's state: recorded stopped, and no key on disk for it. + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + assert!( + endpoint_keys::endpoint_api_key(&paths, "svc-guarded-engine").is_none(), + "the retry has no credentials left — that is the whole scenario" + ); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + let failure = report + .failed + .iter() + .find(|failure| failure.service_id == "svc-guarded-engine") + .unwrap_or_else(|| { + panic!( + "an authenticated survivor must fail the gate, not be waved through: {report:?}" + ) + }); + assert_eq!( + failure.remedy, + StopFailureRemedy::RemoveTheStoppedRecord, + "an authenticated answer on an old record's port is cleared by removing the record" + ); + assert!( + failure.reason.contains("authenticated server answers"), + "the abort has to say an authenticated server answers there: {}", + failure.reason + ); + assert!( + uninstall_removal_gate(&report).is_err(), + "the gate must refuse to remove anything" + ); + drop(endpoint); + let _ = fs::remove_dir_all(root); + } + + // Gated to Linux with its siblings: nothing here is Linux-specific (loopback HTTP and an injected stop), but no Windows lane has run it yet. + #[cfg(target_os = "linux")] + #[test] + fn a_forbidden_answer_on_a_stopped_records_port_blocks_like_an_unauthorized_one() { + // A server that answers 403 is as much a live, guarded HTTP server as one + // that answers 401; narrowing the refusal check to 401 alone would wave + // it through as an unrelated listener. + let endpoint = + ServingEndpoint::bind_refusing_with("amd/guarded-model", "127.0.0.1", true, 403) + .expect("bind the forbidding engine"); + let (root, paths) = test_paths("uninstall-forbidden-survivor"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-forbidden-engine", + "vllm", + "amd/guarded-model", + "amd/guarded-model", + "127.0.0.1", + endpoint.port, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + let failure = report + .failed + .iter() + .find(|failure| failure.service_id == "svc-forbidden-engine") + .unwrap_or_else(|| panic!("a 403 survivor must fail the gate: {report:?}")); + assert_eq!(failure.remedy, StopFailureRemedy::RemoveTheStoppedRecord); + drop(endpoint); + let _ = fs::remove_dir_all(root); + } + + // Gated to Linux with its siblings: nothing here is Linux-specific (loopback HTTP and an injected stop), but no Windows lane has run it yet. + #[cfg(target_os = "linux")] + #[test] + fn the_stored_endpoint_key_reaches_the_identity_probe() { + // A keyed server answers 401 without its key and lists its model with it. + // The record's key, when one is still stored, has to be what the probe + // sends: passing no key reads the server as a stranger-or-refusal, and + // here that changes the outcome from "serving our model" to "auth refused". + let endpoint = ServingEndpoint::serving_with_authorization("amd/keyed-model"); + let (root, paths) = test_paths("uninstall-stored-key-probe"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-keyed-engine", + "vllm", + "amd/keyed-model", + "amd/keyed-model", + "127.0.0.1", + endpoint.port, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + endpoint_keys::store_endpoint_api_key(&paths, "svc-keyed-engine", "sekret") + .expect("store the endpoint key"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + let failure = report + .failed + .iter() + .find(|failure| failure.service_id == "svc-keyed-engine") + .unwrap_or_else(|| panic!("a keyed survivor must fail the gate: {report:?}")); + assert_eq!( + failure.remedy, + StopFailureRemedy::StopWhatHoldsThePort, + "with its key the probe sees the record's own model, not a refusal: {report:?}" + ); + drop(endpoint); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn only_json_files_in_the_services_dir_are_service_manifests() { + // The directory also holds non-record files (editor backups, logs); a + // non-.json file that does not parse must not read as a corrupt record. + let (root, paths) = test_paths("uninstall-non-json-manifest"); + fs::create_dir_all(paths.services_dir()).expect("create services dir"); + fs::write(paths.services_dir().join("svc-x.txt"), b"{ not json").expect("seed"); + fs::write(paths.services_dir().join("svc-y.json.bak"), b"{ not json").expect("seed"); + assert!( + unreadable_service_manifests(&paths) + .expect("scan") + .is_empty() + ); + fs::write(paths.services_dir().join("svc-z.json"), b"{ not json").expect("seed"); + assert_eq!( + unreadable_service_manifests(&paths).expect("scan"), + vec![paths.services_dir().join("svc-z.json")] + ); + let _ = fs::remove_dir_all(root); + } + + // Gated to Linux: drives a real spawned stand-in verified through its start time and zombie-state detection. + #[cfg(target_os = "linux")] + #[test] + fn a_doomed_run_stops_nothing_on_its_way_to_the_abort() { + // Stopping is not free and not undoable: each confirmed stop drops that + // service's endpoint key, and a publicly bound service has to be served + // again with an explicit flag to get one back. Once the gate is certain + // to abort — nothing removed, tooling intact — every stop performed + // first is pure cost for a removal that will not happen. + // + // The daemon failure used here is the unreadable-state one, which this + // file already documents at length as the reason to leave the helper + // alone. That same reasoning has to cover the services: it made no sense + // to spare the daemon and then stop everything else for no gain. + let server = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn managed server"); + let pid = server.id(); + let (root, paths) = test_paths("uninstall-doomed-run-stops-nothing"); + paths.ensure().expect("create the app directories"); + fs::write(paths.automation_state_path(), b"{ not json") + .expect("seed a corrupt runtime state"); + let real = rocm_core::process_start_ticks(pid).expect("start-ticks"); + let mut record = managed_record_for_pid(&paths, pid, Some(real)); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + assert!( + report.stopped.is_empty(), + "a run that is already going to abort must stop nothing: {report:?}" + ); + let mut server = server; + assert!( + server.try_wait().expect("poll the server").is_none(), + "the managed server must be left running by a run that aborts anyway" + ); + assert!( + uninstall_removal_gate(&report).is_err(), + "the unreadable runtime state must still abort the uninstall" + ); + let _ = server.kill(); + let _ = server.wait(); + let _ = fs::remove_dir_all(root); + } + + // Gated to Linux with its siblings: nothing here is Linux-specific (loopback HTTP and an injected stop), but no Windows lane has run it yet. + #[cfg(target_os = "linux")] + #[test] + fn serving_the_same_model_again_after_a_stop_does_not_doom_the_uninstall() { + // serve M -> stop -> serve M: record A "stopped", record B "ready", same + // host:port, and B's own server answers there. A must not block the run; + // B gets stopped (by the injected stop) and only then is A judged. + let endpoint = std::sync::Mutex::new(Some(ServingEndpoint::serving("amd/reused-model"))); + let port = endpoint.lock().unwrap().as_ref().unwrap().port; + let (root, paths) = test_paths("uninstall-reused-port"); + let make = |id: &str, status: &str| { + let mut record = ManagedServiceRecord::new( + &paths, + id, + "vllm", + "amd/reused-model", + "amd/reused-model", + "127.0.0.1", + port, + "managed", + 0, + None, + None, + None, + ); + record.status = status.to_owned(); + record.write().expect("write record"); + }; + make("svc-a", "stopped"); + make("svc-b", "ready"); + let calls = std::sync::Mutex::new(Vec::::new()); + + let report = stop_managed_services_with(&paths, |_, id| { + calls.lock().unwrap().push(id.to_owned()); + // Stopping B is what frees the port. + drop(endpoint.lock().unwrap().take()); + Ok(serde_json::json!({ "status": "stopped" })) + }) + .expect("the pass succeeds"); + + assert_eq!(*calls.lock().unwrap(), vec!["svc-b".to_owned()]); + assert!( + !report.failed.iter().any(|f| f.service_id == "svc-a"), + "the stopped record shares the live record's port and must not doom the run first: {report:?}" + ); + let _ = fs::remove_dir_all(root); + } + + #[cfg(unix)] + #[test] + fn the_plan_read_error_warning_only_promises_a_refusal_where_the_stop_pass_runs() { + use std::os::unix::fs::PermissionsExt; + + let (root, paths) = test_paths("uninstall-plan-read-error"); + paths.ensure().expect("create the app directories"); + // A services directory that exists but cannot be listed makes the read fail. + fs::create_dir_all(paths.services_dir()).expect("create services dir"); + fs::set_permissions(paths.services_dir(), fs::Permissions::from_mode(0o000)) + .expect("make the services dir unreadable"); + if fs::read_dir(paths.services_dir()).is_ok() { + // Running as root: permissions do not bind, so this cannot stage a failure. + fs::set_permissions(paths.services_dir(), fs::Permissions::from_mode(0o755)).ok(); + let _ = fs::remove_dir_all(root); + eprintln!("SKIPPED the_plan_read_error_warning: permissions do not bind this user"); + return; + } + + let full = build_uninstall_plan( + &paths, + &UninstallOptions { + yes: true, + force_dev_binaries: true, + ..UninstallOptions::default() + }, + ) + .expect("plan"); + let kept = build_uninstall_plan( + &paths, + &UninstallOptions { + yes: true, + keep_binaries: true, + keep_config: true, + keep_data: true, + ..UninstallOptions::default() + }, + ) + .expect("plan"); + let find = |plan: &UninstallPlan| { + plan.warnings + .iter() + .find(|w| w.contains("could not read the managed service records")) + .cloned() + .unwrap_or_default() + }; + assert!( + find(&full).contains("will refuse to remove anything"), + "{}", + find(&full) + ); + assert!( + find(&kept).contains("keeps the tooling") && !find(&kept).contains("will refuse"), + "{}", + find(&kept) + ); + // And the behaviour the warning promises: the real command, against the + // same unreadable directory, errors out and removes nothing. + let marker = paths.config_dir.join("keep-me"); + fs::create_dir_all(&paths.config_dir).expect("config dir"); + fs::write(&marker, b"x").expect("seed a planned file"); + // `keep_binaries` keeps this off the real executable-discovery path: with + // the dev-binary override a regression here would delete the shared build + // output. The services dir lives under the data dir, so it still refuses. + let outcome = crate::uninstall::uninstall_with_paths( + &paths, + &UninstallOptions { + yes: true, + keep_binaries: true, + ..UninstallOptions::default() + }, + ); + assert!( + outcome.is_err(), + "an unreadable services dir must abort uninstall" + ); + assert!(marker.exists(), "an aborted uninstall must remove nothing"); + fs::set_permissions(paths.services_dir(), fs::Permissions::from_mode(0o755)).ok(); + let _ = fs::remove_dir_all(root); + } + + // Gated to Linux with its siblings: nothing here is Linux-specific (loopback HTTP and an injected stop), but no Windows lane has run it yet. + #[cfg(target_os = "linux")] + #[test] + fn a_stopped_record_spelled_with_a_wildcard_host_still_waits_for_its_live_partner() { + // Same shape as the port-reuse case, but the stopped record says + // `0.0.0.0` and the live one `127.0.0.1`: both name the one listener. + let endpoint = std::sync::Mutex::new(Some(ServingEndpoint::serving("amd/reused-model"))); + let port = endpoint.lock().unwrap().as_ref().unwrap().port; + let (root, paths) = test_paths("uninstall-reused-port-wildcard"); + let make = |id: &str, host: &str, status: &str| { + let mut record = ManagedServiceRecord::new( + &paths, + id, + "vllm", + "amd/reused-model", + "amd/reused-model", + host, + port, + "managed", + 0, + None, + None, + None, + ); + record.status = status.to_owned(); + record.write().expect("write record"); + }; + make("svc-a", "0.0.0.0", "stopped"); + make("svc-b", "127.0.0.1", "ready"); + let report = stop_managed_services_with(&paths, |_, _| { + drop(endpoint.lock().unwrap().take()); + Ok(serde_json::json!({ "status": "stopped" })) + }) + .expect("the pass succeeds"); + // Only the stopped record is judged here, as in the sibling test above. + // The live record's own post-stop probe asks whether the freed port is + // quiet, and a concurrent test binding an ephemeral port can answer it, + // which says nothing about the host spelling this test pins. + assert!( + !report.failed.iter().any(|f| f.service_id == "svc-a"), + "a differently spelled host for the same listener must not doom the run: {report:?}" + ); + let _ = fs::remove_dir_all(root); + } + + // Gated to Linux: drives a real spawned stand-in verified through its start time and zombie-state detection. + #[cfg(target_os = "linux")] + #[test] + fn a_deferred_stopped_record_is_judged_once_its_live_partner_has_stopped() { + // The stopped record shares `::1` with the live one, so it waits for the + // live stop. The live record's own probe looks at `::1` and finds + // nothing, so it is confirmed stopped; but the stopped record's wildcard + // host also answers on IPv4 loopback, where a survivor still serves its + // model. Only the post-stop judgement of the deferred record can see it, + // so dropping that judgement lets the removal through. + let endpoint = ServingEndpoint::serving("amd/reused-model"); + let port = endpoint.port; + let (root, paths) = test_paths("uninstall-deferred-survivor"); + // The live record is held live by a real process, not by an answering + // endpoint: nothing answers on `::1`, and a record whose process is gone + // and whose endpoint is silent is demoted to stopped on load. + let supervisor = KillOnDrop( + crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn the live record's supervisor"), + ); + let supervisor_pid = supervisor.0.id(); + let make = |id: &str, host: &str, status: &str, pid: u32| { + let mut record = ManagedServiceRecord::new( + &paths, + id, + "vllm", + "amd/reused-model", + "amd/reused-model", + host, + port, + "managed", + pid, + None, + None, + None, + ); + if pid != 0 { + record.supervisor_start_ticks = rocm_core::process_start_ticks(pid); + } + record.status = status.to_owned(); + record.write().expect("write record"); + }; + make("svc-a", "::", "stopped", 0); + make("svc-b", "::1", "ready", supervisor_pid); + + let report = stop_managed_services_with(&paths, |_, _| { + Ok(serde_json::json!({ "status": "stopped" })) + }) + .expect("the pass succeeds"); + + let ids: Vec<&str> = report + .failed + .iter() + .map(|f| f.service_id.as_str()) + .collect(); + assert_eq!(ids, vec!["svc-a"], "{report:?}"); + assert_eq!( + report.failed[0].remedy, + StopFailureRemedy::StopWhatHoldsThePort + ); + assert_eq!(report.stopped, vec!["svc-b".to_owned()], "{report:?}"); + drop(endpoint); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_real_stop_that_a_survivor_outlives_says_the_key_is_gone_and_it_is() { + // A keyed service is stopped for real (the production stop, which drops + // its endpoint key), then something still answers with its model, so the + // run aborts. The abort must say the key was dropped, and the key must + // actually be gone: the message and the state asserted together. + let endpoint = ServingEndpoint::serving("amd/keyed-model"); + let (root, paths) = test_paths("uninstall-keyed-survivor"); + let mut server = crate::test_support::bare_command("sleep") + .arg("60") + .stdout(std::process::Stdio::null()) + .spawn() + .expect("spawn the managed server stand-in"); + let pid = server.id(); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-keyed", + "vllm", + "amd/keyed-model", + "amd/keyed-model", + "127.0.0.1", + endpoint.port, + "managed", + pid, + None, + None, + None, + ); + record.engine_pid = Some(pid); + record.supervisor_start_ticks = rocm_core::process_start_ticks(pid); + record.status = "ready".to_owned(); + record.write().expect("write record"); + endpoint_keys::store_endpoint_api_key(&paths, "svc-keyed", "live-secret") + .expect("store the key"); + assert!(endpoint_keys::endpoint_api_key(&paths, "svc-keyed").is_some()); + + let report = stop_managed_services_before_uninstall(&paths).expect("the pass runs"); + let message = format!( + "{:#}", + uninstall_removal_gate(&report).expect_err("the survivor aborts the run") + ); + assert!( + message.contains("the stop already dropped this service's endpoint key"), + "{message}" + ); + assert_eq!( + endpoint_keys::endpoint_api_key(&paths, "svc-keyed"), + None, + "the message claims the key was dropped: {message}" + ); + assert_stopped_within_grace( + &mut server, + "the stop that dropped the key must also have stopped the process", + ); + drop(endpoint); + let _ = fs::remove_dir_all(root); + } + + // Gated to Linux with its siblings: nothing here is Linux-specific (loopback HTTP and an injected stop), but no Windows lane has run it yet. + #[cfg(target_os = "linux")] + #[test] + fn a_failed_live_partner_is_not_named_twice_through_its_stopped_twin() { + // Same port, one record stopped and one live. When the live one's stop + // fails, its own failure explains the listener there; the stopped twin + // must not add a second, wrong "the port holder" failure. + let endpoint = ServingEndpoint::serving("amd/reused-model"); + let (root, paths) = test_paths("uninstall-failed-partner"); + let make = |id: &str, status: &str| { + let mut record = ManagedServiceRecord::new( + &paths, + id, + "vllm", + "amd/reused-model", + "amd/reused-model", + "127.0.0.1", + endpoint.port, + "managed", + 0, + None, + None, + None, + ); + record.status = status.to_owned(); + record.write().expect("write record"); + }; + make("svc-a", "stopped"); + make("svc-b", "ready"); + + let report = stop_managed_services_with(&paths, |_, _| anyhow::bail!("stop blew up")) + .expect("the pass succeeds"); + + let ids: Vec<&str> = report + .failed + .iter() + .map(|f| f.service_id.as_str()) + .collect(); + assert_eq!(ids, vec!["svc-b"], "{report:?}"); + assert_eq!(report.failed[0].remedy, StopFailureRemedy::StopTheService); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn the_daemon_stop_skips_pid_zero_and_its_own_pid() { + for pid in [0, std::process::id()] { + let (root, paths) = test_paths("uninstall-daemon-self-pid"); + let mut state = runtime_state(true, pid); + state.daemon_start_ticks = rocm_core::process_start_ticks(pid); + state.write(&paths).expect("write runtime state"); + let mut report = ManagedServiceStopReport::default(); + stop_background_helper_with(&paths, &mut report, |_, _| { + panic!("pid {pid} must never be signalled") + }); + assert!( + report.failed.is_empty() && !report.helper_stopped, + "{report:?}" + ); + let _ = fs::remove_dir_all(root); + } + } + + #[test] + fn localhost_and_loopback_name_one_listener_but_other_hosts_do_not() { + assert!(hosts_name_one_listener("localhost", "127.0.0.1")); + assert!(hosts_name_one_listener("127.0.0.1", "127.0.0.1")); + // `localhost` resolves to `::1` on many hosts (typically Windows), so it + // can name a listener on either family. + assert!(hosts_name_one_listener("localhost", "::1")); + assert!(hosts_name_one_listener("::1", "localhost")); + assert!(!hosts_name_one_listener("127.0.0.1", "::1")); + assert!(!hosts_name_one_listener("10.0.0.5", "127.0.0.1")); + assert!(!hosts_name_one_listener("localhost", "10.0.0.5")); + } + + #[test] + fn a_helper_only_stop_prints_a_success_line() { + let report = ManagedServiceStopReport { + stopped: Vec::new(), + failed: Vec::new(), + warnings: Vec::new(), + helper_stopped: true, + ..ManagedServiceStopReport::default() + }; + assert_eq!( + uninstall_removal_gate(&report).unwrap().as_deref(), + Some("stopped the background helper (rocmd)") + ); + } + + #[test] + fn the_stopped_service_advice_names_a_serve_command_that_restores_a_public_bind() { + let report = ManagedServiceStopReport { + stopped: vec!["svc-a".to_owned()], + relaunch: vec![format!( + "svc-a: `{}`", + rocm_engine_protocol::relaunch_command("0.0.0.0", true) + )], + failed: vec![FailedManagedServiceStop { + service_id: "svc-b".to_owned(), + reason: "r".to_owned(), + remedy: StopFailureRemedy::StopTheService, + }], + warnings: Vec::new(), + helper_stopped: false, + }; + let message = format!("{:#}", uninstall_removal_gate(&report).unwrap_err()); + assert!( + message.contains(&format!( + "svc-a: `{}`", + rocm_engine_protocol::relaunch_command("0.0.0.0", true) + )) && message.contains("--allow-public-bind"), + "the stopped public service gets the command that restores its bind: {message}" + ); + // The same builder the two refusals use: a keyed service gets + // `--require-api-key`, and the key itself is optional. + assert!(message.contains("--require-api-key"), "{message}"); + assert!( + message.contains(rocm_engine_protocol::RELAUNCH_KEY_CHOICE), + "{message}" + ); + } + + // Gated to Linux with its siblings: loopback HTTP plus an injected stop. + #[cfg(target_os = "linux")] + #[test] + fn the_abort_advice_for_a_stopped_service_carries_its_own_model_engine_and_port() { + // The production stop pass builds `report.relaunch` from the stopped + // record; a failure recorded afterwards (a later service that would not + // stop) is what turns the pass into an abort that prints it. + let endpoint = std::sync::Mutex::new(Some(ServingEndpoint::serving("amd/org-model-7b"))); + let port = endpoint.lock().unwrap().as_ref().unwrap().port; + let (root, paths) = test_paths("uninstall-relaunch-advice"); + let make = |id: &str, host: &str, port: u16| { + let mut record = ManagedServiceRecord::new( + &paths, + id, + "vllm", + "amd/org-model-7b", + "amd/org-model-7b", + host, + port, + "managed", + 0, + None, + None, + None, + ); + record.status = "ready".to_owned(); + record.requires_api_key = true; + record.write().expect("write record"); + }; + make("svc-ok", "0.0.0.0", port); + + let mut report = stop_managed_services_with(&paths, |_, _| { + drop(endpoint.lock().unwrap().take()); + Ok(serde_json::json!({ "status": "stopped" })) + }) + .expect("the pass succeeds"); + report.failed.push(FailedManagedServiceStop { + service_id: "svc-later".to_owned(), + reason: "r".to_owned(), + remedy: StopFailureRemedy::StopTheService, + }); + + let expected = format!( + "svc-ok: `rocm serve amd/org-model-7b --host 0.0.0.0 --port {port} \ + --allow-public-bind --require-api-key --engine vllm`" + ); + assert_eq!(report.relaunch, vec![expected.clone()], "{report:?}"); + let message = format!("{:#}", uninstall_removal_gate(&report).unwrap_err()); + assert!(message.contains(&expected), "{message}"); + assert!( + !message.contains("") && !message.contains(""), + "the advice must be runnable as printed: {message}" + ); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_survivor_behind_an_already_stopped_record_dooms_the_run_before_anything_is_stopped() { + // The retry the abort message asks for: last run's survivor reads as an + // already-stopped record whose endpoint still serves its own model. That + // verdict needs nothing this pass stops, so it must land before the live + // server is touched. + let endpoint = ServingEndpoint::serving("amd/survivor-model"); + let mut server = KillOnDrop( + crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn managed server"), + ); + let pid = server.0.id(); + let mut daemon = KillOnDrop( + crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn a verified daemon stand-in"), + ); + let daemon_pid = daemon.0.id(); + let (root, paths) = test_paths("uninstall-doomed-by-stopped-record"); + let mut survivor = ManagedServiceRecord::new( + &paths, + "svc-survivor", + "vllm", + "amd/survivor-model", + "amd/survivor-model", + "127.0.0.1", + endpoint.port, + "managed", + 0, + None, + None, + None, + ); + survivor.status = "stopped".to_owned(); + survivor.write().expect("write survivor record"); + let mut state = runtime_state(true, daemon_pid); + state.daemon_start_ticks = rocm_core::process_start_ticks(daemon_pid); + assert!(state.daemon_start_ticks.is_some(), "a verified daemon"); + state.write(&paths).expect("write runtime state"); + let mut live = managed_record_for_pid(&paths, pid, rocm_core::process_start_ticks(pid)); + live.status = "ready".to_owned(); + live.write().expect("write live record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + assert!( + report.stopped.is_empty(), + "nothing may be stopped: {report:?}" + ); + assert!( + server.0.try_wait().expect("poll the server").is_none(), + "the live server must be left running by a run that aborts anyway" + ); + assert!( + daemon.0.try_wait().expect("poll the daemon").is_none() && !report.helper_stopped, + "a verified daemon must not be force-killed by a run that aborts anyway: {report:?}" + ); + assert!( + report + .failed + .iter() + .any(|failure| failure.service_id == "svc-survivor"), + "{report:?}" + ); + assert!(uninstall_removal_gate(&report).is_err()); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_stopped_helper_is_reported_apart_from_services_and_worded_as_a_helper() { + let failed_after = ManagedServiceStopReport { + stopped: Vec::new(), + failed: vec![FailedManagedServiceStop { + service_id: "svc".to_owned(), + reason: "still ready".to_owned(), + remedy: StopFailureRemedy::StopTheService, + }], + warnings: Vec::new(), + helper_stopped: true, + ..ManagedServiceStopReport::default() + }; + let message = format!("{:#}", uninstall_removal_gate(&failed_after).unwrap_err()); + assert!( + message.contains("background helper (rocmd) was stopped"), + "{message}" + ); + assert!( + !message.contains("endpoint keys"), + "the helper has no endpoint key, and no services were stopped: {message}" + ); + + let ok = ManagedServiceStopReport { + stopped: vec!["a".to_owned()], + failed: Vec::new(), + warnings: Vec::new(), + helper_stopped: true, + ..ManagedServiceStopReport::default() + }; + assert_eq!( + uninstall_removal_gate(&ok).unwrap().as_deref(), + Some( + "stopped 1 managed service(s) before removal; stopped the background helper (rocmd)" + ) + ); + } + + #[test] + fn an_endpoint_that_answers_without_a_usable_listing_discloses_the_fail_open() { + // `ProceedUnidentified`: the port still accepts connections, the record is + // already stopped, and uninstall proceeds. The warning is the only thing + // that makes that a disclosed tradeoff. + let listener = std::net::TcpListener::bind("127.0.0.1:0").expect("bind"); + let port = listener.local_addr().expect("addr").port(); + let (root, paths) = test_paths("uninstall-unidentified-listener"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-wedged", + "vllm", + "amd/m", + "amd/m", + "127.0.0.1", + port, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + assert!(report.failed.is_empty(), "{report:?}"); + assert!( + report.warnings.iter().any( + |w| w.contains("did not answer the identity probe") && w.contains("svc-wedged") + ), + "{report:?}" + ); + drop(listener); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn an_unparseable_manifest_aborts_before_any_service_is_stopped() { + // Same shape as the daemon case and worse: a manifest that cannot be + // parsed may itself describe a live, GPU-holding server, so collecting + // it last meant every other service was already down — and its key + // already dropped — before the run turned out to be doomed. + // + // The daemon is staged live and fully verifiable here on purpose. It is + // the one thing the gate would otherwise still destroy on a doomed run: + // the manifest scan only reads, so putting the force-kill ahead of it + // terminates a healthy `rocmd` for an uninstall that removes nothing. + let server = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn managed server"); + let pid = server.id(); + let daemon = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn background helper"); + let daemon_pid = daemon.id(); + let (root, paths) = test_paths("uninstall-bad-manifest-ordering"); + paths.ensure().expect("create the app directories"); + let mut state = runtime_state(true, daemon_pid); + state.daemon_start_ticks = rocm_core::process_start_ticks(daemon_pid); + state.write(&paths).expect("write runtime state"); + let real = rocm_core::process_start_ticks(pid).expect("start-ticks"); + let mut record = managed_record_for_pid(&paths, pid, Some(real)); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + fs::write(paths.services_dir().join("svc-corrupt.json"), b"{ not json") + .expect("seed an unparseable manifest"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + assert!( + report.stopped.is_empty(), + "an unparseable manifest must abort before anything is stopped: {report:?}" + ); + let mut server = server; + let mut daemon = daemon; + assert!( + server.try_wait().expect("poll the server").is_none(), + "the healthy service must be untouched when the run aborts on another record" + ); + assert!( + daemon.try_wait().expect("poll the helper").is_none(), + "a live, verifiable rocmd must not be force-killed for a run that removes nothing" + ); + assert!( + report + .failed + .iter() + .any(|failure| failure.remedy == StopFailureRemedy::RepairTheRecord), + "the unparseable manifest must be the recorded failure: {report:?}" + ); + let _ = server.kill(); + let _ = server.wait(); + let _ = daemon.kill(); + let _ = daemon.wait(); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_record_without_a_start_time_is_a_legacy_record_where_start_times_are_readable() { + // The readable-start-time half of the predicate's contract (Linux, Windows), and no more than that. + // + // What actually prevents the conflation this predicate exists for — + // answering "can this platform report a start-time?" by reading the PID + // under inspection, which cannot tell "no `/proc` on this OS" from "that + // read just failed" — is the signature: `record_predates_start_ticks` + // takes no PID, so passing one is a compile error. It is prevented by + // construction, not caught by an assertion, and no assertion here should + // claim otherwise. + // + // The guard's end-to-end behaviour is pinned separately, by + // `uninstall_never_kills_a_daemon_pid_from_a_state_file_that_predates_\ + // start_ticks`, which stages a live unrelated process against a legacy + // record and fails if the guard is removed. + assert!( + record_predates_start_ticks(None), + "where start-times are readable, a record carrying none predates the field" + ); + assert!( + !record_predates_start_ticks(Some(1)), + "a record that carries a start-time is never a pre-upgrade record" + ); + } + + // `sleep` as a stand-in for the process that inherited the pid. + #[cfg(unix)] + #[test] + fn uninstall_never_signals_a_daemon_pid_recorded_as_already_stopped() { + // `running: false` is written by exactly one place — rocmd's clean + // shutdown, on its way out. (A daemon started without + // `--automations-enabled` returns before its first state write, so it + // never persists a false flag while alive.) The pid in such a record + // therefore belongs to a process that has already exited, and anything + // live under that number today inherited it. + // + // Windows is where this is the only defence: `process_start_ticks` is + // unset there for a record written before the upgrade, so the identity check falls back to its legacy + // `Matches` verdict and the force tree-kill lands on a stranger. That + // shape cannot be staged on Linux — identity genuinely works here — so + // this stages the same *decision*: a live pid that the identity check + // will confirm, which must still be left alone because the record says + // the daemon is stopped. + let mut stranger = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn unrelated process"); + let pid = stranger.id(); + let (root, paths) = test_paths("uninstall-daemon-recorded-stopped"); + let mut state = runtime_state(false, pid); + state.daemon_start_ticks = rocm_core::process_start_ticks(pid); + state.write(&paths).expect("write runtime state"); + + let report = + stop_managed_services_before_uninstall(&paths).expect("stop pass should succeed"); + + // `try_wait`, not `process_is_running`: a signalled child we have not + // reaped is a zombie, and a zombie still reads as running — so the + // liveness check would pass over the very kill this test exists to + // catch. An exit status here means it was signalled. + assert!( + stranger.try_wait().expect("poll the stranger").is_none(), + "a pid recorded as already stopped must never be signalled: uninstall killed an \ + unrelated process" + ); + assert!( + report.stopped.is_empty(), + "the daemon was already stopped, so nothing was stopped here: {report:?}" + ); + assert!( + report.failed.is_empty(), + "an already-stopped daemon is not an obstacle to uninstall: {report:?}" + ); + let _ = stranger.kill(); + let _ = stranger.wait(); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn every_wildcard_bind_spelling_is_probed_on_both_loopback_families() { + // A wildcard probed literally is not portable and would read as "nothing + // is serving" — the gate's failure-open direction, so it matters. + // + // Both families, not one: a wildcard record does not say which protocol + // the listener bound, and the two loopbacks are not interchangeable. A + // `::` bind with the usual `IPV6_V6ONLY=1` — the default on Windows — + // answers on `::1` and refuses `127.0.0.1`, so probing IPv4 alone reads + // a live engine as gone and waves the removal through. + for wildcard in [ + "0.0.0.0", + "::", + "[::]", + "0:0:0:0:0:0:0:0", + "*", + "", + " 0.0.0.0 ", + ] { + assert_eq!( + probe_hosts(wildcard), + vec!["127.0.0.1".to_owned(), "::1".to_owned()], + "wildcard {wildcard:?} must be probed on both loopback families" + ); + } + for literal in ["127.0.0.1", "192.168.1.10", "example.internal"] { + assert_eq!( + probe_hosts(literal), + vec![literal.to_owned()], + "a concrete host must be probed as recorded, and only there" + ); + } + // A concrete host still has to come back in a form that resolves. + // `(host, port).to_socket_addrs()` rejects a bracketed literal and + // anything padded, and the caller reads a resolution failure as "nothing + // is serving" — so handing the raw spelling through would delete the + // recovery tooling while the endpoint is live. + for (recorded, probed) in [ + ("[::1]", "::1"), + (" 127.0.0.1 ", "127.0.0.1"), + ("[FE80::1]", "fe80::1"), + ] { + assert_eq!( + probe_hosts(recorded), + vec![probed.to_owned()], + "a concrete host must be probed in a resolvable form" + ); + } + // Every probed form, wildcard expansions included, has to resolve: + // `(host, port).to_socket_addrs()` rejects a bracketed literal and + // anything padded, and the caller reads a resolution failure as "nothing + // is serving" — so handing a raw spelling through would delete the + // recovery tooling while the endpoint is live. + for recorded in ["[::1]", " 127.0.0.1 ", "[FE80::1]", "::", "0.0.0.0", "*"] { + for probed in probe_hosts(recorded) { + assert!( + std::net::ToSocketAddrs::to_socket_addrs(&(probed.as_str(), 1u16)).is_ok(), + "the probed form {probed:?} of {recorded:?} must resolve" + ); + } + } + } + + #[test] + fn only_an_uninstall_that_removes_the_recovery_tooling_stops_servers() { + // `--keep-binaries --keep-data` removes the cache alone: `rocm services + // stop` and every service record survive, so there is nothing to protect + // by force-stopping live servers. + let (root, paths) = test_paths("uninstall-recovery-tooling-scope"); + let cache_only = UninstallPlan { + actions: vec![UninstallPlanEntry { + kind: "cache", + path: paths.cache_dir.clone(), + }], + ..UninstallPlan::default() + }; + assert!( + !plan_removes_recovery_tooling(&cache_only, &paths), + "a cache-only uninstall keeps the tooling that stops servers" + ); + + let with_binaries = UninstallPlan { + actions: vec![UninstallPlanEntry { + kind: "binary", + path: PathBuf::from("/usr/local/bin/rocm"), + }], + ..UninstallPlan::default() + }; + assert!( + plan_removes_recovery_tooling(&with_binaries, &paths), + "removing the binaries takes away `rocm services stop`" + ); + + let with_data = UninstallPlan { + actions: vec![UninstallPlanEntry { + kind: "data", + path: paths.data_dir.clone(), + }], + ..UninstallPlan::default() + }; + assert!( + plan_removes_recovery_tooling(&with_data, &paths), + "removing the data dir takes away the service records" + ); + + // The other direction: an entry INSIDE the services directory removes + // service records without being (or containing) the data dir itself. + let inside_services = UninstallPlan { + actions: vec![UninstallPlanEntry { + kind: "data", + path: paths.services_dir().join("svc-a.json"), + }], + ..UninstallPlan::default() + }; + assert!( + plan_removes_recovery_tooling(&inside_services, &paths), + "removing a service record takes away what stops its server" + ); + let elsewhere = UninstallPlan { + actions: vec![UninstallPlanEntry { + kind: "cache", + path: paths.data_dir.join("unrelated").join("file"), + }], + ..UninstallPlan::default() + }; + assert!( + !plan_removes_recovery_tooling(&elsewhere, &paths), + "a path beside the services directory leaves it alone" + ); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn the_plan_does_not_promise_a_stop_for_a_live_record_the_gate_refuses() { + // A live record whose pid has no recorded start time is refused, not + // stopped. The plan is the last thing read before confirming, so it + // must say that — and the claim is only worth anything next to the + // outcome it describes, so both come from the one run. + let mut child = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn a live managed server"); + let pid = child.id(); + let (root, paths) = test_paths("uninstall-plan-tickless"); + let mut record = managed_record_for_pid(&paths, pid, None); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + + let plan = build_uninstall_plan( + &paths, + &UninstallOptions { + yes: true, + force_dev_binaries: true, + ..UninstallOptions::default() + }, + ) + .expect("build the plan"); + let refusal = plan + .warnings + .iter() + .find(|warning| warning.contains("will refuse to remove anything")) + .expect("the plan says the run will refuse"); + assert!( + refusal.contains("no start time was recorded") + && refusal.contains("will stop nothing") + && refusal.contains("svc-managed-stop"), + "the plan must say the stop will be refused and for which record: {refusal}" + ); + let count = plan + .warnings + .iter() + .find(|warning| warning.contains("managed service record(s) under")) + .expect("the plan counts the managed services"); + assert!( + count.contains("none of them will be stopped") + && !count.contains("will be stopped before removal"), + "the plan must not promise the stop the gate refuses: {count}" + ); + + let report = stop_managed_services_with(&paths, |_, _| { + panic!("the record the plan said would be refused must not be stopped") + }) + .expect("stop pass"); + assert_eq!( + report.failed.len(), + 1, + "the gate refuses exactly what the plan said it would: {report:?}" + ); + assert_eq!( + report.failed[0].remedy, + StopFailureRemedy::ConfirmTheServicePid + ); + assert!(child.try_wait().expect("poll").is_none(), "left alive"); + let _ = child.kill(); + let _ = child.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + fn full_plan_options() -> UninstallOptions { + UninstallOptions { + yes: true, + force_dev_binaries: true, + ..UninstallOptions::default() + } + } + + #[cfg(target_os = "linux")] + fn plan_warning_containing(plan: &UninstallPlan, needle: &str) -> Option { + plan.warnings + .iter() + .find(|warning| warning.contains(needle)) + .cloned() + } + + #[cfg(target_os = "linux")] + #[test] + fn an_unparseable_record_beside_a_live_one_makes_the_plan_refuse_and_the_stop_pass_stop_nothing() + { + // A live, verifiable server plus a manifest that does not parse. The plan + // used to count the live record and say "will be stopped before removal" + // while the stop pass refused on the manifest before touching anything. + let mut child = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn a live managed server"); + let pid = child.id(); + let (root, paths) = test_paths("uninstall-plan-bad-manifest"); + let mut record = managed_record_for_pid(&paths, pid, rocm_core::process_start_ticks(pid)); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + let bad = paths.services_dir().join("bad.json"); + fs::write(&bad, b"{").expect("plant an unparseable manifest"); + + let plan = build_uninstall_plan(&paths, &full_plan_options()).expect("build the plan"); + let refusal = plan_warning_containing(&plan, "will refuse to remove anything") + .expect("the plan says the run will refuse"); + assert!( + refusal.contains(&bad.display().to_string()) && refusal.contains("will stop nothing"), + "{refusal}" + ); + let count = plan_warning_containing(&plan, "managed service record(s) under") + .expect("the plan counts the managed services"); + assert!( + count.contains("none of them will be stopped") + && !count.contains("will be stopped before removal"), + "{count}" + ); + + let report = stop_managed_services_with(&paths, |_, _| { + panic!("the run the plan said would refuse must stop nothing") + }) + .expect("stop pass"); + assert!(report.stopped.is_empty(), "{report:?}"); + assert_eq!(report.failed.len(), 1, "{report:?}"); + assert_eq!(report.failed[0].remedy, StopFailureRemedy::RepairTheRecord); + assert!(child.try_wait().expect("poll").is_none(), "left alive"); + let _ = child.kill(); + let _ = child.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_verifiable_record_beside_a_tickless_one_is_not_stopped_and_the_plan_says_so() { + // The refusal returns before any stop, so "the others will be stopped" + // was never true. Two live servers, one without a start time: neither is + // stopped, and the plan says so for both. + let mut ticked = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn a verifiable server"); + let mut tickless = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn a tickless server"); + let (root, paths) = test_paths("uninstall-plan-partial-refusal"); + let mut good = managed_record_named( + &paths, + "svc-good", + ticked.id(), + rocm_core::process_start_ticks(ticked.id()), + ); + good.status = "ready".to_owned(); + good.write().expect("write the verifiable record"); + let mut bad = managed_record_named(&paths, "svc-bad", tickless.id(), None); + bad.status = "ready".to_owned(); + bad.write().expect("write the tickless record"); + + let plan = build_uninstall_plan(&paths, &full_plan_options()).expect("build the plan"); + let refusal = plan_warning_containing(&plan, "will refuse to remove anything") + .expect("the plan says the run will refuse"); + assert!( + refusal.contains("svc-bad") && !refusal.contains("svc-good"), + "{refusal}" + ); + let count = plan_warning_containing(&plan, "managed service record(s) under") + .expect("the plan counts the managed services"); + assert!( + count.contains("2 of 2") + && count.contains("none of them will be stopped") + && !count.contains("the others will be stopped"), + "{count}" + ); + + let report = stop_managed_services_with(&paths, |_, _| { + panic!("a refused run must not stop the verifiable record either") + }) + .expect("stop pass"); + assert!(report.stopped.is_empty(), "{report:?}"); + assert!(ticked.try_wait().expect("poll").is_none(), "left alive"); + for child in [&mut ticked, &mut tickless] { + let _ = child.kill(); + let _ = child.wait(); + } + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_dead_tickless_pid_beside_a_live_verifiable_one_does_not_make_the_run_refuse() { + // `unprovable_live_pids` filters on liveness: a recorded pid that is gone + // cannot be a reused one that gets killed. Dropping the filter would turn + // every record with a stale engine pid into a refusal. The plan says the + // server will be stopped, and the stop pass really does. + let child = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn the live supervisor"); + let pid = child.id(); + let mut gone = crate::test_support::bare_command("true") + .spawn() + .expect("spawn a short-lived process"); + let dead_pid = gone.id(); + gone.wait().expect("reap it"); + let (root, paths) = test_paths("uninstall-dead-tickless-engine"); + let mut record = managed_record_for_pid(&paths, pid, rocm_core::process_start_ticks(pid)); + record.engine_pid = Some(dead_pid); + record.engine_start_ticks = None; + record.status = "ready".to_owned(); + record.write().expect("write service record"); + + let plan = build_uninstall_plan(&paths, &full_plan_options()).expect("build the plan"); + assert!( + plan_warning_containing(&plan, "will refuse to remove anything").is_none(), + "{:?}", + plan.warnings + ); + let count = plan_warning_containing(&plan, "managed service record(s) under") + .expect("the plan counts the managed services"); + assert!(count.contains("will be stopped before removal"), "{count}"); + + let report = stop_managed_services_with(&paths, |_, _| { + Ok(serde_json::json!({ "status": "stopped" })) + }) + .expect("stop pass"); + assert!(report.failed.is_empty(), "{report:?}"); + assert_eq!(report.stopped, vec!["svc-managed-stop".to_owned()]); + let mut child = child; + let _ = child.kill(); + let _ = child.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_failed_record_whose_engine_is_still_alive_is_stopped_not_waved_through() { + // `refresh_from_engine_state` adopts `failed` while the engine process is + // still up, and the liveness refresh does not look at pids for a non-live + // status. The status alone read as "nothing to stop", and the tooling was + // removed under a process that may still hold the GPU. + let child = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn the engine"); + let pid = child.id(); + let (root, paths) = test_paths("uninstall-failed-record-live-engine"); + let mut record = managed_record_for_pid(&paths, pid, rocm_core::process_start_ticks(pid)); + record.status = "failed".to_owned(); + record.write().expect("write service record"); + + let plan = build_uninstall_plan(&paths, &full_plan_options()).expect("build the plan"); + let count = plan_warning_containing(&plan, "managed service record(s) under") + .expect("the plan counts the managed services"); + assert!( + count.contains("1 of 1") && count.contains("will be stopped before removal"), + "{count}" + ); + + let report = stop_managed_services_before_uninstall(&paths).expect("stop pass"); + assert_eq!( + report.stopped, + vec!["svc-managed-stop".to_owned()], + "{report:?}" + ); + let mut child = child; + assert_stopped_within_grace( + &mut child, + "the failed record's live engine must be stopped", + ); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_stopped_record_with_a_reused_live_pid_is_left_to_the_port_probe() { + // The narrower half of the same predicate: a record that reads `stopped` + // is written only once its processes are confirmed gone, so a live pid + // under it is a reused number. Refusing on it would block every uninstall + // that finds an old, tickless record. + let mut child = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn an unrelated process"); + let (root, paths) = test_paths("uninstall-stopped-record-reused-pid"); + let mut record = managed_record_for_pid(&paths, child.id(), None); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let report = stop_managed_services_with(&paths, |_, _| { + panic!("a stopped record is not stopped again") + }) + .expect("stop pass"); + assert!(report.failed.is_empty(), "{report:?}"); + assert!(child.try_wait().expect("poll").is_none(), "left alone"); + let _ = child.kill(); + let _ = child.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_stop_that_does_not_confirm_is_a_failure_whatever_the_status_says() { + // The decision is the confirmed flag. A supervisor's clean-exit write + // leaves the record reading `stopped` while a grandchild is still alive; + // the status string then says "stopped" for a stop that confirmed nothing. + let child = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn the managed server"); + let pid = child.id(); + let (root, paths) = test_paths("uninstall-unconfirmed-stop"); + let mut record = managed_record_for_pid(&paths, pid, rocm_core::process_start_ticks(pid)); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + + let report = stop_managed_services_confirmed_with(&paths, |_, _| { + Ok((serde_json::json!({ "status": "stopped" }), false)) + }) + .expect("stop pass"); + assert!(report.stopped.is_empty(), "{report:?}"); + assert_eq!(report.failed.len(), 1, "{report:?}"); + assert_eq!(report.failed[0].remedy, StopFailureRemedy::StopTheService); + assert!( + report.failed[0].reason.contains("could not confirm"), + "{}", + report.failed[0].reason + ); + + // And the converse: a confirmed stop counts whatever the report's status + // string says, because the flag is what the stop actually established. + let report = + stop_managed_services_confirmed_with(&paths, |_, _| Ok((serde_json::json!({}), true))) + .expect("stop pass"); + assert_eq!( + report.stopped, + vec!["svc-managed-stop".to_owned()], + "{report:?}" + ); + assert!(report.failed.is_empty(), "{report:?}"); + let mut child = child; + let _ = child.kill(); + let _ = child.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_stop_that_finds_the_record_already_stopped_still_reports_it_unconfirmed() { + // The premise of the test above, on the real stop: a record that reads + // `stopped` keeps that status through a stop whose termination did not + // confirm. Only the flag distinguishes it from a confirmed stop. + let (root, paths) = test_paths("uninstall-stop-keeps-status"); + let mut record = managed_record_for_pid(&paths, std::process::id(), None); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let (report, confirmed) = + stop_internal_managed_service_confirmed_with(&paths, "svc-managed-stop", |_| { + (Vec::new(), false) + }) + .expect("stop"); + assert_eq!(report["status"], "stopped"); + assert!( + !confirmed, + "an unconfirmed termination is not a confirmed stop" + ); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn an_unreadable_helper_state_makes_the_plan_refuse_and_the_stop_pass_say_which_file() { + let (root, paths) = test_paths("uninstall-plan-helper-state-unreadable"); + let state_file = paths.automation_state_path(); + fs::create_dir_all(state_file.parent().expect("a parent")).expect("create the dir"); + fs::write(&state_file, b"{ not json").expect("plant an unparseable state file"); + + let plan = build_uninstall_plan(&paths, &full_plan_options()).expect("build the plan"); + let refusal = plan_warning_containing(&plan, "will refuse to remove anything") + .expect("the plan says the run will refuse"); + assert!( + refusal.contains(&state_file.display().to_string()) + && refusal.contains("could not be read"), + "{refusal}" + ); + let report = stop_managed_services_before_uninstall(&paths).expect("stop pass"); + assert_eq!(report.failed.len(), 1, "{report:?}"); + assert_eq!( + report.failed[0].remedy, + StopFailureRemedy::RepairTheDaemonState + ); + + // A run that keeps the tooling does not check, and says so. + let cache_only = UninstallOptions { + yes: true, + keep_binaries: true, + keep_config: true, + keep_data: true, + ..UninstallOptions::default() + }; + let plan = build_uninstall_plan(&paths, &cache_only).expect("build the cache plan"); + assert!( + plan_warning_containing(&plan, "will refuse to remove anything").is_none(), + "{:?}", + plan.warnings + ); + let note = plan_warning_containing(&plan, "background helper") + .expect("the cache-only plan names the unreadable state"); + assert!(note.contains("does not check it"), "{note}"); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_recycled_helper_pid_is_neither_planned_nor_stopped() { + // The recorded pid is alive but its start time does not match: that is a + // reused number, so there is nothing of ours to stop. The plan must not + // say "will be stopped", and the stop pass must leave the process alone. + let mut child = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn an unrelated process"); + let pid = child.id(); + let (root, paths) = test_paths("uninstall-helper-recycled"); + let mut state = runtime_state(true, pid); + state.daemon_start_ticks = rocm_core::process_start_ticks(pid).map(|t| t.wrapping_add(1)); + state.write(&paths).expect("write runtime state"); + + let plan = build_uninstall_plan(&paths, &full_plan_options()).expect("build the plan"); + assert!( + plan_warning_containing(&plan, "background helper").is_none(), + "{:?}", + plan.warnings + ); + let report = stop_managed_services_before_uninstall(&paths).expect("stop pass"); + assert!( + !report.helper_stopped && report.failed.is_empty(), + "{report:?}" + ); + assert!(child.try_wait().expect("poll").is_none(), "left alone"); + let _ = child.kill(); + let _ = child.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_helper_recorded_as_not_running_is_neither_planned_nor_stopped() { + // `running: false` is the clean-shutdown write: the pid in the file names + // a process that already exited, and whatever is alive under it now is + // someone else's — even when its start time happens to match. + let mut child = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn an unrelated process"); + let pid = child.id(); + let (root, paths) = test_paths("uninstall-helper-not-running"); + let mut state = runtime_state(false, pid); + state.daemon_start_ticks = rocm_core::process_start_ticks(pid); + state.write(&paths).expect("write runtime state"); + + let plan = build_uninstall_plan(&paths, &full_plan_options()).expect("build the plan"); + assert!( + plan_warning_containing(&plan, "background helper").is_none(), + "{:?}", + plan.warnings + ); + let report = stop_managed_services_before_uninstall(&paths).expect("stop pass"); + assert!( + !report.helper_stopped && report.failed.is_empty(), + "{report:?}" + ); + assert!(child.try_wait().expect("poll").is_none(), "left alone"); + let _ = child.kill(); + let _ = child.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn the_helper_is_stopped_with_the_whole_tree_scope() { + // The production stop hands the termination `HELPER_KILL_SCOPE`; a + // single-process kill would leave the helper's engine children holding + // the GPU. The injected termination sees the scope the real one would. + let mut child = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn the helper stand-in"); + let pid = child.id(); + let (root, paths) = test_paths("uninstall-helper-scope"); + let mut state = runtime_state(true, pid); + state.daemon_start_ticks = rocm_core::process_start_ticks(pid); + state.write(&paths).expect("write runtime state"); + + let mut seen = None; + let mut report = ManagedServiceStopReport::default(); + stop_background_helper_with(&paths, &mut report, |_, scope| { + seen = Some(scope); + rocm_core::TerminationOutcome::Graceful + }); + assert_eq!(seen, Some(rocm_core::KillScope::Tree)); + assert_eq!(seen, Some(HELPER_KILL_SCOPE)); + assert!(report.helper_stopped, "{report:?}"); + let _ = child.kill(); + let _ = child.wait(); + let _ = fs::remove_dir_all(root); + } + + /// A live helper whose identity verifies: the one state in which the plan's + /// helper line can promise a stop. + #[cfg(target_os = "linux")] + fn plant_verified_helper(paths: &AppPaths) -> std::process::Child { + let helper = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn the helper stand-in"); + let mut state = runtime_state(true, helper.id()); + state.daemon_start_ticks = rocm_core::process_start_ticks(helper.id()); + state.write(paths).expect("write runtime state"); + helper + } + + #[cfg(target_os = "linux")] + #[test] + fn a_verified_helper_is_left_running_by_a_run_that_refuses_on_a_tickless_service() { + // The plan is the last thing read before confirming. With a verified live + // helper and a service whose live pid has no recorded start time, the run + // refuses before stopping anything, so the helper must not be described as + // about to be stopped, and the stop pass must leave it alone: no helper + // termination, no per-service stop. + let mut helper = None; + let (root, paths) = test_paths("uninstall-helper-with-refusing-service"); + helper.replace(plant_verified_helper(&paths)); + let mut server = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn a live managed server"); + let mut record = managed_record_for_pid(&paths, server.id(), None); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + + let plan = build_uninstall_plan(&paths, &full_plan_options()).expect("build the plan"); + let note = plan_warning_containing(&plan, "background helper (rocmd") + .expect("the plan names the live helper"); + assert!( + note.contains("left running") && !note.contains("will be stopped"), + "the helper line must not promise a stop on a run that refuses: {note}" + ); + assert!( + plan_warning_containing(&plan, "will refuse to remove anything").is_some(), + "{:?}", + plan.warnings + ); + + let stops = std::sync::atomic::AtomicUsize::new(0); + let terminations = std::sync::atomic::AtomicUsize::new(0); + let report = stop_managed_services_with_helper_stop( + &paths, + |_, _| { + stops.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok((serde_json::json!({ "status": "stopped" }), true)) + }, + |_, _| { + terminations.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + rocm_core::TerminationOutcome::Graceful + }, + ) + .expect("stop pass"); + assert!( + report + .failed + .iter() + .any(|failure| failure.remedy == StopFailureRemedy::ConfirmTheServicePid), + "the run refuses exactly as the plan said: {report:?}" + ); + assert_eq!(stops.load(std::sync::atomic::Ordering::SeqCst), 0); + assert_eq!(terminations.load(std::sync::atomic::Ordering::SeqCst), 0); + assert!(!report.helper_stopped, "{report:?}"); + let mut helper = helper.take().expect("helper"); + assert!(helper.try_wait().expect("poll").is_none(), "left running"); + let _ = helper.kill(); + let _ = helper.wait(); + let _ = server.kill(); + let _ = server.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_verified_helper_is_left_running_by_a_run_whose_service_records_cannot_be_read() { + // A `*.json` entry that is a directory cannot be read, as root or not, so + // the plan's own read of the records fails and the stop pass errors. The + // helper line must say the same thing the abort does: left running. + let (root, paths) = test_paths("uninstall-helper-with-unreadable-records"); + let mut helper = plant_verified_helper(&paths); + fs::create_dir_all(paths.services_dir().join("boom.json")) + .expect("plant an unreadable entry"); + + let plan = build_uninstall_plan(&paths, &full_plan_options()).expect("build the plan"); + assert!( + plan_warning_containing(&plan, "could not read the managed service records") + .is_some_and(|line| line.contains("will refuse to remove anything")), + "{:?}", + plan.warnings + ); + let note = plan_warning_containing(&plan, "background helper (rocmd") + .expect("the plan names the live helper"); + assert!( + note.contains("left running") && !note.contains("will be stopped"), + "the helper line must not promise a stop on a run that aborts: {note}" + ); + + let terminations = std::sync::atomic::AtomicUsize::new(0); + let outcome = stop_managed_services_with_helper_stop( + &paths, + |_, _| panic!("no service may be stopped"), + |_, _| { + terminations.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + rocm_core::TerminationOutcome::Graceful + }, + ); + assert!(outcome.is_err(), "the unreadable records abort the pass"); + assert_eq!(terminations.load(std::sync::atomic::Ordering::SeqCst), 0); + assert!(helper.try_wait().expect("poll").is_none(), "left running"); + let _ = helper.kill(); + let _ = helper.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_verified_helper_is_stopped_by_a_run_that_proceeds_and_the_plan_says_so() { + let (root, paths) = test_paths("uninstall-helper-proceeds"); + let mut helper = plant_verified_helper(&paths); + let plan = build_uninstall_plan(&paths, &full_plan_options()).expect("build the plan"); + let note = plan_warning_containing(&plan, "background helper (rocmd") + .expect("the plan names the live helper"); + assert!(note.contains("will be stopped before removal"), "{note}"); + let terminations = std::sync::atomic::AtomicUsize::new(0); + let report = stop_managed_services_with_helper_stop( + &paths, + |_, _| panic!("no services"), + |_, _| { + terminations.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + rocm_core::TerminationOutcome::Graceful + }, + ) + .expect("stop pass"); + assert_eq!(terminations.load(std::sync::atomic::Ordering::SeqCst), 1); + assert!( + report.helper_stopped && report.failed.is_empty(), + "{report:?}" + ); + let _ = helper.kill(); + let _ = helper.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn an_unverifiable_helper_is_named_by_a_plan_that_also_cannot_read_the_records() { + // The records-read error aborts the run before the helper is looked at, so + // the pre-flight refusal line is absent; the helper line must still name the + // unverifiable helper and promise a refusal, never a stop. + let (root, paths) = test_paths("uninstall-unverifiable-helper-and-unreadable"); + let mut helper = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn the helper stand-in"); + let mut state = runtime_state(true, helper.id()); + state.daemon_start_ticks = None; + state.write(&paths).expect("write runtime state"); + fs::create_dir_all(paths.services_dir().join("boom.json")) + .expect("plant an unreadable entry"); + + let plan = build_uninstall_plan(&paths, &full_plan_options()).expect("build the plan"); + let line = plan + .warnings + .iter() + .find(|line| line.contains("rocmd") && line.contains("identity could not be verified")) + .unwrap_or_else(|| panic!("the helper is named: {:?}", plan.warnings)); + assert!( + line.contains("will refuse to remove anything") && !line.contains("will be stopped"), + "{line}" + ); + let _ = helper.kill(); + let _ = helper.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_helper_that_cannot_be_confirmed_stopped_stops_no_service_after_it() { + // The helper is stopped first so it cannot respawn what the services pass + // stops; if that stop does not confirm, no service may be stopped, since + // each stop drops that service's endpoint key for good. + let (root, paths) = test_paths("uninstall-helper-timeout-stops-no-service"); + let mut helper = plant_verified_helper(&paths); + let mut server = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn a live managed server"); + let ticks = rocm_core::process_start_ticks(server.id()); + let mut record = managed_record_for_pid(&paths, server.id(), ticks); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + + let stops = std::sync::atomic::AtomicUsize::new(0); + let report = stop_managed_services_with_helper_stop( + &paths, + |_, _| { + stops.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + Ok((serde_json::json!({ "status": "stopped" }), true)) + }, + |_, _| rocm_core::TerminationOutcome::TimedOut, + ) + .expect("stop pass"); + assert!( + report + .failed + .iter() + .any(|failure| failure.remedy == StopFailureRemedy::StopTheDaemon), + "{report:?}" + ); + assert_eq!( + stops.load(std::sync::atomic::Ordering::SeqCst), + 0, + "no service may be stopped once the helper stop failed: {report:?}" + ); + assert!(report.stopped.is_empty(), "{report:?}"); + let _ = helper.kill(); + let _ = helper.wait(); + let _ = server.kill(); + let _ = server.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_resolvable_host_with_nothing_listening_is_not_a_fail_open() { + // `host_port_resolves` separates "nothing is listening" (the normal, + // silent case) from "the name does not resolve" (evidence of nothing, + // disclosed). A host that resolves must not draw the disclosure. + let (root, paths) = test_paths("uninstall-resolvable-dead-endpoint"); + let mut record = managed_record_for_pid(&paths, std::process::id(), None); + record.status = "stopped".to_owned(); + record.host = "127.0.0.1".to_owned(); + record.port = 0; + record.write().expect("write service record"); + + let report = stop_managed_services_with(&paths, |_, _| { + panic!("a stopped record is not stopped again") + }) + .expect("stop pass"); + assert!(report.failed.is_empty(), "{report:?}"); + assert!( + report.warnings.is_empty(), + "a refused connect is evidence the port is free, not a fail-open: {:?}", + report.warnings + ); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn following_the_tickless_pid_advice_lets_the_retry_proceed() { + // The advice for a live pid with no start time: stop it yourself, or + // delete the record if it is not your server. Either has to clear the + // refusal. + let mut stopped_by_hand = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn a tickless server"); + let (root, paths) = test_paths("uninstall-advice-tickless-stop"); + let mut record = managed_record_for_pid(&paths, stopped_by_hand.id(), None); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + let first = + stop_managed_services_with(&paths, |_, _| panic!("must not stop")).expect("first pass"); + assert_eq!(first.failed.len(), 1, "{first:?}"); + + stopped_by_hand.kill().expect("`kill `"); + stopped_by_hand.wait().expect("reap it"); + let second = stop_managed_services_with(&paths, |_, _| panic!("nothing left to stop")) + .expect("retry after killing the pid"); + assert!(second.failed.is_empty(), "{second:?}"); + let _ = fs::remove_dir_all(root); + + // Or: it was not our server, so delete the record file by hand. + let mut stranger = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn an unrelated process"); + let (root, paths) = test_paths("uninstall-advice-tickless-delete"); + let mut record = managed_record_for_pid(&paths, stranger.id(), None); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + let first = + stop_managed_services_with(&paths, |_, _| panic!("must not stop")).expect("first pass"); + assert_eq!(first.failed.len(), 1, "{first:?}"); + fs::remove_file(paths.services_dir().join("svc-managed-stop.json")) + .expect("delete the record file"); + let second = stop_managed_services_with(&paths, |_, _| panic!("no record left")) + .expect("retry after deleting the record"); + assert!(second.failed.is_empty(), "{second:?}"); + assert!(stranger.try_wait().expect("poll").is_none(), "left alone"); + let _ = stranger.kill(); + let _ = stranger.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn following_the_daemon_pid_advice_lets_the_retry_proceed() { + // The advice for a helper that cannot be proven to be rocmd: stop it if + // it is rocmd, or delete the runtime state file if it is something else. + let mut helper = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn the helper stand-in"); + let (root, paths) = test_paths("uninstall-advice-daemon-stop"); + runtime_state(true, helper.id()) + .write(&paths) + .expect("write runtime state"); + let first = stop_managed_services_before_uninstall(&paths).expect("first pass"); + assert_eq!(first.failed.len(), 1, "{first:?}"); + assert_eq!( + first.failed[0].remedy, + StopFailureRemedy::ConfirmTheDaemonPid + ); + + helper.kill().expect("`kill `"); + helper.wait().expect("reap it"); + let second = stop_managed_services_before_uninstall(&paths).expect("retry after the kill"); + assert!(second.failed.is_empty(), "{second:?}"); + let _ = fs::remove_dir_all(root); + + let mut stranger = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn an unrelated process"); + let (root, paths) = test_paths("uninstall-advice-daemon-delete"); + runtime_state(true, stranger.id()) + .write(&paths) + .expect("write runtime state"); + let first = stop_managed_services_before_uninstall(&paths).expect("first pass"); + assert_eq!(first.failed.len(), 1, "{first:?}"); + fs::remove_file(paths.automation_state_path()).expect("delete the state file"); + let second = stop_managed_services_before_uninstall(&paths) + .expect("retry after deleting the state file"); + assert!( + second.failed.is_empty() && !second.helper_stopped, + "{second:?}" + ); + assert!(stranger.try_wait().expect("poll").is_none(), "left alone"); + let _ = stranger.kill(); + let _ = stranger.wait(); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn the_abort_names_what_the_run_already_stopped_and_the_helper_it_signalled() { + let report = ManagedServiceStopReport { + stopped: vec!["svc-down".to_owned()], + relaunch: vec![format!( + "svc-down: `{}`", + rocm_engine_protocol::relaunch_command("0.0.0.0", true) + )], + failed: vec![ + FailedManagedServiceStop { + service_id: "svc-stuck".to_owned(), + reason: "still \"ready\"".to_owned(), + remedy: StopFailureRemedy::StopTheService, + }, + FailedManagedServiceStop { + service_id: "rocmd (pid 7)".to_owned(), + reason: "not confirmed".to_owned(), + remedy: StopFailureRemedy::StopTheDaemon, + }, + ], + warnings: Vec::new(), + helper_stopped: false, + }; + let message = format!("{:#}", uninstall_removal_gate(&report).unwrap_err()); + assert!( + message.contains("stopped during this run and stay stopped: svc-down"), + "{message}" + ); + assert!(!message.contains("before the failure"), "{message}"); + assert!( + message.contains("signalled but not confirmed gone"), + "{message}" + ); + assert!(message.contains("--require-api-key"), "{message}"); + assert!(message.contains("--allow-public-bind"), "{message}"); + assert!(message.contains("ROCM_SERVE_API_KEY"), "{message}"); + } + + #[cfg(target_os = "linux")] + #[test] + fn the_plan_reports_a_live_helper_even_when_no_service_record_exists() { + // A live `rocmd` is force-stopped whether or not any service record + // exists, so the plan cannot stay silent just because there are none. + let helper = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn the helper stand-in"); + let pid = helper.id(); + let (root, paths) = test_paths("uninstall-plan-helper-only"); + let mut state = runtime_state(true, pid); + state.daemon_start_ticks = rocm_core::process_start_ticks(pid); + state.write(&paths).expect("write runtime state"); + let full = UninstallOptions { + yes: true, + force_dev_binaries: true, + ..UninstallOptions::default() + }; + let cache_only = UninstallOptions { + yes: true, + keep_binaries: true, + keep_config: true, + keep_data: true, + ..UninstallOptions::default() + }; + + let plan = build_uninstall_plan(&paths, &full).expect("build the full plan"); + let note = plan + .warnings + .iter() + .find(|warning| warning.contains("background helper")) + .expect("the plan names the live helper"); + assert!( + note.contains(&format!("pid {pid}")) && note.contains("will be stopped"), + "{note}" + ); + let cache_plan = build_uninstall_plan(&paths, &cache_only).expect("build the cache plan"); + let cache_note = cache_plan + .warnings + .iter() + .find(|warning| warning.contains("background helper")) + .expect("the cache-only plan names the live helper"); + assert!( + cache_note.contains("left running") && !cache_note.contains("will be stopped"), + "{cache_note}" + ); + + // The outcome the full plan promised. + let report = stop_managed_services_before_uninstall(&paths).expect("stop pass"); + assert!(report.helper_stopped, "{report:?}"); + let mut helper = helper; + assert_stopped_within_grace(&mut helper, "the helper the plan named must be stopped"); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn the_plan_says_an_unverifiable_helper_will_block_the_uninstall() { + // A helper whose state file carries no start time cannot be proven to be + // rocmd, so the gate refuses rather than kill it; the plan says so. + let helper = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn the helper stand-in"); + let pid = helper.id(); + let (root, paths) = test_paths("uninstall-plan-helper-unverifiable"); + runtime_state(true, pid) + .write(&paths) + .expect("write runtime state"); + let plan = build_uninstall_plan( + &paths, + &UninstallOptions { + yes: true, + force_dev_binaries: true, + ..UninstallOptions::default() + }, + ) + .expect("build the plan"); + let note = plan + .warnings + .iter() + .find(|warning| warning.contains("will refuse to remove anything")) + .expect("the plan names the unverifiable helper"); + assert!( + note.contains("could not be verified") && note.contains(&format!("pid {pid}")), + "{note}" + ); + let report = stop_managed_services_before_uninstall(&paths).expect("stop pass"); + assert!( + report + .failed + .iter() + .any(|f| f.remedy == StopFailureRemedy::ConfirmTheDaemonPid), + "the gate refuses exactly what the plan said it would: {report:?}" + ); + let mut helper = helper; + let _ = helper.kill(); + let _ = helper.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn removing_the_stopped_record_as_advised_lets_the_retry_proceed() { + // The advice for an authenticated stranger on a stopped record's old + // port is `rocm services remove --yes`. Following it has to clear + // the refusal; advice that left the next run aborting would be a dead + // end. The record is removed the way that command does, by deleting its + // manifest, and the same pass is run again. + let endpoint = ServingEndpoint::serving_with_authorization("amd/guarded-model"); + let (root, paths) = test_paths("uninstall-advice-then-proceed"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-stranger-port", + "vllm", + "amd/guarded-model", + "amd/guarded-model", + "127.0.0.1", + endpoint.port, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let first = stop_managed_services_before_uninstall(&paths).expect("first pass"); + assert!( + first + .failed + .iter() + .any(|f| f.remedy == StopFailureRemedy::RemoveTheStoppedRecord), + "{first:?}" + ); + + remove_managed_service_record(&paths, "svc-stranger-port", true) + .expect("the advised command must succeed"); + + let second = stop_managed_services_before_uninstall(&paths).expect("second pass"); + assert!( + second.failed.is_empty(), + "once the record is removed the retry must proceed: {second:?}" + ); + assert!(uninstall_removal_gate(&second).is_ok()); + drop(endpoint); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn the_plan_warning_only_promises_a_stop_the_uninstall_will_perform() { + // The plan is printed and confirmed BEFORE the stop pass decides whether + // to run, so a warning that promises a stop on a run that keeps the + // tooling would have the operator confirm work that never happens. + // Asserting the predicate alone (above) cannot catch that drift; this + // pins the operator-visible text to the same condition. + let child = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn a live managed server"); + let pid = child.id(); + let (root, paths) = test_paths("uninstall-warning-matches-behaviour"); + let mut record = managed_record_for_pid(&paths, pid, rocm_core::process_start_ticks(pid)); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + + let full = build_uninstall_plan( + &paths, + &UninstallOptions { + yes: true, + force_dev_binaries: true, + ..UninstallOptions::default() + }, + ) + .expect("build the full plan"); + let full_warning = full + .warnings + .iter() + .find(|warning| warning.contains("managed service record")) + .expect("the plan warns about managed services"); + assert!( + full_warning.contains("will be stopped before removal"), + "a removal that takes the tooling away must promise the stop: {full_warning}" + ); + + let cache_only = build_uninstall_plan( + &paths, + &UninstallOptions { + yes: true, + keep_binaries: true, + keep_config: true, + keep_data: true, + ..UninstallOptions::default() + }, + ) + .expect("build the cache-only plan"); + let cache_warning = cache_only + .warnings + .iter() + .find(|warning| warning.contains("managed service record")) + .expect("the plan warns about managed services"); + assert!( + cache_warning.contains("left running"), + "a removal that keeps the tooling must not promise a stop: {cache_warning}" + ); + assert!( + !cache_warning.contains("will be stopped before removal"), + "a removal that keeps the tooling must not promise a stop: {cache_warning}" + ); + + let mut child = child; + let _ = child.kill(); + let _ = child.wait(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(unix)] + #[test] + fn the_plan_warning_for_only_stopped_records_promises_no_stop() { + let (root, paths) = test_paths("uninstall-warning-stopped-records"); + let mut record = managed_record_for_pid(&paths, u32::MAX - 1, None); + record.status = "stopped".to_owned(); + record.write().expect("write service record"); + + let plan = build_uninstall_plan( + &paths, + &UninstallOptions { + yes: true, + force_dev_binaries: true, + ..UninstallOptions::default() + }, + ) + .expect("build the plan"); + let warning = plan + .warnings + .iter() + .find(|warning| warning.contains("managed service record")) + .expect("the plan warns about managed services"); + assert!( + warning.contains("none is recorded as running"), + "records that are not live must be described as such: {warning}" + ); + assert!( + !warning.contains("will be stopped"), + "no stop is promised when nothing is live: {warning}" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn uninstall_removal_gate_reports_count_when_all_stopped() { + let report = ManagedServiceStopReport { + stopped: vec!["a".to_owned(), "b".to_owned()], + failed: Vec::new(), + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }; + let line = uninstall_removal_gate(&report).expect("all stopped must proceed"); + assert_eq!( + line.as_deref(), + Some("stopped 2 managed service(s) before removal") + ); + } + + #[test] + fn uninstall_removal_gate_is_silent_with_nothing_to_stop() { + let report = ManagedServiceStopReport::default(); + assert!( + uninstall_removal_gate(&report) + .expect("no services must proceed") + .is_none(), + "no managed services means no line to print" + ); + } + + pub(crate) fn test_paths(name: &str) -> (PathBuf, AppPaths) { + let root = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .join("..") + .join(".rocm-work") + .join("tests") + .join("main") + .join(format!( + "rocm-cli-main-test-{name}-{}", + crate::test_support::unique_suffix() + )); + let _ = fs::remove_dir_all(&root); + ( + root.clone(), + AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }, + ) + } + + #[test] + fn test_paths_gives_each_call_its_own_root_even_for_one_label() { + crate::test_support::assert_each_call_gets_its_own_root(|| test_paths("same-label").0); + } + + // ---- Phase 9: reroute dispatch (bare `rocm` + interactive `rocm chat`) ---- + // + // The interactive branches require a real TTY (`interactive_terminal()`), + // which is unavailable in CI, and the dash visuals are trust-prior. These + // tests instead PROVE the dispatch TARGET changed: the two interactive + // handlers now call `dash::run_chat` and no longer call `tui::run`. We read + // this source file at test time and assert on the handler bodies. + fn main_rs_source() -> String { + let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("src") + .join("main.rs"); + fs::read_to_string(path).expect("read apps/rocm/src/main.rs") + } + + /// Extract the body of `fn launch_default()` (brace-balanced). + fn launch_default_body(src: &str) -> String { + slice_braced_block(src, "fn launch_default() -> Result<()> {") + } + + /// Extract the whole `Some(Command::Chat { .. }) => { .. }` handler arm, + /// brace-balanced from the destructure pattern through the block body, so + /// both the destructured fields (e.g. `chat_mock`) and the dispatch code are + /// visible. Restricts to the production handler (the `=> {` form), not the + /// variant declaration which has no `=>`. + fn command_chat_handler_body(src: &str) -> String { + // Anchor on the handler arm specifically: the destructure that is + // immediately followed (after the closing `}` + `)`) by `=> {`. + let arm = "Some(Command::Chat {"; + let arm_at = src.find(arm).expect("Command::Chat handler present"); + // Balance from the `{` of the destructure to capture the full arm, + // including the `=> { ... }` block that follows. + slice_braced_to_arm_end(&src[arm_at..]) + } + + /// Given a slice beginning at `Some(Command::Chat {`, return the text from + /// that point through the end of the arm's `=> { .. }` block. + fn slice_braced_to_arm_end(src: &str) -> String { + // 1. balance the destructure `{ .. }`. + let pat_open = src.find('{').expect("destructure open brace"); + let bytes = src.as_bytes(); + let mut depth = 0usize; + let mut pat_end = pat_open; + for (i, &b) in bytes.iter().enumerate().skip(pat_open) { + match b { + b'{' => depth += 1, + b'}' => { + depth -= 1; + if depth == 0 { + pat_end = i + 1; + break; + } + } + _ => {} + } + } + // 2. find the `=> {` block opener after the destructure and balance it. + let after = &src[pat_end..]; + let block_rel = after.find("=> {").expect("arm block opener `=> {`"); + let block_body = slice_braced_block(after, &after[block_rel..block_rel + 4]); + // Return destructure + block body together. + format!("{}{}", &src[pat_open..pat_end], block_body) + } + + /// Return the brace-balanced block that follows `opener` (which must end in + /// `{`), including the contents but excluding the trailing brace's tail. + fn slice_braced_block(src: &str, opener: &str) -> String { + let start = src.find(opener).unwrap_or_else(|| { + panic!("opener not found: {opener}"); + }); + let brace_start = start + opener.len() - 1; // index of the `{` + let bytes = src.as_bytes(); + let mut depth = 0usize; + let mut end = brace_start; + for (i, &b) in bytes.iter().enumerate().skip(brace_start) { + match b { + b'{' => depth += 1, + b'}' => { + depth -= 1; + if depth == 0 { + end = i + 1; + break; + } + } + _ => {} + } + } + src[brace_start..end].to_string() + } + + /// Strip `//` line comments so assertions key on real code, not the prose + /// comments that still mention `tui::run` for documentation. + fn strip_line_comments(block: &str) -> String { + block + .lines() + .map(|line| match line.find("//") { + Some(idx) => &line[..idx], + None => line, + }) + .collect::>() + .join("\n") + } + + #[test] + fn dash_run_chat_is_the_bare_and_chat_dispatch_target() { + // Symbol exists with the expected signature (bare `rocm` / `rocm chat` + // interactive target). Coercing to the fn pointer fails to compile if + // the entrypoint is removed or its signature drifts; casting the pointer + // to an address gives the binding a real effect (no underscore-bind). + let target: fn(bool, ChatInferenceParams, bool) -> Result<()> = dash::run_chat; + assert_ne!(target as usize, 0); + } + + #[test] + fn launch_default_routes_to_dash_run_chat_not_tui() { + let src = main_rs_source(); + let body = strip_line_comments(&launch_default_body(&src)); + assert!( + body.contains("dash::run_launcher(false)"), + "bare `rocm` interactive branch must route to the launcher; body:\n{body}" + ); + assert!( + !body.contains("tui::run"), + "launch_default must NOT call tui::run after the reroute; body:\n{body}" + ); + } + + #[test] + fn command_chat_interactive_routes_to_dash_run_chat_not_tui() { + let src = main_rs_source(); + let body = strip_line_comments(&command_chat_handler_body(&src)); + assert!( + body.contains("dash::run_chat("), + "interactive `rocm chat` must route to dash::run_chat(...); body:\n{body}" + ); + assert!( + !body.contains("tui::run"), + "Command::Chat handler must NOT call tui::run after the reroute; body:\n{body}" + ); + } + + #[test] + fn command_chat_interactive_notes_dropped_provider_flag() { + // Phase-9 polish: --provider on interactive `rocm chat` is no longer + // silently ignored — the handler emits a one-line note when provider is + // set before rerouting to the dash. Proven by reading the handler body + // (the interactive branch requires a TTY, unavailable in CI). + let src = main_rs_source(); + let body = strip_line_comments(&command_chat_handler_body(&src)); + assert!( + body.contains("provider.is_some()"), + "handler must gate the note on a set --provider; body:\n{body}" + ); + assert!( + body.contains("/provider"), + "the note must point the user at /provider for live switching; body:\n{body}" + ); + } + + #[test] + fn command_chat_honors_chat_mock_and_keeps_prompt_passthrough() { + let src = main_rs_source(); + let body = strip_line_comments(&command_chat_handler_body(&src)); + // --chat-mock is forwarded into the dash reroute. + assert!( + body.contains("chat_mock,") || body.contains("chat_mock\n"), + "Command::Chat must destructure the --chat-mock field; body:\n{body}" + ); + assert!( + body.contains("dash::run_chat(") && body.contains("chat_mock,"), + "--chat-mock must be forwarded to run_chat; body:\n{body}" + ); + assert!( + body.contains("ChatInferenceParams") && body.contains("temperature,") && body.contains("top_p,") && body.contains("max_tokens,"), @@ -17331,6 +20993,412 @@ pub(crate) mod tests { ); } + #[test] + fn every_daemon_identity_reading_maps_to_exactly_one_gate_outcome() { + use rocm_core::IdentityState::{Gone, Indeterminate, Matches, Recycled}; + // `Indeterminate` must abort: folding it into the proceed arm would + // delete the tooling while a live `rocmd` keeps respawning services. + assert_eq!( + daemon_identity_outcome(Gone, false), + DaemonIdentityOutcome::NothingOfOurs + ); + assert_eq!( + daemon_identity_outcome(Recycled, false), + DaemonIdentityOutcome::NothingOfOurs + ); + assert_eq!( + daemon_identity_outcome(Indeterminate, false), + DaemonIdentityOutcome::Unverifiable + ); + assert_eq!( + daemon_identity_outcome(Indeterminate, true), + DaemonIdentityOutcome::Unverifiable + ); + assert_eq!( + daemon_identity_outcome(Matches, true), + DaemonIdentityOutcome::Unverifiable + ); + assert_eq!( + daemon_identity_outcome(Matches, false), + DaemonIdentityOutcome::Stop + ); + } + + #[cfg(unix)] + #[test] + fn a_failed_per_service_stop_aborts_with_the_service_advice() { + let (root, paths) = test_paths("uninstall-stop-errs"); + let mut child = KillOnDrop( + crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn a live managed server"), + ); + let pid = child.0.id(); + let mut record = managed_record_for_pid(&paths, pid, rocm_core::process_start_ticks(pid)); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + + let report = stop_managed_services_with(&paths, |_, _| anyhow::bail!("stop blew up")) + .expect("the pass itself succeeds"); + + assert_eq!( + report.failed.len(), + 1, + "the Err must be recorded as a failure" + ); + assert_eq!(report.failed[0].remedy, StopFailureRemedy::StopTheService); + assert!(report.failed[0].reason.contains("stop blew up")); + assert!(report.stopped.is_empty()); + let error = uninstall_removal_gate(&report).expect_err("the gate must abort"); + assert!(format!("{error:#}").contains("rocm services stop --yes")); + child.0.kill().ok(); + let _ = fs::remove_dir_all(root); + } + + #[cfg(unix)] + #[test] + fn a_stop_result_without_a_status_is_not_a_confirmed_stop() { + // A missing `status` has to read as "unknown", never as "stopped": the + // stop result is the only evidence this gate has that the stop happened. + let (root, paths) = test_paths("uninstall-stop-no-status"); + let child = KillOnDrop( + crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn a live managed server"), + ); + let pid = child.0.id(); + let mut record = managed_record_for_pid(&paths, pid, rocm_core::process_start_ticks(pid)); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + + let report = stop_managed_services_with(&paths, |_, _| Ok(serde_json::json!({}))) + .expect("the pass itself succeeds"); + + assert_eq!(report.failed.len(), 1, "{report:?}"); + assert_eq!(report.failed[0].remedy, StopFailureRemedy::StopTheService); + assert!(report.failed[0].reason.contains("unknown"), "{report:?}"); + assert!(report.stopped.is_empty(), "{report:?}"); + let _ = fs::remove_dir_all(root); + } + + #[cfg(unix)] + #[test] + fn a_stop_that_returns_ok_but_not_stopped_is_a_failure() { + let (root, paths) = test_paths("uninstall-stop-ok-not-stopped"); + let child = KillOnDrop( + crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn a live managed server"), + ); + let pid = child.0.id(); + let mut record = managed_record_for_pid(&paths, pid, rocm_core::process_start_ticks(pid)); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + + let report = + stop_managed_services_with(&paths, |_, _| Ok(serde_json::json!({ "status": "ready" }))) + .expect("the pass itself succeeds"); + + assert_eq!(report.failed.len(), 1, "{report:?}"); + assert_eq!(report.failed[0].remedy, StopFailureRemedy::StopTheService); + assert!(report.stopped.is_empty(), "{report:?}"); + assert!(uninstall_removal_gate(&report).is_err()); + let _ = fs::remove_dir_all(root); + } + + #[cfg(target_os = "linux")] + #[test] + fn a_daemon_stop_that_is_not_confirmed_aborts_and_a_confirmed_one_is_recorded() { + let (root, paths) = test_paths("uninstall-daemon-stop-outcome"); + let child = KillOnDrop( + crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn a live daemon stand-in"), + ); + let pid = child.0.id(); + let mut state = runtime_state(true, pid); + state.daemon_start_ticks = rocm_core::process_start_ticks(pid); + assert!(state.daemon_start_ticks.is_some()); + state.write(&paths).expect("write runtime state"); + + let mut report = ManagedServiceStopReport::default(); + stop_background_helper_with(&paths, &mut report, |_, _| { + rocm_core::TerminationOutcome::TimedOut + }); + assert!(report.stopped.is_empty(), "{report:?}"); + assert_eq!(report.failed.len(), 1, "{report:?}"); + assert_eq!(report.failed[0].remedy, StopFailureRemedy::StopTheDaemon); + assert!(uninstall_removal_gate(&report).is_err()); + + let mut report = ManagedServiceStopReport::default(); + stop_background_helper_with(&paths, &mut report, |_, _| { + rocm_core::TerminationOutcome::Unverified + }); + assert_eq!(report.failed.len(), 1, "{report:?}"); + assert_eq!( + report.failed[0].remedy, + StopFailureRemedy::ConfirmTheDaemonPid + ); + + let mut report = ManagedServiceStopReport::default(); + stop_background_helper_with(&paths, &mut report, |_, _| { + rocm_core::TerminationOutcome::Graceful + }); + assert!(report.failed.is_empty(), "{report:?}"); + assert!( + report.helper_stopped && report.stopped.is_empty(), + "{report:?}" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn two_failures_of_one_class_get_that_classes_advice_once_and_name_both_ids() { + let failure = |id: &str| FailedManagedServiceStop { + service_id: id.to_owned(), + reason: "still ready".to_owned(), + remedy: StopFailureRemedy::StopTheService, + }; + let report = ManagedServiceStopReport { + stopped: Vec::new(), + failed: vec![failure("svc-a"), failure("svc-b")], + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }; + let message = format!("{:#}", uninstall_removal_gate(&report).unwrap_err()); + assert_eq!( + message.matches("rocm services stop --yes").count(), + 1, + "one class, one piece of advice: {message}" + ); + assert!( + message.contains("svc-a") && message.contains("svc-b"), + "{message}" + ); + assert!(message.ends_with("No files were removed."), "{message}"); + } + + #[test] + fn interleaved_failure_classes_print_each_advice_once() { + let failure = |id: &str, remedy| FailedManagedServiceStop { + service_id: id.to_owned(), + reason: "r".to_owned(), + remedy, + }; + let report = ManagedServiceStopReport { + stopped: Vec::new(), + failed: vec![ + failure("svc-a", StopFailureRemedy::StopWhatHoldsThePort), + failure("svc-b", StopFailureRemedy::RemoveTheStoppedRecord), + failure("svc-c", StopFailureRemedy::StopWhatHoldsThePort), + ], + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }; + let message = format!("{:#}", uninstall_removal_gate(&report).unwrap_err()); + assert_eq!( + message.matches("Find what holds that port").count(), + 1, + "{message}" + ); + } + + #[test] + fn each_advice_names_only_the_ids_of_its_own_failure_class() { + // Advice that names every failed id sends the operator to hunt a port + // holder for a service whose real problem is a stale record, so the ids + // are asserted under the sentence that carries them, not just somewhere. + let failure = |id: &str, remedy| FailedManagedServiceStop { + service_id: id.to_owned(), + reason: "r".to_owned(), + remedy, + }; + let report = ManagedServiceStopReport { + stopped: Vec::new(), + failed: vec![ + failure("svc-port", StopFailureRemedy::StopWhatHoldsThePort), + failure("svc-old", StopFailureRemedy::RemoveTheStoppedRecord), + failure("svc-pid", StopFailureRemedy::ConfirmTheServicePid), + failure("svc-port-two", StopFailureRemedy::StopWhatHoldsThePort), + ], + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }; + let message = format!("{:#}", uninstall_removal_gate(&report).unwrap_err()); + assert!( + message.contains("Every process recorded for svc-port, svc-port-two is gone"), + "{message}" + ); + assert!( + message.contains("if it is, stop it first: svc-old."), + "{message}" + ); + assert!(message.contains("it was left alone: svc-pid."), "{message}"); + } + + #[cfg(unix)] + #[test] + fn a_live_record_without_start_ticks_is_refused_not_killed() { + let mut child = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn child"); + let child_pid = child.id(); + let (root, paths) = test_paths("tickless-live-record"); + let record = managed_record_for_pid(&paths, child_pid, None); + fs::create_dir_all(paths.services_dir()).unwrap(); + fs::write( + paths.services_dir().join("svc-managed-stop.json"), + serde_json::to_vec(&record).unwrap(), + ) + .unwrap(); + let report = stop_managed_services_with(&paths, |_, _| { + panic!("a tick-less live record must not be stopped") + }) + .unwrap(); + let alive = child.try_wait().unwrap().is_none(); + child.kill().ok(); + child.wait().ok(); + fs::remove_dir_all(root).ok(); + assert!(alive); + assert_eq!(report.failed.len(), 1); + assert_eq!( + report.failed[0].remedy, + StopFailureRemedy::ConfirmTheServicePid + ); + // The advice tells the operator to "check what those pids are", so the + // refusal has to show them. + assert!( + report.failed[0].reason.contains(&child_pid.to_string()), + "the refusal must name the pid it left alone: {report:?}" + ); + } + + #[test] + fn the_tickless_pid_advice_text_names_the_id_and_both_platforms_kill_commands() { + let report = ManagedServiceStopReport { + stopped: Vec::new(), + failed: vec![FailedManagedServiceStop { + service_id: "svc-tickless".to_owned(), + reason: "r".to_owned(), + remedy: StopFailureRemedy::ConfirmTheServicePid, + }], + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }; + let message = format!("{:#}", uninstall_removal_gate(&report).unwrap_err()); + assert!( + message.contains("it was left alone: svc-tickless."), + "{message}" + ); + assert!(message.contains("kill "), "{message}"); + assert!(message.contains("Stop-Process -Id "), "{message}"); + assert!(!message.contains("services remove"), "{message}"); + } + + #[cfg(unix)] + #[test] + fn a_vllm_shaped_record_with_a_distinct_engine_pid_carries_engine_ticks() { + let (root, paths) = test_paths("vllm-engine-ticks"); + let mut record = managed_record_for_pid(&paths, std::process::id(), Some(7)); + record.engine_pid = None; + fs::create_dir_all(&root).unwrap(); + record.status = "running".to_owned(); + record.engine_state_path = root.join("engine-state.json"); + fs::write( + &record.engine_state_path, + serde_json::json!({"status": "running", "pid": 4242, "server_pid": 4242, "start_ticks": 99}).to_string(), + ) + .unwrap(); + record.refresh_from_engine_state().unwrap(); + fs::remove_dir_all(root).ok(); + assert_eq!(record.engine_pid, Some(4242)); + assert_eq!(record.engine_start_ticks, Some(99)); + } + + #[test] + fn the_endpoint_sentence_follows_the_failure_class() { + let sentence = "may still be serving"; + for (remedy, expected) in [ + (StopFailureRemedy::StopTheService, true), + (StopFailureRemedy::StopWhatHoldsThePort, true), + (StopFailureRemedy::RemoveTheStoppedRecord, false), + (StopFailureRemedy::StopTheDaemon, false), + (StopFailureRemedy::ConfirmTheServicePid, false), + (StopFailureRemedy::ConfirmTheDaemonPid, false), + (StopFailureRemedy::RepairTheDaemonState, false), + (StopFailureRemedy::RepairTheRecord, false), + ] { + let report = ManagedServiceStopReport { + stopped: Vec::new(), + failed: vec![FailedManagedServiceStop { + service_id: "x".to_owned(), + reason: "r".to_owned(), + remedy, + }], + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }; + let message = format!("{:#}", uninstall_removal_gate(&report).unwrap_err()); + assert_eq!( + message.contains(sentence), + expected, + "{remedy:?}: {message}" + ); + } + } + + #[test] + fn the_abort_only_claims_endpoints_may_serve_when_a_service_failed() { + let repair_only = ManagedServiceStopReport { + stopped: Vec::new(), + failed: vec![FailedManagedServiceStop { + service_id: "/x/broken.json".to_owned(), + reason: "service record could not be parsed".to_owned(), + remedy: StopFailureRemedy::RepairTheRecord, + }], + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }; + let message = format!("{:#}", uninstall_removal_gate(&repair_only).unwrap_err()); + assert!(!message.contains("may still be serving"), "{message}"); + + let service = ManagedServiceStopReport { + stopped: Vec::new(), + failed: vec![FailedManagedServiceStop { + service_id: "svc".to_owned(), + reason: "still ready".to_owned(), + remedy: StopFailureRemedy::StopTheService, + }], + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }; + let message = format!("{:#}", uninstall_removal_gate(&service).unwrap_err()); + assert!(message.contains("may still be serving"), "{message}"); + } + + #[cfg(unix)] + struct KillOnDrop(std::process::Child); + + #[cfg(unix)] + impl Drop for KillOnDrop { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } + } + /// Hosts whose engine choice differs, so the assertion below is exercised on /// more than one branch of `select_serve_engine`. fn engine_selection_hosts() -> Vec<(&'static str, rocm_core::HostGpuSummary)> { diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 46e6431c67..95bc9033c6 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -200,18 +200,18 @@ pub(crate) fn ensure_public_service_has_endpoint_key( "managed service was launched with `--require-api-key` but has no endpoint API key, \ so restarting it would reopen it without authentication. Something outside this \ machine may still be publishing its port. The key is dropped when a service stops \ - and cannot be recovered. Launch it again with \ - `rocm serve --require-api-key` (add `--api-key `, or set ROCM_SERVE_API_KEY, \ - to choose the key instead of generating one)." + and cannot be recovered. Launch it again with `{}` {}.", + rocm_engine_protocol::relaunch_command(host, true), + rocm_engine_protocol::RELAUNCH_KEY_CHOICE ); } if rocm_engine_protocol::is_public_bind_host(host) && !key_present { bail!( "managed service is bound to the public host `{host}` but has no endpoint API key, \ so restarting it would reopen it without authentication. The key is dropped when a \ - service stops and cannot be recovered. Launch it again with \ - `rocm serve --host {host} --allow-public-bind` (add `--api-key `, or set \ - ROCM_SERVE_API_KEY, to choose the key instead of generating one)." + service stops and cannot be recovered. Launch it again with `{}` {}.", + rocm_engine_protocol::relaunch_command(host, false), + rocm_engine_protocol::RELAUNCH_KEY_CHOICE ); } Ok(()) @@ -1155,7 +1155,7 @@ fn stream_attached_logs_no_tty(log_path: &Path, child_pid: u32) -> Result --host 127.0.0.1 --port "), + "{rendered}" + ); + assert!( + !rendered.contains("--allow-public-bind"), + "a loopback service is not told to open a public bind: {rendered}" + ); + + // A public host that also demanded auth is told how to bind publicly. + let error = ensure_public_service_has_endpoint_key("0.0.0.0", false, true) + .expect_err("a public service that demanded auth must not come back without it"); + let rendered = format!("{error:#}"); + assert!( + rendered.contains("--host 0.0.0.0 --port --allow-public-bind --require-api-key"), + "{rendered}" ); } @@ -1600,8 +1617,14 @@ mod tests { let message = error.to_string(); assert!(message.contains("0.0.0.0"), "{error:#}"); assert!(message.contains("without authentication"), "{error:#}"); - // Actionable: name the command that mints a fresh key. + // Actionable: name the command that mints a fresh key, in full. assert!(message.contains("--allow-public-bind"), "{error:#}"); + assert!( + format!("{error:#}") + .contains("rocm serve --host 0.0.0.0 --port --allow-public-bind"), + "the relaunch command must be the shared one, flags included: {error:#}" + ); + assert!(!message.contains("--require-api-key"), "{error:#}"); // A public service that still has its key restarts normally. ensure_public_service_has_endpoint_key("0.0.0.0", true, false).unwrap(); @@ -1650,11 +1673,15 @@ mod tests { } /// Build an `AutomationRuntimeState` for the no-double-spawn guard tests. - fn runtime_state(running: bool, daemon_pid: u32) -> AutomationRuntimeState { + /// + /// `daemon_start_ticks` is left unrecorded; tests that care about the + /// daemon's identity set it explicitly. + pub(crate) fn runtime_state(running: bool, daemon_pid: u32) -> AutomationRuntimeState { AutomationRuntimeState { running, automations_enabled: true, daemon_pid, + daemon_start_ticks: None, started_at_unix_ms: 1, last_tick_unix_ms: 1, local_webhook_endpoint: None, diff --git a/apps/rocm/src/test_support.rs b/apps/rocm/src/test_support.rs index 5ad5476afa..fc7a3c60b7 100644 --- a/apps/rocm/src/test_support.rs +++ b/apps/rocm/src/test_support.rs @@ -171,3 +171,20 @@ mod tests { assert_eq!(suffixes.len(), 64, "two calls must never share a suffix"); } } + +/// A [`std::process::Command`] for a coreutils binary, by absolute path. +/// +/// Some tests narrow `PATH` for the whole process while they run, and a test that +/// spawns a bare name such as `sleep` at that moment fails with `ENOENT`. The +/// stand-in processes the uninstall tests spawn have no reason to depend on +/// `PATH`, so they take the standard location instead of racing it. +#[cfg(unix)] +pub(crate) fn bare_command(name: &str) -> std::process::Command { + for dir in ["/usr/bin", "/bin"] { + let candidate = std::path::Path::new(dir).join(name); + if candidate.is_file() { + return std::process::Command::new(candidate); + } + } + std::process::Command::new(name) +} diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index 5b7143e819..9835648d50 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -5142,6 +5142,18 @@ fn temp_sibling_path(path: &Path, suffix: &OsStr) -> Result { Ok(parent.join(file_name)) } +/// Stage-and-publish a file here, sharing only the publish step with `rocm-core`. +/// +/// Deliberately not [`rocm_core::write_file_atomically`], and not a copy of it +/// either: only the Windows-sensitive publish (`ReplaceFileW` and its fallbacks) +/// is single-sourced, via [`publish_temp_file`]. The staging half stays local +/// because it carries the `suffix_for_attempt` and `before_publish` seams the +/// tests below drive to force temp-name collisions, write failures and rename +/// races — injection points `rocm_core`'s caller-facing helper does not expose. +/// +/// Consequence worth knowing: this path does **not** `sync_all` before +/// publishing, so unlike the `rocm-core` helper it is atomic but carries no +/// crash-durability guarantee for the staged bytes. pub(crate) fn write_file_atomically(path: &Path, bytes: &[u8]) -> Result<()> { let temp_id = format!("{}-{}", std::process::id(), unix_time_millis()); write_file_atomically_with( @@ -5258,60 +5270,10 @@ where .with_context(|| format!("failed to publish {}", path.display())) } -#[cfg(not(windows))] -fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { - fs::rename(tmp, path) -} - -#[cfg(windows)] +/// The publish step lives in `rocm-core` so there is one implementation of the +/// Windows `ReplaceFileW` handling for the whole workspace. fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { - if path.try_exists()? { - return replace_file_windows(path, tmp); - } - - match fs::rename(tmp, path) { - Ok(()) => Ok(()), - Err(rename_error) => { - if path.try_exists()? { - replace_file_windows(path, tmp) - } else { - Err(rename_error) - } - } - } -} - -#[cfg(windows)] -#[allow(unsafe_code)] -fn replace_file_windows(path: &Path, replacement: &Path) -> io::Result<()> { - use std::os::windows::ffi::OsStrExt; - use windows_sys::Win32::Storage::FileSystem::ReplaceFileW; - - let path_wide: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); - let replacement_wide: Vec = replacement - .as_os_str() - .encode_wide() - .chain(Some(0)) - .collect(); - - // SAFETY: both path buffers are valid, NUL-terminated UTF-16 strings and - // remain alive for the duration of the synchronous Windows API call. The - // optional backup, exclude, and reserved pointers are intentionally null. - let replaced = unsafe { - ReplaceFileW( - path_wide.as_ptr(), - replacement_wide.as_ptr(), - std::ptr::null(), - 0, - std::ptr::null(), - std::ptr::null(), - ) - }; - if replaced == 0 { - Err(io::Error::last_os_error()) - } else { - Ok(()) - } + rocm_core::publish_temp_file(tmp, path) } fn extract_tarball(archive_path: &Path, target_dir: &Path) -> Result<()> { diff --git a/apps/rocm/src/uninstall.rs b/apps/rocm/src/uninstall.rs index 8c32f9b7bd..9c6f74f4fc 100644 --- a/apps/rocm/src/uninstall.rs +++ b/apps/rocm/src/uninstall.rs @@ -9,7 +9,10 @@ //! owns what only uninstall does on top of removing files: //! //! - undoing the PATH change the installer recorded making; -//! - deleting the running `rocm.exe` on Windows, which cannot unlink itself. +//! - deleting the running `rocm.exe` on Windows, which cannot unlink itself; +//! - stopping the servers this machine manages before any of that, and +//! refusing to remove anything when a stop cannot be confirmed (EAI-8014), +//! plus the report for a removal that stops partway. use std::ffi::OsString; use std::fmt::Write as _; @@ -20,9 +23,13 @@ use std::path::{Path, PathBuf}; use anyhow::{Context, Result, bail}; use rocm_core::{AppPaths, interactive_terminal}; +use crate::uninstall_gate::{ + ManagedServiceStopReport, stop_managed_services_before_uninstall, stopped_during_run_note, + uninstall_removal_gate, +}; use crate::uninstall_plan::{ UninstallOptions, UninstallPlan, build_uninstall_plan, confirm_uninstall, - is_rocm_install_entry_name, remove_path, render_uninstall_plan, + is_rocm_install_entry_name, plan_removes_recovery_tooling, remove_path, render_uninstall_plan, }; /// The file, next to the installed binaries, in which `install.sh` and @@ -65,8 +72,17 @@ impl PathEdit { pub(crate) fn uninstall(options: UninstallOptions) -> Result<()> { let paths = AppPaths::discover()?; - let plan = build_uninstall_plan(&paths, &options)?; - print!("{}", render_uninstall_plan(&plan, &options)); + uninstall_with_paths(&paths, &options) +} + +/// The whole `uninstall` command against a given [`AppPaths`]. +/// +/// Split from [`uninstall`] only so a test can drive the real command — plan, +/// confirm gate, stop pass, removal — against an isolated root, instead of +/// exercising the pieces separately and taking the wiring between them on faith. +pub(crate) fn uninstall_with_paths(paths: &AppPaths, options: &UninstallOptions) -> Result<()> { + let plan = build_uninstall_plan(paths, options)?; + print!("{}", render_uninstall_plan(&plan, options)); if plan_is_empty(&plan) || options.dry_run { return Ok(()); @@ -82,7 +98,78 @@ pub(crate) fn uninstall(options: UninstallOptions) -> Result<()> { } } - execute_plan(&plan, &mut |line| println!("{line}"))?; + // Only an uninstall that takes away the means of stopping a server has to + // stop it first; a cache-only run leaves `rocm services stop` and every + // service record in place. + let removes_recovery_tooling = plan_removes_recovery_tooling(&plan, paths); + stop_managed_services_then_remove(&plan, || { + if removes_recovery_tooling { + stop_managed_services_before_uninstall(paths) + } else { + Ok(ManagedServiceStopReport::default()) + } + }) +} + +/// Stop the servers this machine manages, then remove the planned paths — in +/// that order, and only if every stop was confirmed. +/// +/// Uninstall used to report success while a publicly-bound, GPU-holding endpoint +/// kept serving, then delete the tooling needed to stop it (EAI-8014). The stop +/// runs first and the gate aborts on any unconfirmed stop, so the recovery +/// tooling stays in place. +/// +/// The stop pass is a parameter, not a direct call, so the ordering guarantee is +/// testable rather than merely inspectable: a test can hand in a stop that fails +/// — which no test can reliably provoke from a real process — and assert that +/// not one planned path was removed. Deleting the stop from this function makes +/// those tests fail. +fn stop_managed_services_then_remove( + plan: &UninstallPlan, + stop_managed_services: impl FnOnce() -> Result, +) -> Result<()> { + stop_then_remove_with(plan, stop_managed_services, remove_path) +} + +/// [`stop_managed_services_then_remove`] with the removal of one path supplied, +/// so a test can make it fail partway, which a real filesystem does not do on +/// demand. +fn stop_then_remove_with( + plan: &UninstallPlan, + stop_managed_services: impl FnOnce() -> Result, + remove: impl Fn(&Path) -> Result<()>, +) -> Result<()> { + stop_then_remove_writing_to(plan, stop_managed_services, remove, &mut std::io::stderr()) +} + +/// [`stop_then_remove_with`] with the stream the stop pass's warnings go to, so +/// a test can read what the operator would see on stderr and in which order +/// relative to the abort. +fn stop_then_remove_writing_to( + plan: &UninstallPlan, + stop_managed_services: impl FnOnce() -> Result, + remove: impl Fn(&Path) -> Result<()>, + warnings_out: &mut impl std::io::Write, +) -> Result<()> { + let stop_report = stop_managed_services()?; + // Every fail-open the stop pass took, on stderr rather than stdout: these + // are the places uninstall proceeded without proving the port was free, so + // they have to survive the operator piping its output somewhere. Printed + // before the gate, so they are on screen whether or not it aborts. + write_stop_warnings(warnings_out, &stop_report); + if let Some(line) = uninstall_removal_gate(&stop_report)? { + println!("{line}"); + } + + // PATH changes, the planned entries and the running executable go through + // `execute_plan_with`, which says exactly what is and is not removed when a + // step fails partway. + execute_plan_with( + plan, + &stopped_during_run_note(&stop_report), + &mut |line| println!("{line}"), + &remove, + )?; println!("uninstall complete"); Ok(()) } @@ -95,29 +182,89 @@ pub(crate) fn uninstall(options: UninstallOptions) -> Result<()> { /// the record of the change are all still there to retry with. Then the /// config, data, and cache go, while `rocm` is still there to retry a failure /// with; then the binaries, and the record only once every change is undone. +#[cfg(all(test, unix))] fn execute_plan(plan: &UninstallPlan, report: &mut dyn FnMut(String)) -> Result<()> { + execute_plan_with(plan, "", report, &remove_path) +} + +/// [`execute_plan`] with the removal of one path supplied, so a test can make it +/// fail partway, which a real filesystem does not do on demand. +/// +/// The removal is not transactional: there is no rollback, so a failure partway +/// can leave the binaries and service records gone while a server the gate judged +/// stopped is not. The error therefore says what was removed, what failed (and +/// may be partly removed, since a directory is deleted child by child), and what +/// was not attempted, instead of a bare "failed to remove". +fn execute_plan_with( + plan: &UninstallPlan, + stopped_note: &str, + report: &mut dyn FnMut(String), + remove: &dyn Fn(&Path) -> Result<()>, +) -> Result<()> { for edit in &plan.path_edits { match apply_path_edit(edit) { Ok(line) => report(line), - Err(error) => bail!( - "could not undo the installer's PATH change ({}): {error:#}\n\ - Nothing else was removed, so you can fix this and run `rocm uninstall` again.", - edit.describe() - ), + Err(error) => { + let consequence = if stopped_note.is_empty() { + "Nothing else was removed, so you can fix this and run `rocm uninstall` \ + again." + .to_owned() + } else { + format!( + "No files were removed, so you can fix this and run `rocm uninstall` \ + again, but this run already stopped what it manages:{stopped_note}" + ) + }; + bail!( + "could not undo the installer's PATH change ({}): {error:#}\n{consequence}", + edit.describe() + ) + } } } let is_binary = |kind: &str| kind == "binary"; let non_binaries = plan.actions.iter().filter(|entry| !is_binary(entry.kind)); let binaries = plan.actions.iter().filter(|entry| is_binary(entry.kind)); - for entry in non_binaries.chain(binaries) { - remove_path(&entry.path) - .with_context(|| format!("failed to remove {}", entry.path.display()))?; - report(format!("removed {} {}", entry.kind, entry.path.display())); - } + // Every step in the order it runs, named as the operator will read it. + let mut steps: Vec<(String, PathBuf)> = non_binaries + .chain(binaries) + .map(|entry| { + ( + format!("{} {}", entry.kind, entry.path.display()), + entry.path.clone(), + ) + }) + .collect(); if let Some(record) = &plan.path_record { - remove_path(record).with_context(|| format!("failed to remove {}", record.display()))?; - report(format!("removed install record {}", record.display())); + steps.push(( + format!("install record {}", record.display()), + record.clone(), + )); + } + let mut removed: Vec = Vec::new(); + for (index, (name, path)) in steps.iter().enumerate() { + if let Err(error) = remove(path) { + let untouched: Vec = steps[index + 1..] + .iter() + .map(|(name, _)| name.clone()) + .chain( + plan.self_delete + .iter() + .map(|exe| format!("binary {} (the running program)", exe.display())), + ) + .collect(); + return Err(error + .context(format!("failed to remove {}", path.display())) + .context(partial_removal_note( + &removed, + name, + &untouched, + stopped_note, + ))); + } + report(format!("removed {name}")); + removed.push(name.clone()); } if let Some(exe) = &plan.self_delete { @@ -126,6 +273,49 @@ fn execute_plan(plan: &UninstallPlan, report: &mut dyn FnMut(String)) -> Result< Ok(()) } +/// One `warning:` line per fail-open the stop pass took, on `out`. +fn write_stop_warnings(out: &mut impl std::io::Write, report: &ManagedServiceStopReport) { + for warning in &report.warnings { + let _ = writeln!(out, "warning: {warning}"); + } +} + +/// What a removal that stopped partway has to tell its operator. +/// +/// The entry that failed is reported apart from the untouched ones: removing a +/// directory deletes its children one by one, so it can fail after deleting +/// some of them, and "still present" would then overstate what is left. +fn partial_removal_note( + removed: &[String], + failed: &str, + untouched: &[String], + stopped_note: &str, +) -> String { + let removed_part = if removed.is_empty() { + "no entry was removed completely".to_owned() + } else { + format!("Already removed: {}", removed.join(", ")) + }; + let untouched_part = if untouched.is_empty() { + String::new() + } else { + format!(". Not attempted: {}", untouched.join(", ")) + }; + // What this run's stop pass already did, which a removal failure does not undo. + let stopped_part = if stopped_note.is_empty() { + String::new() + } else { + format!(".{stopped_note}") + }; + format!( + "uninstall stopped partway. {removed_part}. Failed, and may be partly removed: \ + {failed}{untouched_part}. What was already removed can include the `rocm`/`rocmd` \ + binaries and the service records, so `rocm services stop` may no longer work; if a \ + managed server may still be running, find and stop it by hand, then re-run \ + uninstall (or delete what is left) once the cause is fixed{stopped_part}" + ) +} + pub(crate) const fn plan_is_empty(plan: &UninstallPlan) -> bool { plan.actions.is_empty() && plan.path_edits.is_empty() @@ -1316,3 +1506,873 @@ mod tests { } } } + +#[cfg(test)] +mod stop_gate_tests { + use std::fs; + use std::path::PathBuf; + + use anyhow::bail; + + use super::{ + stop_managed_services_then_remove, stop_then_remove_with, stop_then_remove_writing_to, + write_stop_warnings, + }; + use crate::uninstall_gate::{ + FailedManagedServiceStop, ManagedServiceStopReport, StopFailureRemedy, + }; + use crate::uninstall_plan::{UninstallPlan, UninstallPlanEntry}; + + /// A port nothing can ever be serving on. + /// + /// These tests need a record whose endpoint reads as dead, and the gate + /// really does probe it. Port 0 resolves — so the probe runs rather than + /// being skipped by a resolution failure — and the connect always fails, + /// on every platform. Binding an ephemeral port and dropping it would leave + /// a window in which something else on a busy runner grabs the port and + /// fails these tests through the gate's own probe. + /// + /// Both users are Linux-only, so the constant is too — `-D warnings` makes + /// dead code a build failure on the other platforms. + #[cfg(target_os = "linux")] + const UNSERVABLE_PORT: u16 = 0; + + /// An isolated root and a plan whose one action removes a real file in it. + /// + /// The file is what makes these tests assertions about *removal* rather than + /// about return values: it exists before the call, and its presence + /// afterwards is the evidence that the abort happened before the removal + /// loop, not after it. + fn plan_removing_one_file(name: &str) -> (PathBuf, UninstallPlan) { + let root = std::env::temp_dir().join(format!( + "rocm-cli-uninstall-test-{name}-{}-{}", + std::process::id(), + rocm_core::unix_time_millis() + )); + let _ = fs::remove_dir_all(&root); + fs::create_dir_all(&root).expect("failed to create the test root"); + let doomed = root.join("rocm"); + fs::write(&doomed, b"binary").expect("failed to seed the file the plan removes"); + ( + root, + UninstallPlan { + actions: vec![UninstallPlanEntry { + kind: "binary", + path: doomed, + }], + skipped: Vec::new(), + warnings: Vec::new(), + ..UninstallPlan::default() + }, + ) + } + + #[test] + fn a_removal_failure_after_a_stop_says_what_the_stop_already_did() { + let root = std::env::temp_dir().join(format!( + "rocm-cli-uninstall-test-removefail-{}-{}", + std::process::id(), + rocm_core::unix_time_millis() + )); + let _ = fs::remove_dir_all(&root); + fs::create_dir_all(&root).expect("create the test root"); + let doomed = root.join("config.toml"); + fs::write(&doomed, b"config").expect("seed config"); + let plan = UninstallPlan { + actions: vec![UninstallPlanEntry { + kind: "config", + path: doomed.clone(), + }], + ..UninstallPlan::default() + }; + let stopped = || { + Ok(ManagedServiceStopReport { + stopped: vec!["svc-a".to_owned()], + relaunch: vec!["rocm serve model-a --port 8001".to_owned()], + ..ManagedServiceStopReport::default() + }) + }; + + let error = stop_then_remove_with(&plan, stopped, |_| bail!("disk on fire")) + .expect_err("the removal fails the uninstall"); + let message = format!("{error:#}"); + assert!(message.contains("uninstall stopped partway"), "{message}"); + assert!( + message.contains("svc-a") + && message.contains("stay stopped") + && message.contains("rocm serve model-a --port 8001"), + "the message names what the stop already did: {message}" + ); + assert!(doomed.exists(), "the file the failed removal left is there"); + + let error = stop_then_remove_with( + &plan, + || Ok(ManagedServiceStopReport::default()), + |_| bail!("disk on fire"), + ) + .expect_err("the removal still fails"); + assert!( + !format!("{error:#}").contains("stay stopped"), + "with nothing stopped there is nothing to add" + ); + let _ = fs::remove_dir_all(root); + } + + #[cfg(unix)] + #[test] + fn a_path_undo_failure_after_a_stop_says_what_the_stop_already_did() { + let root = std::env::temp_dir().join(format!( + "rocm-cli-uninstall-test-pathfail-{}-{}", + std::process::id(), + rocm_core::unix_time_millis() + )); + let _ = fs::remove_dir_all(&root); + fs::create_dir_all(&root).expect("create the test root"); + let config = root.join("config.toml"); + fs::write(&config, b"config").expect("seed config"); + // A directory where the profile should be makes the PATH undo fail. + let profile = root.join(".bashrc"); + fs::create_dir(&profile).expect("seed an unreadable profile"); + let plan = UninstallPlan { + actions: vec![UninstallPlanEntry { + kind: "config", + path: config.clone(), + }], + path_edits: vec![super::PathEdit::ShellProfile { + profile, + path_exprs: Vec::new(), + }], + ..UninstallPlan::default() + }; + let stopped = || { + Ok(ManagedServiceStopReport { + stopped: vec!["svc-a".to_owned()], + relaunch: vec!["rocm serve model-a --port 8001".to_owned()], + helper_stopped: true, + ..ManagedServiceStopReport::default() + }) + }; + + let error = stop_then_remove_with(&plan, stopped, crate::uninstall_plan::remove_path) + .expect_err("the PATH undo fails the uninstall"); + let message = format!("{error:#}"); + assert!( + message.contains("could not undo the installer's PATH change"), + "{message}" + ); + assert!( + message.contains("svc-a") + && message.contains("stay stopped") + && message.contains("rocm serve model-a --port 8001") + && message.contains("background helper (rocmd) was stopped"), + "the message names what the stop already did: {message}" + ); + assert!( + !message.contains("Nothing else was removed"), + "it must not claim nothing happened: {message}" + ); + assert!(config.exists(), "no file was removed, as the message says"); + + let error = stop_then_remove_with( + &plan, + || Ok(ManagedServiceStopReport::default()), + crate::uninstall_plan::remove_path, + ) + .expect_err("the PATH undo still fails"); + assert!( + format!("{error:#}").contains("Nothing else was removed"), + "with nothing stopped the plain wording stays" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_removal_that_fails_partway_says_what_is_gone_and_what_is_left() { + // The removal is not transactional, so a failure after the first entry + // leaves that entry gone and the next one behind (config, data and cache + // go before the binaries). The error has to say so, and the report must + // match what is actually on disk. + let root = std::env::temp_dir().join(format!( + "rocm-cli-uninstall-test-partial-{}-{}", + std::process::id(), + rocm_core::unix_time_millis() + )); + let _ = fs::remove_dir_all(&root); + fs::create_dir_all(&root).expect("create the test root"); + let binary = root.join("config.toml"); + let data = root.join("data"); + fs::write(&binary, b"binary").expect("seed binary"); + fs::create_dir_all(&data).expect("seed data"); + let plan = UninstallPlan { + actions: vec![ + UninstallPlanEntry { + kind: "config", + path: binary.clone(), + }, + UninstallPlanEntry { + kind: "data", + path: data.clone(), + }, + ], + skipped: Vec::new(), + warnings: Vec::new(), + ..UninstallPlan::default() + }; + + let error = stop_then_remove_with( + &plan, + || Ok(ManagedServiceStopReport::default()), + |path| { + if path == data { + bail!("directory not empty") + } + fs::remove_file(path)?; + Ok(()) + }, + ) + .expect_err("a failing removal must fail the uninstall"); + + let message = format!("{error:#}"); + assert!( + message.contains(&format!("Already removed: config {}", binary.display())), + "{message}" + ); + assert!( + message.contains(&format!( + "Failed, and may be partly removed: data {}", + data.display() + )), + "{message}" + ); + assert!(message.contains("directory not empty"), "{message}"); + assert!( + message.contains("`rocm services stop` may no longer work"), + "{message}" + ); + // The text above is only true if the disk agrees with it. + assert!(!binary.exists(), "the entry the note calls removed is gone"); + assert!( + data.exists(), + "the data the note calls present is still there" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_removal_failing_in_the_middle_names_the_later_entries_as_not_attempted() { + let root = std::env::temp_dir().join(format!( + "rocm-cli-uninstall-test-three-{}-{}", + std::process::id(), + rocm_core::unix_time_millis() + )); + let _ = fs::remove_dir_all(&root); + fs::create_dir_all(&root).expect("create the test root"); + let first = root.join("config.toml"); + let middle = root.join("data"); + let last = root.join("cache"); + fs::write(&first, b"config").expect("seed first"); + fs::create_dir_all(&middle).expect("seed middle"); + fs::create_dir_all(&last).expect("seed last"); + let entry = |kind: &'static str, path: &PathBuf| UninstallPlanEntry { + kind, + path: path.clone(), + }; + let plan = UninstallPlan { + actions: vec![ + entry("config", &first), + entry("data", &middle), + entry("cache", &last), + ], + skipped: Vec::new(), + warnings: Vec::new(), + ..UninstallPlan::default() + }; + + let error = stop_then_remove_with( + &plan, + || Ok(ManagedServiceStopReport::default()), + |path| { + if path == middle { + bail!("directory not empty") + } + if path.is_dir() { + fs::remove_dir_all(path)?; + } else { + fs::remove_file(path)?; + } + Ok(()) + }, + ) + .expect_err("a failing removal must fail the uninstall"); + + let message = format!("{error:#}"); + assert!( + message.contains(&format!("Already removed: config {}", first.display())), + "{message}" + ); + assert!( + message.contains(&format!("Not attempted: cache {}", last.display())), + "{message}" + ); + assert!( + !message.contains("Not attempted: data"), + "the failed entry is reported apart from the untouched ones: {message}" + ); + assert!(!first.exists(), "removed, as the note says"); + assert!(middle.exists(), "the failed entry is still there"); + assert!(last.exists(), "the not-attempted entry is still on disk"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_removal_that_fails_first_reports_the_failed_entry_as_possibly_partial() { + let (root, plan) = plan_removing_one_file("fails-first"); + let doomed = plan.actions[0].path.clone(); + + let error = stop_then_remove_with( + &plan, + || Ok(ManagedServiceStopReport::default()), + |_| bail!("permission denied"), + ) + .expect_err("a failing removal must fail the uninstall"); + + let message = format!("{error:#}"); + assert!( + message.contains("no entry was removed completely"), + "{message}" + ); + assert!( + message.contains(&format!( + "Failed, and may be partly removed: {} {}", + plan.actions[0].kind, + doomed.display() + )), + "{message}" + ); + assert!(doomed.is_file(), "nothing was removed, as the note says"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_directory_removal_that_deletes_some_children_then_fails_is_reported_as_partial() { + let root = std::env::temp_dir().join(format!( + "rocm-cli-uninstall-test-partial-dir-{}-{}", + std::process::id(), + rocm_core::unix_time_millis() + )); + let _ = fs::remove_dir_all(&root); + let data = root.join("data"); + fs::create_dir_all(&data).expect("seed data"); + fs::write(data.join("a"), b"a").expect("seed a"); + fs::write(data.join("b"), b"b").expect("seed b"); + let plan = UninstallPlan { + actions: vec![UninstallPlanEntry { + kind: "data", + path: data.clone(), + }], + skipped: Vec::new(), + warnings: Vec::new(), + ..UninstallPlan::default() + }; + + let error = stop_then_remove_with( + &plan, + || Ok(ManagedServiceStopReport::default()), + |path| { + // Like `remove_dir_all` failing midway: one child is gone, the + // directory is not. + fs::remove_file(path.join("a"))?; + bail!("directory not empty") + }, + ) + .expect_err("a failing removal must fail the uninstall"); + + let message = format!("{error:#}"); + assert!( + message.contains(&format!( + "Failed, and may be partly removed: data {}", + data.display() + )), + "{message}" + ); + assert!(!data.join("a").exists() && data.join("b").exists()); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn every_fail_open_the_stop_pass_took_is_written_as_a_warning_line() { + let report = ManagedServiceStopReport { + warnings: vec!["first fail-open".to_owned(), "second fail-open".to_owned()], + ..ManagedServiceStopReport::default() + }; + let mut out = Vec::new(); + write_stop_warnings(&mut out, &report); + assert_eq!( + String::from_utf8(out).expect("utf8"), + "warning: first fail-open\nwarning: second fail-open\n" + ); + } + + #[test] + fn the_stop_warnings_reach_the_operator_whether_or_not_the_run_aborts() { + // `write_stop_warnings` is tested on its own above; this pins the call + // site in the command, which a no-op there would otherwise leave green: + // the fail-opens are the only disclosure that uninstall proceeded without + // proving a port was free. + let (root, plan) = plan_removing_one_file("warnings-proceed"); + let mut out = Vec::new(); + stop_then_remove_writing_to( + &plan, + || { + Ok(ManagedServiceStopReport { + warnings: vec!["could not check port 9".to_owned()], + ..ManagedServiceStopReport::default() + }) + }, + |path| { + fs::remove_file(path)?; + Ok(()) + }, + &mut out, + ) + .expect("a run with only fail-open warnings proceeds"); + assert_eq!( + String::from_utf8(out).expect("utf8"), + "warning: could not check port 9\n" + ); + let _ = fs::remove_dir_all(root); + + // And on the abort path: printed before the gate refuses, so they are on + // screen even though nothing was removed. + let (root, plan) = plan_removing_one_file("warnings-abort"); + let doomed = plan.actions[0].path.clone(); + let mut out = Vec::new(); + let error = stop_then_remove_writing_to( + &plan, + || { + Ok(ManagedServiceStopReport { + failed: vec![FailedManagedServiceStop { + service_id: "svc-stuck".to_owned(), + reason: "still \"ready\"".to_owned(), + remedy: StopFailureRemedy::StopTheService, + }], + warnings: vec!["could not check port 9".to_owned()], + ..ManagedServiceStopReport::default() + }) + }, + |_| panic!("an aborted run removes nothing"), + &mut out, + ) + .expect_err("a failed stop aborts"); + assert!(format!("{error:#}").contains("svc-stuck")); + assert_eq!( + String::from_utf8(out).expect("utf8"), + "warning: could not check port 9\n" + ); + assert!(doomed.is_file(), "nothing was removed"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_service_that_cannot_be_stopped_leaves_every_planned_path_in_place() { + // The EAI-8014 guarantee itself: while a managed server may still be + // serving, uninstall removes NOTHING — not the binaries, not the service + // records — so `rocm services stop` is still there to recover with. + let (root, plan) = plan_removing_one_file("stop-unconfirmed"); + let doomed = plan.actions[0].path.clone(); + + let error = stop_managed_services_then_remove(&plan, || { + Ok(ManagedServiceStopReport { + stopped: Vec::new(), + failed: vec![FailedManagedServiceStop { + service_id: "svc-stuck".to_owned(), + reason: "still \"ready\" after the stop attempt".to_owned(), + remedy: StopFailureRemedy::StopTheService, + }], + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }) + }) + .expect_err("an unconfirmed stop must abort uninstall"); + + let message = format!("{error:#}"); + // Assert on text the GATE owns first. `svc-stuck` alone would be this + // test reading back its own input — the id is folded into the detail + // whatever the remedy is, so that assertion survives swapping the + // remedy to the daemon variant and would leave this branch of the + // remedy selection unexercised while reading as its coverage. + assert!( + message.contains("rocm services stop --yes"), + "the abort must carry the service remedy, not a generic one: {message}" + ); + assert!( + !message.contains("restarts managed services on its own"), + "a service is not the background helper, so its remedy must not be offered: {message}" + ); + assert!( + message.contains("svc-stuck"), + "the abort still names which service: {message}" + ); + assert!( + doomed.is_file(), + "the planned path must survive an aborted uninstall" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_background_helper_that_cannot_be_stopped_leaves_every_planned_path_in_place() { + // The daemon is the one failure class that is not a service: it restarts + // managed servers on its own, so removing the tooling while it is up + // recreates EAI-8014 from the other end. When it was signalled but not + // confirmed gone, that must abort the uninstall rather than quietly + // proceed. + let (root, plan) = plan_removing_one_file("daemon-unconfirmed"); + let doomed = plan.actions[0].path.clone(); + + let error = stop_managed_services_then_remove(&plan, || { + Ok(ManagedServiceStopReport { + stopped: Vec::new(), + failed: vec![FailedManagedServiceStop { + service_id: "rocmd (pid 4321)".to_owned(), + reason: "the background helper was signalled but not confirmed gone".to_owned(), + remedy: StopFailureRemedy::StopTheDaemon, + }], + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }) + }) + .expect_err("an unstopped background helper must abort uninstall"); + + let message = format!("{error:#}"); + // Assert on text the GATE owns, not on the strings this test handed it. + // The service id and reason above are folded into the detail whatever + // the remedy is, so asserting they appear only proves the input reached + // the output — it passes with the remedy swapped to the service variant, + // which would leave this remedy branch unexercised while reading as its + // coverage. The daemon remedy's own sentence is what distinguishes it. + assert!( + message.contains("restarts managed services on its own"), + "the abort must carry the daemon remedy, not a generic one: {message}" + ); + assert!( + !message.contains("rocm services stop --yes"), + "the daemon is not a service, so the service remedy must not be offered: {message}" + ); + assert!( + message.contains("rocmd (pid 4321)"), + "the abort still names which helper: {message}" + ); + assert!( + doomed.is_file(), + "the planned path must survive an aborted uninstall" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_stop_pass_that_cannot_run_leaves_every_planned_path_in_place() { + // Discovery failing (an unreadable services directory) is the same class + // of danger as a stop failing: uninstall would be removing the tooling + // while blind to what it manages. + let (root, plan) = plan_removing_one_file("stop-undiscoverable"); + let doomed = plan.actions[0].path.clone(); + + let error = stop_managed_services_then_remove(&plan, || { + bail!("failed to read the services directory") + }) + .expect_err("a stop pass that cannot run must abort uninstall"); + + // Unlike the two remedy tests above, there is no gate-owned text to + // assert on here: a discovery failure never reaches the gate, it is + // propagated by the `?` on the stop pass. So this substring is + // deliberately the test's own input, and it pins exactly one thing — + // that the cause is carried through rather than swallowed or replaced + // by a generic "uninstall aborted", which is the `.ok()` defect an + // earlier round found on this path. The survival assertion below is + // what pins the abort itself. + assert!( + error.to_string().contains("services directory"), + "the abort carries the discovery failure verbatim: {error:#}" + ); + assert!( + doomed.is_file(), + "the planned path must survive an aborted uninstall" + ); + let _ = fs::remove_dir_all(root); + } + + // The bounded-exit assertion and its budget are shared with the crate root's + // test module rather than copied here. Two near-identical copies, carrying + // near-identical comments, is exactly what made it easy to believe both + // process-based tests were bounded when only one of them actually was. + #[cfg(target_os = "linux")] + use crate::tests::assert_stopped_within_grace; + + /// Gated to Linux because it spawns a real stand-in process and relies on the + /// zombie-state detection in `terminate_verified`, which is implemented only there. + #[cfg(target_os = "linux")] + #[test] + fn a_cache_only_uninstall_leaves_a_live_server_running() { + // The other direction of the recovery-tooling predicate, pinned through + // the command rather than on the predicate alone. `--keep-binaries + // --keep-data` leaves `rocm services stop` and every service record in + // place, so there is nothing to protect by force-stopping a live server + // — and doing it anyway would kill a user's running model for a run that + // only clears a cache. + // + // Unit tests already assert `plan_removes_recovery_tooling` is false for + // a cache-only plan, but nothing checked the wiring: forcing that + // predicate true left every other test green, because they all uninstall + // something that *does* remove the tooling. + let root = std::env::temp_dir().join(format!( + "rocm-cli-uninstall-cache-only-{}-{}", + std::process::id(), + rocm_core::unix_time_millis() + )); + let _ = fs::remove_dir_all(&root); + let paths = rocm_core::AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + paths.ensure().expect("create the app directories"); + fs::write(paths.cache_dir.join("blob"), b"cached").expect("seed a cache file"); + + let mut server = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("failed to spawn the managed server"); + let pid = server.id(); + let mut record = rocm_core::ManagedServiceRecord::new( + &paths, + "svc-cache-only", + "vllm", + "m", + "m", + "127.0.0.1", + UNSERVABLE_PORT, + "managed", + pid, + None, + None, + None, + ); + record.engine_pid = Some(pid); + record.supervisor_start_ticks = rocm_core::process_start_ticks(pid); + record.status = "ready".to_owned(); + record.write().expect("failed to write the service record"); + + super::uninstall_with_paths( + &paths, + &crate::UninstallOptions { + yes: true, + keep_binaries: true, + keep_data: true, + ..crate::UninstallOptions::default() + }, + ) + .expect("a cache-only uninstall must succeed"); + + assert!( + server.try_wait().expect("poll the server").is_none(), + "a cache-only uninstall must not stop a live managed server" + ); + assert!( + record.manifest_path.is_file(), + "a cache-only uninstall keeps the service records" + ); + let _ = server.kill(); + let _ = server.wait(); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn removal_proceeds_once_every_managed_server_is_confirmed_stopped() { + let (root, plan) = plan_removing_one_file("stop-confirmed"); + let doomed = plan.actions[0].path.clone(); + + stop_managed_services_then_remove(&plan, || { + Ok(ManagedServiceStopReport { + stopped: vec!["svc-stopped".to_owned()], + failed: Vec::new(), + warnings: Vec::new(), + helper_stopped: false, + ..ManagedServiceStopReport::default() + }) + }) + .expect("a confirmed stop must let uninstall proceed"); + + assert!( + !doomed.exists(), + "the planned path must be removed once nothing is left serving" + ); + let _ = fs::remove_dir_all(root); + } + + /// Gated to Linux: relies on a spawned stand-in and on + /// the zombie-state detection in `terminate_verified`. + #[cfg(target_os = "linux")] + #[test] + fn a_live_managed_server_is_stopped_before_the_planned_paths_are_removed() { + // The whole ordering, driven through the real stop pass rather than an + // injected report: a live server dies first, and only then do the files + // go. + // + // The mutation this is sensitive to is the ordering inversion — running + // the removal loop before the stop pass. That only bites because the + // plan below also removes the *services directory*: remove-first deletes + // the record, so the stop pass then finds nothing to stop, the stand-in + // survives its deadline and the assertion fails. With the plan removing + // only an unrelated file (as it did originally) the record survived the + // inversion, the stop still ran, and this test passed under exactly the + // defect it claims to guard. + let (root, mut plan) = plan_removing_one_file("live-server"); + let doomed = plan.actions[0].path.clone(); + let paths = rocm_core::AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + plan.actions + .push(crate::uninstall_plan::UninstallPlanEntry { + kind: "data", + path: paths.data_dir.join("services"), + }); + let child = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("failed to spawn the managed server"); + let pid = child.id(); + let free_port = UNSERVABLE_PORT; + let mut record = rocm_core::ManagedServiceRecord::new( + &paths, + "svc-live", + "vllm", + "m", + "m", + "127.0.0.1", + free_port, + "managed", + pid, + None, + None, + None, + ); + record.engine_pid = Some(pid); + record.supervisor_start_ticks = rocm_core::process_start_ticks(pid); + record.status = "ready".to_owned(); + record.write().expect("failed to write the service record"); + + stop_managed_services_then_remove(&plan, || { + crate::uninstall_gate::stop_managed_services_before_uninstall(&paths) + }) + .expect("a server that stops must not block uninstall"); + + // The stand-in is our own child, so its exit is observable directly — + // and bounded, so this pins that uninstall killed it rather than that it + // outlived the test. + let mut child = child; + assert_stopped_within_grace( + &mut child, + "the managed server must be stopped by uninstall", + ); + assert!( + !rocm_core::process_is_running(pid), + "the managed server must be stopped by uninstall" + ); + assert!( + !doomed.exists(), + "the planned path must be removed once the server is stopped" + ); + let _ = fs::remove_dir_all(root); + } + + /// Gated to Linux for the same reason as the test above: a spawned stand-in + /// and zombie-state detection. + #[cfg(target_os = "linux")] + #[test] + fn the_uninstall_command_itself_stops_a_managed_server_before_removing_anything() { + // Drives the real command end to end — plan, confirm gate, stop pass, + // removal — rather than the pieces separately. + // + // Two distinct claims, and it is worth being exact about which is which, + // because an earlier version of this comment claimed both and only + // delivered one. ORDERING: inlining the old remove-first loop back into + // `uninstall_with_paths` fails here even though every narrower test + // still passes. STOPPING: the assertion below is bounded, so skipping + // the stop pass fails it instead of merely making the suite take as long + // as the stand-in lives. + let root = std::env::temp_dir().join(format!( + "rocm-cli-uninstall-cmd-test-{}-{}", + std::process::id(), + rocm_core::unix_time_millis() + )); + let _ = fs::remove_dir_all(&root); + let paths = rocm_core::AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + for dir in [&paths.config_dir, &paths.data_dir, &paths.cache_dir] { + fs::create_dir_all(dir).expect("seed an isolated dir the plan will remove"); + } + let marker = paths.data_dir.join("state.json"); + fs::write(&marker, b"{}").expect("seed state the removal must delete"); + + let server = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn the managed server"); + let pid = server.id(); + let free_port = UNSERVABLE_PORT; + let mut record = rocm_core::ManagedServiceRecord::new( + &paths, + "svc-cmd-live", + "vllm", + "m", + "m", + "127.0.0.1", + free_port, + "managed", + pid, + None, + None, + None, + ); + record.engine_pid = Some(pid); + record.supervisor_start_ticks = rocm_core::process_start_ticks(pid); + record.status = "ready".to_owned(); + record.write().expect("write the service record"); + + // `--keep-binaries` keeps the test off the real executable-discovery + // path; removing the data dir still takes the service records away, so + // the stop pass is required to run. + super::uninstall_with_paths( + &paths, + &crate::UninstallOptions { + yes: true, + keep_binaries: true, + ..crate::UninstallOptions::default() + }, + ) + .expect("uninstall should succeed once the server is stopped"); + + let mut server = server; + assert_stopped_within_grace( + &mut server, + "`rocm uninstall` must stop the server it manages", + ); + assert!( + !rocm_core::process_is_running(pid), + "`rocm uninstall` must stop the server it manages" + ); + assert!( + !marker.exists(), + "`rocm uninstall` must remove the planned state once the server is stopped" + ); + let _ = fs::remove_dir_all(root); + } +} diff --git a/apps/rocm/src/uninstall_gate.rs b/apps/rocm/src/uninstall_gate.rs new file mode 100644 index 0000000000..0bd304b31d --- /dev/null +++ b/apps/rocm/src/uninstall_gate.rs @@ -0,0 +1,1387 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The stop-before-remove gate behind `rocm uninstall` (EAI-8014). +//! +//! Owns the types describing what a stop pass managed to do and the functions +//! that stop the background helper and every live managed service, classify +//! what is still answering afterwards, and decide whether removal may proceed. +//! The command handler in `uninstall.rs` calls in; the shared +//! `UninstallOptions`/`UninstallPlan` types live in `uninstall_plan.rs`. + +use anyhow::{Context, Result, bail}; +use rocm_core::{AppPaths, AutomationRuntimeState, ManagedServiceRecord}; +use std::fs; +use std::path::PathBuf; +use std::time::Duration; + +use crate::chat::loopback_tcp_port_is_reachable; +use crate::{ + MANAGED_STOP_GRACE, endpoint_keys, host_port_resolves, load_managed_services, + managed_service_is_live, managed_service_record_is_in_use, + stop_internal_managed_service_confirmed, +}; + +/// A managed service uninstall could not confirm stopped, and why. +/// +/// The reason is carried rather than discarded so the abort message says what +/// went wrong — an operator facing "could not stop svc-x" with no cause has +/// nothing to act on. +#[derive(Debug, Clone)] +pub(crate) struct FailedManagedServiceStop { + pub(crate) service_id: String, + pub(crate) reason: String, + pub(crate) remedy: StopFailureRemedy, +} + +/// What will actually clear a failed stop. +/// +/// Not cosmetic. This gate refuses to remove anything while a stop is +/// unconfirmed, so the advice it prints is the operator's only way out, and +/// advice that cannot work turns the refusal into a dead end: `rocm services +/// stop` re-reads the same unparseable JSON and fails the same way, so a record +/// that will not parse would abort every retry identically. Each failure class +/// carries the remedy that can actually clear it. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum StopFailureRemedy { + /// The record parses and its processes are still there, so `rocm services + /// stop` can act on it. + StopTheService, + /// The recorded processes are gone but the endpoint still serves — an engine + /// grandchild outlived its supervisor. `rocm services stop` has nothing left + /// to kill, so the process holding the port has to be found and stopped. + StopWhatHoldsThePort, + /// A record already stopped before this run, whose old port now answers with + /// an authentication refusal. That is not evidence of a survivor of ours (a + /// keyed server of someone else's looks identical), and the record is what + /// keeps tripping the check, so removing it is the way out if it isn't ours. + RemoveTheStoppedRecord, + /// The record does not parse, so no `rocm` command can act on it — the file + /// itself has to be repaired or removed. + RepairTheRecord, + /// The background helper is still alive, so it can restart what was just + /// stopped. It has to go before anything is removed. + StopTheDaemon, + /// A live record carries no start time for a pid that is still alive, so the + /// pid cannot be proven to be the server and is left alone. + ConfirmTheServicePid, + /// A live pid is recorded as the helper but cannot be proven to be `rocmd` + /// (a pre-upgrade state file, or a reboot since). Killing it would risk an + /// unrelated process, so the operator has to decide, and the state file is the + /// thing to delete if it is not the helper. + ConfirmTheDaemonPid, + /// The helper's runtime-state file does not parse, so no pid was ever + /// recovered from it — "kill that pid" is advice nobody can act on. The file + /// itself has to be repaired or deleted, so the remedy names it. + RepairTheDaemonState, +} + +impl StopFailureRemedy { + /// The recovery advice for this class, naming `ids` where the advice is + /// useless without them. + /// + /// Exhaustive on purpose, like [`StoppedRecordVerdict::blocks`]: a new + /// variant must get advice, because the gate's whole contract is that its + /// advice is the operator's only way out. + pub(crate) fn advice(self, ids: &[String]) -> String { + match self { + Self::StopTheService => "Stop them with `rocm services stop --yes`, then re-run \ + uninstall. A server started by another user, or one wedged \ + in the kernel, needs elevated privileges or a manual kill \ + first." + .to_owned(), + Self::StopWhatHoldsThePort => format!( + "Every process recorded for {} is gone, yet the endpoint still answers; the \ + engine outlived its supervisor, so `rocm services stop` has nothing left to \ + kill. Find what holds that port (`ss -ltnp` on Linux, `Get-NetTCPConnection \ + -LocalPort ` on Windows), stop it, then re-run uninstall.", + ids.join(", ") + ), + Self::ConfirmTheServicePid => format!( + "No start time was recorded for a live pid of each service listed, so it cannot \ + be proven to still be the server (the pid may have been reused after a reboot) \ + and it was left alone: {}. Check what those pids are. If one is your server, \ + stop it yourself (`kill ` on Linux, `Stop-Process -Id ` on Windows). \ + If it is something else, delete the service's record file under the data \ + directory's `services` folder by hand. Then re-run uninstall.", + ids.join(", ") + ), + Self::StopTheDaemon => "The background helper restarts managed services on its own, \ + so it has to be stopped before uninstall can safely remove \ + anything: kill that pid (`kill ` on Linux, `Stop-Process -Id -Force` \ + on Windows), then re-run uninstall." + .to_owned(), + Self::RemoveTheStoppedRecord => format!( + "An authenticated server answers on the old port of a record that was already \ + stopped. If that server is not one of yours, delete the stale record with \ + `rocm services remove --yes` and re-run uninstall; if it is, stop it first: \ + {}.", + ids.join(", ") + ), + Self::ConfirmTheDaemonPid => format!( + "The recorded pid could not be proven to be the background helper, so it was left \ + alone. Check what that pid is ({}): if it is `rocmd`, stop it; if it is something \ + else (the state file may predate a reboot), delete the runtime state file named \ + there, then re-run uninstall.", + ids.join(", ") + ), + Self::RepairTheDaemonState => format!( + "The background helper's runtime state does not parse, so no pid could be read \ + from it and `rocm` cannot tell whether the helper is running. Check for a live \ + `rocmd` process and stop it, then repair or delete the file and re-run \ + uninstall: {}.", + ids.join(", ") + ), + Self::RepairTheRecord => format!( + "No `rocm` command can act on an unparseable record, so these have to be handled \ + on disk: check whether the server each one describes is still running (`rocm \ + services list` skips them), stop it, then repair or delete the file and re-run \ + uninstall: {}.", + ids.join(", ") + ), + } + } + + /// Where this class sits in the printed advice. + /// + /// Also exhaustive, and for a second reason beyond ordering: without it, a + /// new variant could compile an `advice()` arm and still never be printed, + /// because nothing would have added it to the list of classes to walk. + /// Ranking every variant means the set that gets advice is derived from the + /// failures themselves rather than from a list somebody has to remember to + /// extend. Most actionable first; the two "repair a file by hand" classes + /// last. + pub(crate) const fn advice_rank(self) -> u8 { + match self { + Self::StopTheService => 0, + Self::StopWhatHoldsThePort => 1, + Self::RemoveTheStoppedRecord => 2, + Self::ConfirmTheServicePid => 3, + Self::StopTheDaemon => 4, + Self::ConfirmTheDaemonPid => 5, + Self::RepairTheDaemonState => 6, + Self::RepairTheRecord => 7, + } + } +} + +/// The processes a stop of `record` would signal, each paired with the start +/// time recorded for it, de-duplicated on pid and preferring a known token. +/// +/// Shared by the stop itself and by this gate's pre-check, so the two cannot +/// disagree about which pid still needs a start time. +pub(crate) fn recorded_pid_identities(record: &ManagedServiceRecord) -> Vec<(u32, Option)> { + let mut entries: Vec<(u32, Option)> = Vec::new(); + for (pid, ticks) in [ + (Some(record.supervisor_pid), record.supervisor_start_ticks), + (record.engine_pid, record.engine_start_ticks), + ] { + let Some(pid) = pid.filter(|pid| *pid != 0 && *pid != std::process::id()) else { + continue; + }; + if let Some(existing) = entries.iter_mut().find(|(seen, _)| *seen == pid) { + if existing.1.is_none() { + existing.1 = ticks; + } + } else { + entries.push((pid, ticks)); + } + } + entries +} + +/// The live pids of `record` that carry no start time, and so cannot be told +/// apart from a reused pid. The stop refuses a record that has any, rather than +/// force-killing a tree it cannot prove is ours; the plan reads the same answer +/// so it never promises a stop this refuses. +pub(crate) fn unprovable_live_pids(record: &ManagedServiceRecord) -> Vec { + recorded_pid_identities(record) + .into_iter() + .filter(|(pid, ticks)| { + record_predates_start_ticks(*ticks) && rocm_core::process_is_running(*pid) + }) + .map(|(pid, _)| pid) + .collect() +} + +/// How long the gate waits for a still-listening endpoint to say what it serves. +/// +/// Only reached when something already answered a TCP connect, and only for +/// records that were already stopped, so it costs nothing on the normal path. +pub(crate) const ENDPOINT_IDENTITY_PROBE_TIMEOUT: Duration = Duration::from_secs(2); + +/// What [`stop_managed_services_before_uninstall`] managed to do, so the caller +/// can report the services it stopped and refuse to proceed while any is still +/// alive. +#[derive(Debug, Default)] +pub(crate) struct ManagedServiceStopReport { + /// Services confirmed stopped (every recorded process observed gone). + pub(crate) stopped: Vec, + /// Services that could not be confirmed stopped — a still-serving endpoint, + /// an unverifiable process, or a record too corrupt to locate one. Uninstall + /// must abort rather than remove the tooling that stops them. + pub(crate) failed: Vec, + /// Every place this pass decided to proceed without proving the port was + /// free, for the caller to put on stderr. + /// + /// These are values rather than `eprintln!`s so a test can assert on them. + /// While they were printed in place, the disclosure that keeps each + /// fail-open a tradeoff rather than a silent removal was pinned by nothing: + /// emptying a warning body left every test green while changing what a + /// destructive command tells its operator. Returning them makes the + /// disclosure part of this function's answer, and `stderr` the caller's + /// business. + pub(crate) warnings: Vec, + /// One relaunch command per entry of `stopped`, built from that service's own + /// record by the shared `rocm_engine_protocol::relaunch_command`, so the + /// abort message names the flags each one needs instead of a generic line. + pub(crate) relaunch: Vec, + /// The background helper (`rocmd`) was stopped by this pass. Kept apart from + /// `stopped` because it is not a service: it has no endpoint key, and the + /// consequence of its being down is that nothing supervises or recovers + /// services until it is restarted. + pub(crate) helper_stopped: bool, +} + +/// The kill breadth the stop applies to the background helper. +/// +/// A constant so a test can see the scope the production stop hands the +/// termination, rather than only that some termination was requested. +pub(crate) const HELPER_KILL_SCOPE: rocm_core::KillScope = rocm_core::KillScope::Tree; + +/// How much of a process tree the stop signals, in words, for the plan and the +/// docs: only Linux enumerates descendants (`process_tree_pids` returns just the +/// root on Windows), so the claim cannot be the same on both. +pub(crate) const fn tree_scope_words() -> &'static str { + if rocm_core::runtime_is_windows() { + "only the recorded process (descendants are not enumerated on Windows)" + } else { + "its whole process tree" + } +} + +/// What the gate decides about the background helper, read without touching it. +/// +/// The one decision behind the plan's note, the pre-flight and the stop itself, +/// so the plan cannot promise a stop that the stop then refuses. +pub(crate) enum HelperDecision { + /// No state file, `running` false, a free pid, or a pid that belongs to + /// someone else. + NothingToStop, + /// The state file exists but does not read or parse. + UnreadableState(String), + /// A live pid is recorded as the helper but cannot be proven to be it. + Unverifiable { pid: u32 }, + /// Verified as the helper: stop it. + Stop { + pid: u32, + identity: rocm_core::ProcessIdentity, + }, +} + +/// Decide [`HelperDecision`] from the state file and the pid's identity. +pub(crate) fn helper_decision(paths: &AppPaths) -> HelperDecision { + // Three outcomes, not two. `load` returns `Ok(None)` only when there is no + // state file at all; a permission error or a half-written file returns + // `Err`, and discarding that would skip the daemon stop silently and let + // uninstall delete the tooling while a live `rocmd` respawns what the + // service pass just stopped. + let state = match AutomationRuntimeState::load(paths) { + Ok(Some(state)) => state, + Ok(None) => return HelperDecision::NothingToStop, + Err(error) => return HelperDecision::UnreadableState(format!("{error:#}")), + }; + // `running` is the same inactive contract `background_helper_already_running` + // spawns on. Only the clean-shutdown path writes `running: false`, so a false + // flag means the recorded pid belongs to a daemon that already exited, and + // anything alive under it now inherited the number. + if !state.running || state.daemon_pid == 0 || state.daemon_pid == std::process::id() { + return HelperDecision::NothingToStop; + } + if !rocm_core::process_is_running(state.daemon_pid) { + return HelperDecision::NothingToStop; + } + // Kill nothing this cannot identify. `terminate_verified` with `force` is a + // SIGKILL across the whole process tree, and `runtime-state.json` outlives a + // crash, OOM-kill or reboot with `running` still true, so the recorded pid + // can belong to an unrelated process by the time uninstall runs. An absent + // start-time is unverifiable on its own (see `record_predates_start_ticks`). + // `identity_state` takes its liveness check and its reading in the order that + // keeps a recycled pid from reading as a match, so it is called directly. + let identity = rocm_core::ProcessIdentity::new(state.daemon_pid, state.daemon_start_ticks); + match daemon_identity_outcome( + rocm_core::identity_state(&identity), + record_predates_start_ticks(state.daemon_start_ticks), + ) { + DaemonIdentityOutcome::NothingOfOurs => HelperDecision::NothingToStop, + DaemonIdentityOutcome::Unverifiable => HelperDecision::Unverifiable { + pid: state.daemon_pid, + }, + DaemonIdentityOutcome::Stop => HelperDecision::Stop { + pid: state.daemon_pid, + identity, + }, + } +} + +/// The refusal a [`HelperDecision`] amounts to, if any. +fn helper_refusal(paths: &AppPaths, decision: &HelperDecision) -> Option { + match decision { + // Name the file, not a pid: nothing parsed, so no pid was ever + // recovered and "kill that pid" would be advice nobody can follow. + HelperDecision::UnreadableState(error) => Some(FailedManagedServiceStop { + service_id: format!("rocmd ({})", paths.automation_state_path().display()), + reason: format!( + "the background helper's runtime state could not be read, so it cannot be \ + confirmed stopped: {error}" + ), + remedy: StopFailureRemedy::RepairTheDaemonState, + }), + HelperDecision::Unverifiable { pid } => Some(daemon_identity_unverified( + *pid, + &paths.automation_state_path(), + )), + HelperDecision::NothingToStop | HelperDecision::Stop { .. } => None, + } +} + +/// What the plan should tell the operator about the background helper, derived +/// from the same decision the stop pass acts on, so the line cannot promise a +/// stop that pass then refuses. +/// +/// `None` when there is nothing to say: nothing to stop, or a refusal the plan's +/// consolidated refusal line already names. `run_refuses` is whether the run +/// aborts before stopping anything: a helper that is verified but sits in such a +/// run is left running, and saying "will be stopped" would be false. +/// `refusal_line_present` is whether the pre-flight ran and its refusal line is +/// in the plan; it is absent when the service records could not be read at all, +/// and then a helper that cannot be verified has to be named here. +pub(crate) fn background_helper_plan_note( + paths: &AppPaths, + removes_recovery_tooling: bool, + run_refuses: bool, + refusal_line_present: bool, +) -> Option { + let state_file = paths.automation_state_path(); + let decision = helper_decision(paths); + match (&decision, removes_recovery_tooling) { + (HelperDecision::NothingToStop, _) => None, + // Named by the refusal line, which comes from the same decision. + (HelperDecision::UnreadableState(_) | HelperDecision::Unverifiable { .. }, true) + if refusal_line_present => + { + None + } + (HelperDecision::UnreadableState(_) | HelperDecision::Unverifiable { .. }, true) => { + helper_refusal(paths, &decision).map(|failure| { + format!( + "{} ({}); uninstall will refuse to remove anything until this is resolved", + failure.service_id, failure.reason + ) + }) + } + (HelperDecision::UnreadableState(error), false) => Some(format!( + "could not read the background helper's runtime state {}: {error}. This run keeps \ + the tooling, so it does not check it", + state_file.display() + )), + (HelperDecision::Unverifiable { pid } | HelperDecision::Stop { pid, .. }, false) => { + Some(format!( + "the background helper (rocmd, pid {pid}) is running; this removal keeps the \ + tooling, so it is left running" + )) + } + (HelperDecision::Stop { pid, .. }, true) if run_refuses => Some(format!( + "the background helper (rocmd, pid {pid}) is running; uninstall will refuse (see \ + above) before stopping anything, so it is left running" + )), + (HelperDecision::Stop { pid, .. }, true) => Some(format!( + "the background helper (rocmd, pid {pid}) is running; it will be stopped before \ + removal, and the kill covers {}", + tree_scope_words() + )), + } +} + +/// The failure recorded when the background helper is live but cannot be proven +/// to be `rocmd`, so it was deliberately left alone. +pub(crate) fn daemon_identity_unverified( + daemon_pid: u32, + state_file: &std::path::Path, +) -> FailedManagedServiceStop { + FailedManagedServiceStop { + service_id: format!( + "rocmd (pid {daemon_pid}; state file {})", + state_file.display() + ), + reason: + "the background helper's identity could not be verified (no start time was recorded \ + for it, or its start time cannot be read now), so it was left running rather \ + than risk signalling an unrelated process that inherited its pid" + .to_owned(), + remedy: StopFailureRemedy::ConfirmTheDaemonPid, + } +} + +/// What the gate does with the daemon's recorded pid once its identity has been +/// read back. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum DaemonIdentityOutcome { + /// The pid is free or belongs to someone else: nothing of ours to stop. + NothingOfOurs, + /// Could not be proven ours, so it is neither killed nor ignored: abort. + Unverifiable, + /// Verified as the daemon: stop it. + Stop, +} + +/// Map an identity reading to the gate's action. Pure so every input is pinned +/// by a test: folding `Indeterminate` into `NothingOfOurs` would remove the +/// tooling while a live `rocmd` keeps respawning services. +pub(crate) const fn daemon_identity_outcome( + state: rocm_core::IdentityState, + record_predates_start_ticks: bool, +) -> DaemonIdentityOutcome { + match state { + rocm_core::IdentityState::Gone | rocm_core::IdentityState::Recycled => { + DaemonIdentityOutcome::NothingOfOurs + } + rocm_core::IdentityState::Indeterminate => DaemonIdentityOutcome::Unverifiable, + rocm_core::IdentityState::Matches if record_predates_start_ticks => { + DaemonIdentityOutcome::Unverifiable + } + rocm_core::IdentityState::Matches => DaemonIdentityOutcome::Stop, + } +} + +/// Stop the background helper before uninstall stops the services it supervises. +/// +/// `rocmd` recovers managed services: a `ready`/`running` record whose endpoint +/// is unreachable is treated as recoverable and respawned +/// (`endpoint_status_unreachable`). That is precisely the state the stop pass +/// creates — the engine is killed, and the record still says `ready` until +/// `stop_internal_managed_service` writes it back. A daemon polling in that +/// window brings the server straight back, after which uninstall would delete +/// the binaries and every service record while a brand-new engine holds the GPU. +/// Stopping the supervisor first closes the window instead of racing it. +/// +/// A daemon that cannot be confirmed stopped is a blocking failure for the same +/// reason a service is: it can resurrect a server after the tooling is gone. +/// +/// It is never killed on the recorded pid alone. `runtime-state.json` outlives a +/// crash, OOM-kill or reboot with `running` still true, so that pid can name an +/// unrelated process — and this call site signals a whole tree with `force`. +/// Only a pid whose recorded start-time still matches is signalled; a recycled +/// one means the daemon is already gone (nothing to stop), and one that can be +/// neither confirmed nor refuted is left alone and reported as a failure, which +/// aborts the uninstall with the tooling intact. Being told to kill a pid is +/// recoverable; having an unrelated process tree killed is not. +/// +/// A record whose `running` is false is not signalled at all. Only rocmd's clean +/// shutdown writes that flag, on its way out, so the pid in such a record names +/// a process that has already exited and anything live under that number +/// inherited it. This is the same inactive contract `background_helper_already_running` +/// spawns on. +/// +/// Linux (`/proc`) and Windows (`GetProcessTimes` creation time) can record and +/// compare a start-time, so a recycled pid is told apart there. +/// +/// One residual gap, and the list above is otherwise the complete set of +/// inputs: a *missing* `runtime-state.json` is taken at face value as "no +/// daemon". Deleting that file by hand while `rocmd` is live therefore skips the +/// daemon stop silently. This is deliberate — a missing file is the ordinary +/// never-started case, and there is no pid to verify or signal without it — but +/// it does mean the gate is only as good as the state file. An unreadable one is +/// the case that aborts; an absent one is the case that proceeds. +/// +/// The verified termination is a parameter, so a test can make the stop come +/// back unconfirmed and see the [`HELPER_KILL_SCOPE`] the production stop asks +/// for; production passes `terminate_verified` with `force`. +pub(crate) fn stop_background_helper_with( + paths: &AppPaths, + report: &mut ManagedServiceStopReport, + terminate: impl FnOnce( + &rocm_core::ProcessIdentity, + rocm_core::KillScope, + ) -> rocm_core::TerminationOutcome, +) { + let decision = helper_decision(paths); + if let Some(failure) = helper_refusal(paths, &decision) { + report.failed.push(failure); + return; + } + let HelperDecision::Stop { pid, identity } = decision else { + return; + }; + println!("stopping the background helper (rocmd)"); + let outcome = terminate(&identity, HELPER_KILL_SCOPE); + if outcome.stopped() { + report.helper_stopped = true; + } else if matches!(outcome, rocm_core::TerminationOutcome::Unverified) { + report.failed.push(daemon_identity_unverified( + pid, + &paths.automation_state_path(), + )); + } else { + report.failed.push(FailedManagedServiceStop { + service_id: format!("rocmd (pid {pid})"), + reason: "the background helper was signalled but could not be confirmed stopped (the \ + servers it supervised may already be down), and it restarts managed \ + services whose endpoint stops answering" + .to_owned(), + remedy: StopFailureRemedy::StopTheDaemon, + }); + } +} + +/// Whether the stop pass has to deal with `record`: it reads as running, or its +/// status says something other than `stopped` while a recorded process is still +/// alive. +/// +/// The status string alone is not enough: `refresh_from_engine_state` adopts +/// `failed` from the engine's own state file, and the liveness refresh returns +/// early for a non-live status without consulting the pids, so a server whose +/// engine reported failure while its process is still up reads as removable. +/// `services remove` guards with [`managed_service_record_is_in_use`] for the +/// same reason. A record that reads `stopped` is left to the port probe: that +/// status is written only once the recorded processes are confirmed gone, so a +/// live pid under it is a reused number, and refusing on it would block every +/// uninstall that finds an old record. +pub(crate) fn record_needs_stopping(record: &ManagedServiceRecord) -> bool { + managed_service_is_live(record) + || (record.status != "stopped" && managed_service_record_is_in_use(record)) +} + +/// The records a stop pass partitions after the live ones are set aside: those +/// already stopped, and those stopped but sharing an endpoint with a live record. +struct RecordPartition<'a> { + already_stopped: Vec<&'a ManagedServiceRecord>, + deferred_partners: Vec<(&'a ManagedServiceRecord, Vec)>, +} + +fn partition_records(records: &[ManagedServiceRecord]) -> RecordPartition<'_> { + // A record already stopped before this run needs no stop to be judged, and a + // survivor from the previous run reads exactly like that: probe those first. + // + // Except a stopped record that shares its host:port with a live one: serving + // the same model again after a stop leaves exactly that pair, and the live + // record's own server answers there. That answer says nothing about the + // stopped record until the live one has been stopped, so it is probed after. + let live_endpoints: Vec<(String, Vec, u16)> = records + .iter() + .filter(|record| record_needs_stopping(record)) + .map(|record| { + ( + record.service_id.clone(), + probe_hosts(&record.host), + record.port, + ) + }) + .collect(); + // Overlap, not equality: `0.0.0.0`, `localhost` and `127.0.0.1` can all name + // the one listener, so any shared probe address on the same port counts. + let live_partners = |record: &ManagedServiceRecord| -> Vec { + let mine = probe_hosts(&record.host); + live_endpoints + .iter() + .filter(|(_, hosts, port)| { + *port == record.port + && mine + .iter() + .any(|a| hosts.iter().any(|b| hosts_name_one_listener(a, b))) + }) + .map(|(id, _, _)| id.clone()) + .collect() + }; + RecordPartition { + already_stopped: records + .iter() + .filter(|record| !record_needs_stopping(record) && live_partners(record).is_empty()) + .collect(), + deferred_partners: records + .iter() + .filter(|record| !record_needs_stopping(record)) + .map(|record| (record, live_partners(record))) + .filter(|(_, partners)| !partners.is_empty()) + .collect(), + } +} + +/// Everything the stop pass would refuse before it signals or removes anything, +/// decided without touching a process, and the records it read to decide. +/// +/// This is the one place those refusals live. The stop pass runs it first and +/// returns on any failure, so a run that is going to abort has stopped nothing +/// on the way (each confirmed stop drops that service's endpoint key, which +/// cannot be re-minted); the plan runs the same function, so the line the +/// operator confirms against describes the run that will happen. Order matters +/// and is the order of cost: an unparseable manifest, then a live pid that +/// cannot be proven ours, then a stopped record whose port still answers as its +/// own model, then a background helper that cannot be identified. Each group +/// returns before the next, as the stop pass always did. +/// +/// Fail-closed on discovery too: if the services directory exists but cannot be +/// enumerated, the error propagates so uninstall aborts rather than deleting the +/// tooling while blind to what it manages. +fn preflight_with_records( + paths: &AppPaths, +) -> Result<(ManagedServiceStopReport, Vec)> { + let mut report = ManagedServiceStopReport::default(); + // One reading of the helper serves the whole pre-flight, and its refusal is + // appended to whatever else refuses, so the plan can name it on a run that + // already refuses for another reason instead of the operator learning of it + // only on the retry. + let helper_failure = helper_refusal(paths, &helper_decision(paths)); + let with_helper = |mut report: ManagedServiceStopReport| { + report.failed.extend(helper_failure.clone()); + report + }; + // A manifest that exists but does not parse can describe a live, + // GPU-holding server; `load_managed_services` skips it silently. + for manifest in unreadable_service_manifests(paths)? { + report.failed.push(FailedManagedServiceStop { + // The full path, not the file name: the only remedy is to act on the + // file, so the message has to say which file. + service_id: manifest.display().to_string(), + reason: + "service record could not be parsed, so its server cannot be located or stopped" + .to_owned(), + remedy: StopFailureRemedy::RepairTheRecord, + }); + } + if !report.failed.is_empty() { + return Ok((with_helper(report), Vec::new())); + } + let records = load_managed_services(paths)?; + // A live record with no recorded start time cannot prove its pid is still the + // server (the pid may have been reused after a reboot), and the stop below + // force-kills a process tree. Refuse before anything is stopped, as the + // daemon path does for a tick-less state file. + for record in records + .iter() + .filter(|record| record_needs_stopping(record)) + { + let unprovable = unprovable_live_pids(record); + if !unprovable.is_empty() { + report.failed.push(FailedManagedServiceStop { + service_id: record.service_id.clone(), + reason: format!( + "no start time was recorded for live pid(s) {unprovable:?}, so they cannot be \ + told apart from a reused pid" + ), + remedy: StopFailureRemedy::ConfirmTheServicePid, + }); + } + } + if !report.failed.is_empty() { + return Ok((with_helper(report), records)); + } + let partition = partition_records(&records); + probe_records_for_survivors(paths, &partition.already_stopped, &[], &mut report); + Ok((with_helper(report), records)) +} + +/// [`preflight_with_records`] for the plan: the refusals a run would hit and the +/// fail-open disclosures it would make, with nothing stopped. +pub(crate) fn uninstall_preflight(paths: &AppPaths) -> Result { + preflight_with_records(paths).map(|(report, _)| report) +} + +/// The plan's one line for a run that will refuse, built from the failures +/// [`uninstall_preflight`] found. It says what the run does not do as well as +/// what blocks it: the pre-flight returns before anything is stopped. +pub(crate) fn preflight_refusal_line(report: &ManagedServiceStopReport) -> Option { + if report.failed.is_empty() { + return None; + } + let detail = report + .failed + .iter() + .map(|failure| format!("{} ({})", failure.service_id, failure.reason)) + .collect::>() + .join("; "); + Some(format!( + "uninstall will refuse to remove anything, and will stop nothing, until this is \ + resolved: {detail}" + )) +} + +/// Stop every managed service that is in use before uninstall removes the +/// binaries and service records needed to stop them. +/// +/// The defect this closes (EAI-8014): uninstall reported success while a +/// publicly-bound, GPU-holding managed server kept serving, and deleted the +/// `rocm`/`rocmd` binaries and service manifests — so the supported +/// `rocm services stop` path was gone and only a manual PID kill remained. +/// +/// A service is only counted stopped when the stop confirms every recorded +/// process gone (the boolean [`stop_internal_managed_service_confirmed`] +/// returns — the report's `status` string cannot say it, because a record that +/// already read `stopped` keeps that status through a stop that confirmed +/// nothing); anything else lands in `failed` so the caller aborts and keeps the +/// tooling. +pub(crate) fn stop_managed_services_before_uninstall( + paths: &AppPaths, +) -> Result { + stop_managed_services_confirmed_with(paths, stop_internal_managed_service_confirmed) +} + +/// [`stop_managed_services_before_uninstall`] with a stop that reports only a +/// `status`, which counts as confirmed when it reads `stopped`. Unix-test-only: the +/// production stop decides on the confirmed flag, and the tests that care drive +/// [`stop_managed_services_confirmed_with`] directly. +#[cfg(all(test, unix))] +pub(crate) fn stop_managed_services_with( + paths: &AppPaths, + stop_service: impl Fn(&AppPaths, &str) -> Result, +) -> Result { + stop_managed_services_confirmed_with(paths, |paths, service_id| { + let result = stop_service(paths, service_id)?; + let confirmed = result.get("status").and_then(serde_json::Value::as_str) == Some("stopped"); + Ok((result, confirmed)) + }) +} + +/// [`stop_managed_services_before_uninstall`] with the per-service stop +/// injectable, so a test can make it fail or fail to confirm — which no real +/// process reliably does. +pub(crate) fn stop_managed_services_confirmed_with( + paths: &AppPaths, + stop_service: impl Fn(&AppPaths, &str) -> Result<(serde_json::Value, bool)>, +) -> Result { + stop_managed_services_with_helper_stop(paths, stop_service, |identity, scope| { + rocm_core::terminate_verified(identity, scope, MANAGED_STOP_GRACE, true) + }) +} + +/// [`stop_managed_services_confirmed_with`] with the helper's termination +/// injectable too, so a test can make the helper stop come back unconfirmed +/// and see that no service is stopped after it. +pub(crate) fn stop_managed_services_with_helper_stop( + paths: &AppPaths, + stop_service: impl Fn(&AppPaths, &str) -> Result<(serde_json::Value, bool)>, + terminate_helper: impl FnOnce( + &rocm_core::ProcessIdentity, + rocm_core::KillScope, + ) -> rocm_core::TerminationOutcome, +) -> Result { + // The refusals are decided before anything is stopped, by the function the + // plan also calls. Past this point a stop can still fail, the helper can + // still not die, and a survivor can still answer after the stops, so the + // abort message names what the run already did. + let (mut report, records) = preflight_with_records(paths)?; + if !report.failed.is_empty() { + return Ok(report); + } + let partition = partition_records(&records); + stop_background_helper_with(paths, &mut report, terminate_helper); + if !report.failed.is_empty() { + return Ok(report); + } + let mut attempted: Vec<&ManagedServiceRecord> = Vec::new(); + for record in &records { + if !record_needs_stopping(record) { + continue; + } + attempted.push(record); + // Each stop waits out a bounded grace per recorded process (and the + // engine's own stop before that), so name the service first: without + // this, an uninstall with a live server reads as a hang. + println!("stopping managed service {}", record.service_id); + match stop_service(paths, &record.service_id) { + Ok((_, true)) => { + report.stopped.push(record.service_id.clone()); + report.relaunch.push(format!( + "{}: `{}`", + record.service_id, + rocm_engine_protocol::relaunch_command_for( + &record.model_ref, + &record.engine, + &record.host, + record.port, + record.requires_api_key, + ) + )); + } + Ok((result, false)) => { + let status = result + .get("status") + .and_then(serde_json::Value::as_str) + .unwrap_or("unknown"); + report.failed.push(FailedManagedServiceStop { + service_id: record.service_id.clone(), + reason: if status == "stopped" { + "its record reads \"stopped\", but the stop could not confirm every \ + recorded process gone" + .to_owned() + } else { + format!("still \"{status}\" after the stop attempt") + }, + remedy: StopFailureRemedy::StopTheService, + }); + } + Err(error) => report.failed.push(FailedManagedServiceStop { + service_id: record.service_id.clone(), + reason: format!("{error:#}"), + remedy: StopFailureRemedy::StopTheService, + }), + } + } + // Ground the PID bookkeeping in what is actually being served. Confirming a + // stop from recorded PIDs alone is thin: on Windows the kill scope is the + // recorded process only, so an engine grandchild can outlive it and keep the + // port and the GPU while the record reads "stopped". + // + // Every record is probed, not just the ones this pass stopped. A stop + // persists `status = "stopped"` before this loop runs, so the record that + // failed the gate reads as not-live on the very next run — probing only this + // pass's own work would let the retry the abort message asks for sail + // through and remove the tooling while the survivor keeps serving. That is + // the defect this gate exists to prevent, reached by following its own + // instructions. + // + // What differs is the evidence required, because a stopped record keeps its + // manifest and its old port until `rocm services remove` or `prune` deletes + // it, so an unrelated process that later binds that port must not brick + // uninstall with no way out: + // + // * stopped by this pass — any listener fails the gate. We killed the + // recorded processes seconds ago; something answering there is the + // survivor. + // * already stopped before this run — a listener alone proves nothing, so + // it has to identify itself as this record's own model before it counts. + // An unrelated service on a recycled port does not, and is not blocked. + let stopped_this_pass: Vec<&ManagedServiceRecord> = attempted.clone(); + probe_records_for_survivors(paths, &stopped_this_pass, &attempted, &mut report); + // A deferred record whose live partner failed to stop is already explained by + // the partner's own failure: the listener there IS the partner, so judging the + // stopped record too would name one survivor twice, under the wrong remedy. + let deferred: Vec<&ManagedServiceRecord> = partition + .deferred_partners + .iter() + .filter(|(_, partners)| { + !partners + .iter() + .any(|id| report.failed.iter().any(|f| &f.service_id == id)) + }) + .map(|(record, _)| *record) + .collect(); + probe_records_for_survivors(paths, &deferred, &attempted, &mut report); + Ok(report) +} + +/// Probe the endpoints of `to_probe` for a survivor, recording any that should +/// block the removal. `attempted` is the set this pass stopped; for those any +/// listener blocks, for the rest it has to identify itself as the record's own. +fn probe_records_for_survivors( + paths: &AppPaths, + to_probe: &[&ManagedServiceRecord], + attempted: &[&ManagedServiceRecord], + report: &mut ManagedServiceStopReport, +) { + for &record in to_probe { + if report + .failed + .iter() + .any(|failure| failure.service_id == record.service_id) + { + continue; + } + // A wildcard record yields both loopback families; whichever answers is + // the one the identity probe below has to talk to, so keep it rather + // than re-deriving a single host and guessing the family wrong. + let candidates = probe_hosts(&record.host); + let reachable_hosts: Vec = candidates + .iter() + .filter(|candidate| loopback_tcp_port_is_reachable(candidate, record.port)) + .cloned() + .collect(); + if reachable_hosts.is_empty() { + // Not reaching the port is the normal, silent case: nothing is + // listening, which is what a stopped service looks like. But + // `loopback_tcp_port_is_reachable` returns the same `false` when the + // address does not resolve at all, and those are opposite + // situations. A refused connect is evidence the port is free; a name + // that no longer resolves is evidence of nothing, and skipping on it + // means proceeding to delete the tooling without ever having asked + // whether a server is up. That is a fail-open, and it is disclosed. + if !candidates + .iter() + .any(|candidate| host_port_resolves(candidate, record.port)) + { + report.warnings.push(format!( + "{} is recorded for service {}, but that name does not resolve here, so \ + whether anything is still serving on port {} could not be checked at all, \ + and uninstall is proceeding without that check. A record written on \ + another machine, or under a hostname since removed, looks like this. If \ + that service may still be running, it has to be found and stopped by hand: \ + this uninstall does not wait for you to, and removes the tooling that \ + would have stopped it.", + record.host, record.service_id, record.port + )); + } + continue; + } + let stopped_by_this_pass = attempted + .iter() + .any(|candidate| candidate.service_id == record.service_id); + if !stopped_by_this_pass { + let endpoint_api_key = endpoint_keys::endpoint_api_key(paths, &record.service_id); + // A wildcard record is probed on every loopback family that accepts + // a connection, and ANY of them blocking blocks the removal: a + // stranger answering on `127.0.0.1` must not hide the record's own + // model still serving on `::1`. Each probe asks the address the + // reachability probe just succeeded against, because + // `record.endpoint_url` is built from the recorded host, so a + // `0.0.0.0` or `::` bind would be connected to literally — which + // does not resolve, fails the probe, and takes a fail-open branch, + // removing the tooling while a wildcard-bound engine is serving. + let mut blocking: Option = None; + let mut proceeding: Vec = Vec::new(); + for reachable_host in &reachable_hosts { + let mut probe_record = record.clone(); + probe_record.endpoint_url = + rocm_core::format_http_base_url(reachable_host, record.port); + let probe = rocm_core::managed_service_endpoint_identity( + &probe_record, + endpoint_api_key.as_deref(), + ENDPOINT_IDENTITY_PROBE_TIMEOUT, + ); + // Short-circuits: the extra round trip only happens when the + // probe produced no usable listing, which is the only case its + // answer can change. + let auth_refused = probe.is_err() + && endpoint_refused_authorization( + &probe_record.endpoint_url, + endpoint_api_key.as_deref(), + ); + // Every arm below is decided in `stopped_record_verdict` and + // pinned there by + // `every_identity_answer_maps_to_exactly_one_gate_outcome`, with + // `an_endpoint_listing_nothing_does_not_block_uninstall` and + // `a_listener_naming_another_model_does_not_block_uninstall` + // driving the two proceed-on-a-live-socket arms through this + // call site. The tests live in `main.rs`'s `mod tests`, far from + // here; change an arm and expect them, not this loop, to be what + // goes red. + let verdict = stopped_record_verdict(probe.ok(), auth_refused); + if verdict.blocks() { + blocking.get_or_insert(verdict); + } else if !proceeding.contains(&verdict) { + proceeding.push(verdict); + } + } + let Some(verdict) = blocking else { + // The two fail-open outcomes disclose themselves; only + // `ProceedUnrelated` is silent, because a listener that named + // its models and did not name ours is the one case that is + // actually evidence of a stranger. They go into the report + // rather than straight to stderr so the disclosure is a value a + // test can assert on — see the field's own comment. + for verdict in proceeding { + match verdict { + StoppedRecordVerdict::ProceedListingNothing => { + report.warnings.push(format!( + "{}:{} still accepts connections and answered the identity probe \ + with an empty model list, so it cannot be told from an unrelated \ + service; {} is already recorded stopped; proceeding. An engine \ + still loading or unloading looks like this. If that is a server \ + of yours, it has to be found and stopped by hand: this uninstall \ + does not wait for you to.", + record.host, record.port, record.service_id + )); + } + StoppedRecordVerdict::ProceedUnidentified => { + report.warnings.push(format!( + "{}:{} still accepts connections but did not answer the identity \ + probe with a usable model list, and service {} is already \ + recorded stopped; proceeding. That can be a wedged engine, an \ + unrelated server on the port, or a stale endpoint key. If it is \ + a server of yours, it has to be found and stopped by hand: this \ + uninstall does not wait for you to.", + record.host, record.port, record.service_id + )); + } + StoppedRecordVerdict::ProceedUnrelated => {} + StoppedRecordVerdict::BlockServingOurModel + | StoppedRecordVerdict::BlockAuthRefused => { + unreachable!("only non-blocking verdicts are collected") + } + } + } + continue; + }; + if verdict == StoppedRecordVerdict::BlockAuthRefused { + report.failed.push(FailedManagedServiceStop { + service_id: record.service_id.clone(), + reason: format!( + "an authenticated server answers on {}:{}, this record's old port", + record.host, record.port + ), + remedy: StopFailureRemedy::RemoveTheStoppedRecord, + }); + continue; + } + } + // A confirmed stop already dropped this service's endpoint key, and it + // cannot be re-minted: say so on the failure rather than let the abort + // message read as if the survivor were untouched. + let key_dropped = report.stopped.contains(&record.service_id); + report + .stopped + .retain(|stopped| stopped != &record.service_id); + report + .relaunch + .retain(|line| !line.starts_with(&format!("{}: ", record.service_id))); + report.failed.push(FailedManagedServiceStop { + service_id: record.service_id.clone(), + // Two different situations reach here and they need different + // sentences. Only one of them involved a stop: the other is a + // record that was already marked stopped before this run, which + // this pass never attempted to stop, and telling its operator the + // endpoint survived "the stop" describes something that did not + // happen — on the one output they have to reason from. + reason: if stopped_by_this_pass { + format!( + "{}:{} still accepts connections after the stop{}", + record.host, + record.port, + if key_dropped { + "; the stop already dropped this service's endpoint key" + } else { + "" + } + ) + } else { + format!( + "{}:{} is recorded stopped, but something there is still serving this \ + record's own model", + record.host, record.port + ) + }, + // Not `StopTheService`: the recorded processes are gone, so + // `rocm services stop` has nothing left to kill and every retry + // would abort identically. + remedy: StopFailureRemedy::StopWhatHoldsThePort, + }); + } +} + +/// Whether two probe addresses can reach the same listener on one port. +/// +/// `localhost` names whichever loopback family the resolver puts first — `::1` +/// on many Windows hosts — so it can reach a listener on either, and is never +/// told apart from one. +pub(crate) fn hosts_name_one_listener(a: &str, b: &str) -> bool { + let is_loopback = |h: &str| matches!(h, "127.0.0.1" | "::1"); + a == b || (a == "localhost" && is_loopback(b)) || (b == "localhost" && is_loopback(a)) +} + +/// The addresses to probe for a service recorded on `host`, in order. +/// +/// A service bound to a wildcard address is reachable on loopback; connecting to +/// the wildcard itself is not portable. Which loopback, though, depends on what +/// the listener actually bound: `::` with the usual `IPV6_V6ONLY=1` (the default +/// on Windows) answers on `::1` and *refuses* `127.0.0.1`, while an `0.0.0.0` +/// bind is the mirror image. Probing one family only would read a live, +/// port-holding engine as "nothing is serving" and wave the removal through — +/// the exact defect this gate exists to close — so a wildcard yields both and +/// the caller blocks if either answers. +/// +/// The normalized form is what gets probed, not just what gets classified. +/// `loopback_tcp_port_is_reachable` resolves with `(host, port)`, which rejects +/// a bracketed literal like `[::1]` and anything with stray whitespace — and a +/// resolution failure reads as "nothing is serving", which is the wrong +/// direction for a check whose whole job is to catch a surviving engine +/// grandchild. Records do carry bracketed spellings (`loopback_host_key` +/// normalizes them too) because `--host` is free-form. +pub(crate) fn probe_hosts(host: &str) -> Vec { + // Trimmed and case-folded so the spellings a record can carry — `0.0.0.0`, + // `::`, `[::]`, `0:0:0:0:0:0:0:0`, `*`, or empty — all resolve to loopback + // rather than being probed literally (a literal wildcard connect is not + // portable, and would silently read as "nothing is serving"). + let normalized = host.trim().trim_matches(['[', ']']).to_ascii_lowercase(); + match normalized.as_str() { + "0.0.0.0" | "::" | "0:0:0:0:0:0:0:0" | "*" | "" => { + vec!["127.0.0.1".to_owned(), "::1".to_owned()] + } + _ => vec![normalized], + } +} + +/// What the gate does about a listener answering on an already-stopped record's +/// recorded port. +/// +/// Lifted out of the stop pass so each outcome can be asserted directly. Two of +/// these arms once went untested because reaching them meant standing up a +/// server that answers in a particular way and then reading stderr: mutating +/// either "proceed" arm into a block, or deleting a warning, left every test +/// green while changing what a destructive command does. +/// +/// Both halves of that are closed now, so the next reader should not infer a +/// gap from the paragraph above. `every_identity_answer_maps_to_exactly_one_gate_outcome` +/// pins the mapping here; `an_endpoint_listing_nothing_does_not_block_uninstall` +/// and `a_listener_naming_another_model_does_not_block_uninstall` drive the two +/// proceed-on-a-live-socket arms through the real call site; and the warnings +/// are values on [`ManagedServiceStopReport`] rather than `eprintln!`s, so the +/// disclosure is asserted rather than merely emitted. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum StoppedRecordVerdict { + /// Serving this record's own model: the engine outlived the supervisor whose + /// death marked the record stopped. + BlockServingOurModel, + /// Refused our credentials — a live server stating it guards this path. + BlockAuthRefused, + /// Named its models and ours was not among them. The only "not ours" that is + /// evidence of anything, so the only one that proceeds silently. + ProceedUnrelated, + /// Answered, but listed nothing. Not evidence of a stranger: an engine still + /// loading or mid-unload looks exactly like this while holding the port. + ProceedListingNothing, + /// Listening but unidentifiable — not an OpenAI endpoint, an unparseable + /// reply, or a server erroring on a rotated key. + ProceedUnidentified, +} + +impl StoppedRecordVerdict { + /// Whether this outcome stops the uninstall. + /// + /// Exhaustive on purpose, rather than `matches!` over the blocking pair. + /// The wildcard that `matches!` implies would default a newly added variant + /// to "proceed" — the direction that removes the tooling — and it would + /// compile, leaving the mistake to be caught by a test that can only + /// enumerate the variants that existed when it was written. Spelled out, + /// the compiler stops the next variant until somebody decides which side of + /// a destructive command it belongs on. + pub(crate) const fn blocks(self) -> bool { + match self { + Self::BlockServingOurModel | Self::BlockAuthRefused => true, + Self::ProceedUnrelated | Self::ProceedListingNothing | Self::ProceedUnidentified => { + false + } + } + } +} + +/// Decide [`StoppedRecordVerdict`] from what the identity probe answered. +/// +/// `identity` is `None` when the probe produced no usable model list at all; +/// `auth_refused` then says whether that was a 401/403 from a live server, which +/// is stronger evidence the port is held than a listing would be. +pub(crate) const fn stopped_record_verdict( + identity: Option, + auth_refused: bool, +) -> StoppedRecordVerdict { + match identity { + Some(rocm_core::EndpointIdentity::ServesExpectedModel) => { + StoppedRecordVerdict::BlockServingOurModel + } + Some(rocm_core::EndpointIdentity::ServesOtherModels) => { + StoppedRecordVerdict::ProceedUnrelated + } + Some(rocm_core::EndpointIdentity::ListsNoModels) => { + StoppedRecordVerdict::ProceedListingNothing + } + None if auth_refused => StoppedRecordVerdict::BlockAuthRefused, + None => StoppedRecordVerdict::ProceedUnidentified, + } +} + +/// Whether a record carrying no start-time cannot be proven ours. Both supported +/// platforms (Linux, Windows) can record one, so an absent value means the +/// record predates the field or its writer never stamped it; either way the pid +/// is unproven. +pub(crate) const fn record_predates_start_ticks(recorded_start_ticks: Option) -> bool { + recorded_start_ticks.is_none() +} + +/// Whether `endpoint_url` answered the identity probe with an auth refusal. +/// +/// A 401/403 is not a failure to reach the endpoint — it is a live HTTP server +/// stating that it guards this path, which is stronger evidence that the port is +/// still held than a model listing would be. It is also the shape the retry run +/// takes for a public service: stopping the recorded processes clears the stored +/// key, so the next `rocm uninstall` probes an authenticated survivor with no +/// credentials and gets exactly this. +/// +/// Anything else — unreachable, a timeout, a non-HTTP listener, a 200 whose body +/// did not parse — is not an answer this can act on, and stays with the caller's +/// fail-open. +pub(crate) fn endpoint_refused_authorization( + endpoint_url: &str, + endpoint_api_key: Option<&str>, +) -> bool { + matches!( + rocm_core::http_get_with_auth( + endpoint_url, + "/v1/models", + endpoint_api_key, + ENDPOINT_IDENTITY_PROBE_TIMEOUT, + ), + Ok(parts) if parts.status == 401 || parts.status == 403 + ) +} + +/// Service manifests that exist but cannot be parsed back into a record. +/// +/// [`load_managed_services`] skips these silently, which is right for listing — +/// one bad file should not break `rocm services list` — but wrong for uninstall: +/// a corrupt manifest can describe a live, GPU-holding server, and removing the +/// tooling while blind to it is the exact defect this gate closes. Returns the +/// full paths so the abort message can point at what to inspect. +/// +/// Only `*.json` directly under the services directory is a manifest; engine +/// state files live under their own engine directory +/// ([`AppPaths::service_engine_state_path`]), so they are not misread as corrupt +/// records here. +pub(crate) fn unreadable_service_manifests(paths: &AppPaths) -> Result> { + let services_dir = paths.services_dir(); + if !services_dir.is_dir() { + return Ok(Vec::new()); + } + let mut unreadable = Vec::new(); + for entry in fs::read_dir(&services_dir) + .with_context(|| format!("failed to read {}", services_dir.display()))? + { + let path = entry?.path(); + if path.extension().and_then(|value| value.to_str()) != Some("json") { + continue; + } + let bytes = + fs::read(&path).with_context(|| format!("failed to read {}", path.display()))?; + if serde_json::from_slice::(&bytes).is_err() { + unreadable.push(path); + } + } + unreadable.sort(); + Ok(unreadable) +} + +/// What an aborted run already did to the machine, for the abort message. +/// +/// The stop pass is not atomic: the pre-flight refuses before anything is +/// stopped, but a stop that fails, a helper that will not die, a survivor found +/// after the stops, or a later step that fails all abort a run that has already +/// stopped some services. Each stopped one is down for good, and stopping it +/// dropped its endpoint key, which cannot be re-minted: a publicly bound +/// service, or one launched with `--require-api-key`, must be served again to +/// come back. Empty when the run stopped nothing. +pub(crate) fn stopped_during_run_note(report: &ManagedServiceStopReport) -> String { + let services_note = if report.stopped.is_empty() { + String::new() + } else { + format!( + " These services were stopped during this run and stay stopped: {}. Stopping \ + them dropped their endpoint keys, so a service that was publicly bound, or \ + launched with `--require-api-key`, has to be served again; the command for \ + each, with its own model and port filled in: {} {}.", + report.stopped.join(", "), + report.relaunch.join("; "), + rocm_engine_protocol::RELAUNCH_KEY_CHOICE + ) + }; + let helper_note = if report.helper_stopped { + " The background helper (rocmd) was stopped and stays stopped: nothing supervises or \ + recovers managed services until it is started again." + } else if report + .failed + .iter() + .any(|failure| failure.remedy == StopFailureRemedy::StopTheDaemon) + { + " The background helper (rocmd) was signalled but not confirmed gone, so the servers \ + it supervised may already be down." + } else { + "" + }; + format!("{services_note}{helper_note}") +} + +/// Decide whether uninstall may proceed to remove files, given the outcome of +/// stopping managed services. +/// +/// Returns an optional line to print before removal proceeds, or an error that +/// aborts uninstall with nothing removed when a service could not be stopped — +/// so the binaries and service records needed to recover stay in place. Kept +/// separate from the removal it guards so the abort branch (the core safety +/// guarantee) is unit-testable without an unkillable process. +pub(crate) fn uninstall_removal_gate(report: &ManagedServiceStopReport) -> Result> { + if !report.failed.is_empty() { + let detail = report + .failed + .iter() + .map(|failure| format!("{} ({})", failure.service_id, failure.reason)) + .collect::>() + .join("; "); + let already_stopped = stopped_during_run_note(report); + let already_stopped = if already_stopped.is_empty() { + already_stopped + } else { + format!(" No files were removed, but:{already_stopped}") + }; + // Advice per failure class. A record that will not parse cannot be + // stopped by `rocm services stop` — that command loads the same file and + // fails identically — so pointing at it would make every retry abort the + // same way, which is exactly the dead end this gate must not create. + // + // The classes walked here are derived from the failures themselves and + // ordered by `advice_rank`, so no variant can be dropped by forgetting + // to extend a list; both that and `advice` are exhaustive matches. + let mut classes = report + .failed + .iter() + .map(|failure| failure.remedy) + .collect::>(); + classes.sort_unstable_by_key(|remedy| remedy.advice_rank()); + classes.dedup(); + let remedies = classes + .into_iter() + .map(|remedy| { + let ids = report + .failed + .iter() + .filter(|failure| failure.remedy == remedy) + .map(|failure| failure.service_id.clone()) + .collect::>(); + remedy.advice(&ids) + }) + .collect::>(); + let may_be_serving = report.failed.iter().any(|failure| { + matches!( + failure.remedy, + StopFailureRemedy::StopTheService | StopFailureRemedy::StopWhatHoldsThePort + ) + }); + bail!( + "uninstall aborted: could not stop managed service(s): {detail}.{} {}{}", + if may_be_serving { + " Their endpoints may still be serving and holding the GPU." + } else { + "" + }, + remedies.join(" "), + if already_stopped.is_empty() { + " No files were removed." + } else { + &already_stopped + } + ); + } + if report.stopped.is_empty() && !report.helper_stopped { + return Ok(None); + } + let mut line = String::new(); + if !report.stopped.is_empty() { + line = format!( + "stopped {} managed service(s) before removal", + report.stopped.len() + ); + } + if report.helper_stopped { + if !line.is_empty() { + line.push_str("; "); + } + line.push_str("stopped the background helper (rocmd)"); + } + Ok(Some(line)) +} diff --git a/apps/rocm/src/uninstall_plan.rs b/apps/rocm/src/uninstall_plan.rs index 996a201d3c..96777bbd5f 100644 --- a/apps/rocm/src/uninstall_plan.rs +++ b/apps/rocm/src/uninstall_plan.rs @@ -48,7 +48,7 @@ use anyhow::{Context, Result, bail}; use rocm_core::{AppPaths, daemon_binary_path}; use crate::managed_service_status::load_managed_services; -use crate::{remote, uninstall}; +use crate::{remote, uninstall, uninstall_gate}; #[derive(Debug, Clone, Default)] pub(crate) struct UninstallOptions { @@ -125,14 +125,14 @@ pub(crate) fn build_uninstall_plan( plan.warnings.push(note); } - let managed_services = load_managed_services(paths).unwrap_or_default(); - if !managed_services.is_empty() { - plan.warnings.push(format!( - "{} managed service record(s) exist under {}; background processes are not stopped automatically in this pass", - managed_services.len(), - paths.services_dir().display() - )); - } + // The gate propagates this error while the plan does not, so a plan that + // silently read zero records would tell the operator "no managed services" + // about a run that is going to abort on exactly that failure. Keep the plan + // non-fatal (it is also the read-only dry-run path) but say what happened. + let (managed_services, services_read_error) = match load_managed_services(paths) { + Ok(records) => (records, None), + Err(error) => (Vec::new(), Some(format!("{error:#}"))), + }; // Remote sessions are worse than local ones to drop silently. The model runs // on someone else's machine and its endpoint is published there, so removing @@ -176,9 +176,124 @@ pub(crate) fn build_uninstall_plan( plan.actions .sort_by(|left, right| left.path.cmp(&right.path)); plan.actions.dedup_by(|left, right| left.path == right.path); + + if let Some(error) = &services_read_error { + // The refusal is real only where the stop pass runs, and that pass is + // skipped when the tooling is kept; promising it there would be wrong. + let consequence = if plan_removes_recovery_tooling(&plan, paths) { + "Uninstall will refuse to remove anything until they can be read" + } else { + "This run keeps the tooling, so it does not check them" + }; + plan.warnings.push(format!( + "could not read the managed service records under {}: {error}. {consequence}", + paths.services_dir().display() + )); + } + + // What the stop pass would refuse, decided by the very function it runs first + // (`uninstall_preflight`), so the lines below describe the run that will + // happen instead of a promise the gate then breaks. Only meaningful where the + // stop pass runs at all: a run that keeps the tooling does not check. + let removes_recovery_tooling = plan_removes_recovery_tooling(&plan, paths); + let mut preflight_refusal: Option = None; + let mut preflight_warnings: Vec = Vec::new(); + if removes_recovery_tooling && services_read_error.is_none() { + match uninstall_gate::uninstall_preflight(paths) { + Ok(report) => { + preflight_refusal = uninstall_gate::preflight_refusal_line(&report); + preflight_warnings = report.warnings; + } + Err(error) => { + preflight_refusal = Some(format!( + "uninstall will refuse to remove anything, and will stop nothing, until the \ + managed service records under {} can be checked: {error:#}", + paths.services_dir().display() + )); + } + } + } + let refusal_line_present = preflight_refusal.is_some(); + let run_refuses = refusal_line_present || services_read_error.is_some(); + if let Some(line) = preflight_refusal { + plan.warnings.push(line); + } + // The fail-open disclosures the stop pass would make, shown while the + // operator can still decline rather than only after they have confirmed. + plan.warnings.append(&mut preflight_warnings); + + if !managed_services.is_empty() { + // Two independent conditions decide whether a server actually gets + // stopped, and the warning must reflect BOTH or it describes work + // uninstall never does: the record has to need stopping, and the plan has + // to be removing the tooling that stops it (`uninstall()` skips the whole + // stop pass otherwise). A run that will refuse stops nothing at all. + let in_use = managed_services + .iter() + .filter(|record| uninstall_gate::record_needs_stopping(record)) + .count(); + let services_dir = paths.services_dir().display().to_string(); + let total = managed_services.len(); + plan.warnings.push(if in_use == 0 { + // "none is in use", not "none has a running server": this counts + // statuses and recorded pids, and a record reads "stopped" the moment + // its supervisor is killed even if an engine grandchild kept the + // port. The stop pass probes for exactly that, so the plan must not + // promise it away — this line is the last thing read before + // confirming. + format!("{total} managed service record(s) exist under {services_dir}; none is recorded as running") + } else if !removes_recovery_tooling { + // Cache-only and other tooling-preserving runs: say what is true — + // the servers keep running, and they remain stoppable afterwards. + format!( + "{in_use} of {total} managed service record(s) under {services_dir} are in use (recorded as running, or with a recorded process still alive); this removal keeps `rocm services stop` and the service records, so those servers are left running" + ) + } else if run_refuses { + format!( + "{in_use} of {total} managed service record(s) under {services_dir} are in use (recorded as running, or with a recorded process still alive); none of them will be stopped, because uninstall will refuse (see above)" + ) + } else { + format!( + "{in_use} of {total} managed service record(s) under {services_dir} are in use (recorded as running, or with a recorded process still alive); those servers will be stopped before removal" + ) + }); + } + + // The helper is reported whether or not any service record exists: a live + // `rocmd` with an empty services directory still gets force-stopped. + if let Some(note) = uninstall_gate::background_helper_plan_note( + paths, + removes_recovery_tooling, + run_refuses, + refusal_line_present, + ) { + plan.warnings.push(note); + } + Ok(plan) } +/// Whether this plan removes what an operator would need to stop a managed +/// server afterwards — the `rocm`/`rocmd` binaries, or the service records under +/// the data directory. +/// +/// The stop-before-remove gate exists only for that case. A `--keep-binaries +/// --keep-data` run removes the cache alone: the binaries and every service +/// record survive, so `rocm services stop` still works and force-stopping live +/// GPU servers (or hard-aborting a cache cleanup because one will not stop) +/// would be pure collateral damage. +pub(crate) fn plan_removes_recovery_tooling(plan: &UninstallPlan, paths: &AppPaths) -> bool { + let services_dir = paths.services_dir(); + // The running executable is not an action on Windows (it is deleted as the + // command exits), but it is still the `rocm` that stops servers. + plan.self_delete.is_some() + || plan.actions.iter().any(|entry| { + entry.kind == "binary" + || entry.path.starts_with(&services_dir) + || services_dir.starts_with(&entry.path) + }) +} + /// Plan removing the installed binaries beside the running one, which /// `current_exe` names, and undoing the installer's PATH change for their /// folder. `--keep-binaries` keeps both, and so does a cargo build's folder. diff --git a/apps/rocmd/Cargo.toml b/apps/rocmd/Cargo.toml index 2f0064e18a..945a47e903 100644 --- a/apps/rocmd/Cargo.toml +++ b/apps/rocmd/Cargo.toml @@ -24,6 +24,3 @@ serde_json.workspace = true sha2 = "0.10" tokio.workspace = true ureq = { version = "2.12", features = ["native-certs"] } - -[target.'cfg(windows)'.dependencies] -windows-sys = { version = "0.61", features = ["Win32_Storage_FileSystem"] } diff --git a/apps/rocmd/src/common.rs b/apps/rocmd/src/common.rs index 87dfa11ebf..5ab82fa48e 100644 --- a/apps/rocmd/src/common.rs +++ b/apps/rocmd/src/common.rs @@ -333,15 +333,19 @@ pub(crate) fn ensure_public_service_has_endpoint_key( bail!( "refusing to recover a service that was launched with an endpoint API key but no \ longer has one: it would come back up without authentication, and something \ - outside this machine may still be publishing its port. Relaunch it with \ - `rocm serve --require-api-key` to issue a new key." + outside this machine may still be publishing its port. Relaunch it with `{}` {} to \ + issue a new key.", + rocm_engine_protocol::relaunch_command(host, true), + rocm_engine_protocol::RELAUNCH_KEY_CHOICE ); } if rocm_engine_protocol::is_public_bind_host(host) && !key_present { bail!( "refusing to respawn a service bound to the public host `{host}` without an endpoint \ - API key: it would come back up without authentication. Relaunch it with \ - `rocm serve --host {host} --allow-public-bind` to issue a new key." + API key: it would come back up without authentication. Relaunch it with `{}` {} to \ + issue a new key.", + rocm_engine_protocol::relaunch_command(host, false), + rocm_engine_protocol::RELAUNCH_KEY_CHOICE ); } Ok(()) @@ -458,9 +462,15 @@ mod tests { for host in ["127.0.0.1", "localhost", "::1"] { let error = ensure_public_service_has_endpoint_key(host, false, true) .expect_err("a service launched with a key must not be recovered without one"); + let rendered = format!("{error:#}"); + assert!(rendered.contains("without authentication"), "{rendered}"); assert!( - format!("{error:#}").contains("without authentication"), - "{error:#}" + rendered.contains(&format!("rocm serve --host {host} --port ")), + "the relaunch command must be one that can be run: {rendered}" + ); + assert!( + rendered.contains("--require-api-key"), + "a service launched with a key must be told to relaunch with one: {rendered}" ); } // With the key still present, recovery proceeds. @@ -475,9 +485,18 @@ mod tests { // the key gone the child would listen on that public host anonymously, // so the spawn must be refused instead. let error = ensure_public_service_has_endpoint_key("0.0.0.0", false, false).unwrap_err(); + let rendered = format!("{error:#}"); + assert!(rendered.contains("without authentication"), "{rendered}"); + // The command it names must be the public-bind one: `--allow-public-bind` + // and no `--require-api-key`, which this branch never asked for. + assert!( + rendered + .contains("rocm serve --host 0.0.0.0 --port --allow-public-bind`"), + "{rendered}" + ); assert!( - error.to_string().contains("without authentication"), - "{error:#}" + rendered.contains(rocm_engine_protocol::RELAUNCH_KEY_CHOICE), + "{rendered}" ); ensure_public_service_has_endpoint_key("0.0.0.0", true, false).unwrap(); diff --git a/apps/rocmd/src/persistence.rs b/apps/rocmd/src/persistence.rs index 847c92b7aa..a793b19d5b 100644 --- a/apps/rocmd/src/persistence.rs +++ b/apps/rocmd/src/persistence.rs @@ -91,6 +91,7 @@ mod tests { running: true, automations_enabled: true, daemon_pid: 1, + daemon_start_ticks: None, started_at_unix_ms: 1, last_tick_unix_ms: 1, local_webhook_endpoint: None, diff --git a/apps/rocmd/src/sandbox.rs b/apps/rocmd/src/sandbox.rs index d43e206e06..da7b2c8ad0 100644 --- a/apps/rocmd/src/sandbox.rs +++ b/apps/rocmd/src/sandbox.rs @@ -661,6 +661,14 @@ fn temp_sibling_path(path: &Path, suffix: &OsStr) -> Result { Ok(parent.join(file_name)) } +/// Stage-and-publish a file here, sharing only the publish step with `rocm-core`. +/// +/// Deliberately not [`rocm_core::write_file_atomically`]: only the +/// Windows-sensitive publish (`ReplaceFileW` and its fallbacks) is +/// single-sourced. The staging half stays local because it carries the +/// `suffix_for_attempt` and `before_publish` seams the tests drive. This path +/// does not `sync_all` before publishing, so it is atomic but carries no +/// crash-durability guarantee for the staged bytes. fn write_file_atomically(path: &Path, bytes: &[u8]) -> Result<()> { let temp_id = format!("{}-{}", std::process::id(), unix_time_millis()); write_file_atomically_with( @@ -746,60 +754,10 @@ where Ok(()) } -#[cfg(not(windows))] +/// The publish step lives in `rocm-core` so there is one implementation of the +/// Windows `ReplaceFileW` handling for the whole workspace. fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { - fs::rename(tmp, path) -} - -#[cfg(windows)] -fn publish_temp_file(tmp: &Path, path: &Path) -> io::Result<()> { - if path.try_exists()? { - return replace_file_windows(path, tmp); - } - - match fs::rename(tmp, path) { - Ok(()) => Ok(()), - Err(rename_error) => { - if path.try_exists()? { - replace_file_windows(path, tmp) - } else { - Err(rename_error) - } - } - } -} - -#[cfg(windows)] -#[allow(unsafe_code)] -fn replace_file_windows(path: &Path, replacement: &Path) -> io::Result<()> { - use std::os::windows::ffi::OsStrExt; - use windows_sys::Win32::Storage::FileSystem::ReplaceFileW; - - let path_wide: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); - let replacement_wide: Vec = replacement - .as_os_str() - .encode_wide() - .chain(Some(0)) - .collect(); - - // SAFETY: both path buffers are valid, NUL-terminated UTF-16 strings and - // remain alive for the duration of the synchronous Windows API call. The - // optional backup, exclude, and reserved pointers are intentionally null. - let replaced = unsafe { - ReplaceFileW( - path_wide.as_ptr(), - replacement_wide.as_ptr(), - std::ptr::null(), - 0, - std::ptr::null(), - std::ptr::null(), - ) - }; - if replaced == 0 { - Err(io::Error::last_os_error()) - } else { - Ok(()) - } + rocm_core::publish_temp_file(tmp, path) } pub(crate) fn sandbox_check_updates_value(output: CommandCapture) -> Value { diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index 79793d1f8a..2e26a179c3 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -360,14 +360,10 @@ fn terminate_recorded_pid(identity: &rocm_core::ProcessIdentity) -> rocm_core::T // it through an `__engine-serve-http` process. A root-only kill would leave // the engine running and still holding the device. // - // Identity verification does not apply on Windows. `process_start_ticks` - // has no `/proc` to read there and returns `None`, so no recorded identity - // can be refuted and the gate below only ever turns away a PID that is not - // running at all. A recorded PID that has been recycled by an unrelated - // process still passes it, and `taskkill /T /F` then takes that process and - // its whole subtree: the stale-PID hazard is unchanged on Windows, exactly - // as it was before this gate existed. Closing it needs `rocm_core` to read a - // Windows process creation time. + // Identity verification applies on Windows too: `process_start_ticks` reads + // the process creation time there, so a recycled PID is refuted before + // `taskkill /T /F` can take an unrelated process and its subtree. Only a + // record with no recorded start time has nothing to be refuted against. #[cfg(windows)] if matches!( rocm_core::identity_state(identity), @@ -935,6 +931,9 @@ fn build_runtime_state( running: automations_enabled, automations_enabled, daemon_pid: std::process::id(), + // Captured while this process is alive so a later `rocm uninstall` can + // tell this daemon from an unrelated process that inherited the PID. + daemon_start_ticks: rocm_core::ProcessIdentity::capture(std::process::id()).start_ticks, started_at_unix_ms: now, last_tick_unix_ms: now, local_webhook_endpoint: None, @@ -1062,6 +1061,23 @@ fn wait_for_service_ready( #[cfg(test)] mod tests { use super::*; + + #[cfg(any(target_os = "linux", windows))] + #[test] + fn a_fresh_runtime_state_records_this_daemons_start_time() { + // `rocm uninstall` refuses to signal a daemon pid it cannot prove is + // rocmd, and the proof is this field: `None` here would make every + // record look pre-upgrade and abort every uninstall that finds a daemon. + let state = build_runtime_state(&RocmCliConfig::default(), true); + assert_eq!(state.daemon_pid, std::process::id()); + let recorded = state + .daemon_start_ticks + .expect("a live daemon must record its own start-time"); + assert_eq!( + Some(recorded), + rocm_core::process_start_ticks(std::process::id()) + ); + } use crate::test_support::identity_probe_record; use crate::test_support::{ UNCONFIRMED_STOP_PID, seed_keyed_service, temp_app_paths, unique_test_root, diff --git a/apps/rocmd/src/watchers.rs b/apps/rocmd/src/watchers.rs index 79efcb2692..06a3310255 100644 --- a/apps/rocmd/src/watchers.rs +++ b/apps/rocmd/src/watchers.rs @@ -1700,6 +1700,7 @@ mod tests { running: true, automations_enabled: true, daemon_pid: 1, + daemon_start_ticks: None, started_at_unix_ms: 1, last_tick_unix_ms: 1, local_webhook_endpoint: None, @@ -2882,6 +2883,7 @@ mod tests { running: true, automations_enabled: true, daemon_pid: 1, + daemon_start_ticks: None, started_at_unix_ms: 1, last_tick_unix_ms: 1, local_webhook_endpoint: None, @@ -2915,6 +2917,7 @@ mod tests { running: true, automations_enabled: true, daemon_pid: 1, + daemon_start_ticks: None, started_at_unix_ms: 1, last_tick_unix_ms: 1, local_webhook_endpoint: None, @@ -2981,6 +2984,7 @@ mod tests { running: true, automations_enabled: true, daemon_pid: 1, + daemon_start_ticks: None, started_at_unix_ms: 1, last_tick_unix_ms: 1, local_webhook_endpoint: None, @@ -3070,6 +3074,7 @@ mod tests { running: true, automations_enabled: true, daemon_pid: 1, + daemon_start_ticks: None, started_at_unix_ms: 1, last_tick_unix_ms: 1, local_webhook_endpoint: None, @@ -3125,6 +3130,7 @@ mod tests { running: true, automations_enabled: true, daemon_pid: 1, + daemon_start_ticks: None, started_at_unix_ms: 1, last_tick_unix_ms: 1, local_webhook_endpoint: None, diff --git a/crates/rocm-core/src/automation.rs b/crates/rocm-core/src/automation.rs index fc7f727fd6..b8a1d08762 100644 --- a/crates/rocm-core/src/automation.rs +++ b/crates/rocm-core/src/automation.rs @@ -16,10 +16,10 @@ //! line — its cost does not grow with the store. A torn tail — crash debris, //! or left by a write that failed partway — is cut by the next append rather //! than truncated in place: that append writes the store's clean prefix plus -//! its own new record as a single rewrite via `write_atomic`, so a lock-free +//! its own new record as a single rewrite via `write_file_atomically`, so a lock-free //! reader never observes a file shrunk mid-repair. That rewrite costs O(store) //! rather than O(record), the same as a proposal status update, which rewrites -//! the whole store via `write_atomic` too. Nothing in this module ever +//! the whole store via `write_file_atomically` too. Nothing in this module ever //! shrinks a store in place. Readers take no lock, and an unterminated last //! line that is not valid JSON — an append caught mid-write — is skipped //! rather than reported as corruption. The stores are not rotated or capped. @@ -42,6 +42,7 @@ use crate::FileLock; use crate::WatcherMode; use crate::require_nonempty; use crate::unix_time_millis; +use crate::write_file_atomically; #[derive(Debug, Clone, Copy)] pub struct BuiltinWatcherSpec { @@ -122,6 +123,28 @@ pub struct AutomationRuntimeState { pub running: bool, pub automations_enabled: bool, pub daemon_pid: u32, + /// The daemon's kernel start-time, captured at spawn, so `daemon_pid` can be + /// checked for PID recycling before anything signals it. + /// + /// `daemon_pid` alone is not safe to kill: this file survives a crash, + /// OOM-kill or reboot with `running` still true, after which the OS can + /// reissue that PID to an unrelated process. Pairing the PID with its + /// start-time makes a recycled PID detectable ([`identity_state`]). + /// + /// `None` on two very different occasions, which callers must not conflate: + /// a state file written before this field existed, and a platform where + /// [`process_start_ticks`] can never return anything. + /// + /// Conflating them is a live hazard, because [`identity_state`] maps a `None` + /// recorded value to [`IdentityState::Matches`] — the legacy best-effort arm + /// — so **no recycling check happens at all** and a caller that signals on + /// that verdict is killing on a bare pid. Both supported platforms (Linux + /// `/proc`, Windows `GetProcessTimes`) can record a start-time, so there an + /// absent value means the record predates the field or its writer never + /// stamped it, and the pid is unverifiable. Only an unsupported platform + /// (neither source, i.e. macOS) can never produce one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub daemon_start_ticks: Option, pub started_at_unix_ms: u128, pub last_tick_unix_ms: u128, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -143,15 +166,21 @@ impl AutomationRuntimeState { Ok(Some(state)) } + /// Publish the runtime state atomically. + /// + /// Written on every daemon tick, and read by `rocm uninstall` to decide + /// whether a background helper is still up. A plain `fs::write` leaves a + /// truncated file if the daemon dies mid-write, and uninstall has to treat an + /// unreadable state file as "cannot confirm the helper is stopped" — so a + /// crash at the wrong moment would block uninstall until the operator + /// repaired the file by hand. pub fn write(&self, paths: &AppPaths) -> Result<()> { paths.ensure()?; let path = paths.automation_state_path(); - fs::write( - &path, - serde_json::to_vec_pretty(self) - .context("failed to serialize automation runtime state")?, - ) - .with_context(|| format!("failed to write {}", path.display()))?; + let bytes = serde_json::to_vec_pretty(self) + .context("failed to serialize automation runtime state")?; + write_file_atomically(&path, &bytes) + .with_context(|| format!("failed to write {}", path.display()))?; Ok(()) } @@ -298,12 +327,12 @@ fn append_jsonl_record(path: &Path, record: &T, context_label: &st // `file` open at function return. The rename already succeeds // with this handle open regardless — std opens files with // `FILE_SHARE_DELETE` by default on Windows, so a rename isn't - // blocked by it (see `write_atomic`'s doc for the case that + // blocked by it (see `write_file_atomically`'s doc for the case that // *isn't* covered: another process holding the destination open // without delete-sharing) — but this states the intent rather // than leaning on that default. drop(file); - crate::write_atomic(path, &content, "jsonl")?; + crate::write_file_atomically(path, &content)?; } TailState::ValidUnterminated(start) => { line.insert(0, b'\n'); @@ -430,7 +459,7 @@ fn inspect_tail(path: &Path, file: &fs::File) -> Result { /// /// Returns fewer bytes if the file now ends before `end`. Nothing this /// module's own writers do can shrink a store in place — an append only ever -/// grows it, and a torn-tail repair replaces it wholesale via `write_atomic` +/// grows it, and a torn-tail repair replaces it wholesale via `write_file_atomically` /// rather than truncating the live file (see [`inspect_tail`] and /// [`TailState::Torn`]) — so a short read here means the store was edited or /// truncated out-of-band, a case this module does not try to recover from @@ -833,7 +862,7 @@ fn write_automation_proposals( ); text.push('\n'); } - crate::write_atomic(&path, text.as_bytes(), "jsonl") + crate::write_file_atomically(&path, text.as_bytes()) } fn normalize_proposal_identity(proposal: &mut AutomationProposalRecord, index: usize) { diff --git a/crates/rocm-core/src/config.rs b/crates/rocm-core/src/config.rs index b1efe4b1ab..5980c7e4a6 100644 --- a/crates/rocm-core/src/config.rs +++ b/crates/rocm-core/src/config.rs @@ -542,7 +542,7 @@ impl RocmCliConfig { .with_context(|| format!("failed to parse {}", path.display())) } - /// Persist the config by replacing it via the shared `write_atomic` + /// Persist the config by replacing it via the shared `write_file_atomically` /// helper (write a sibling temp file, then rename it over `config.json`; /// see that helper's own doc for the write/rename/cleanup guarantees). /// @@ -556,7 +556,7 @@ impl RocmCliConfig { .with_context(|| format!("failed to create {}", paths.config_dir.display()))?; let bytes = serde_json::to_vec_pretty(self).context("failed to serialize rocm-cli config")?; - crate::write_atomic(&path, &bytes, "json") + crate::write_file_atomically(&path, &bytes) .with_context(|| format!("failed to save rocm-cli config to {}", path.display())) } diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 9ea27e6074..d9262c3a3a 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -3,8 +3,9 @@ // SPDX-License-Identifier: MIT use anyhow::{Context, Result, bail}; -use std::ffi::OsString; +use std::ffi::{OsStr, OsString}; use std::fs; +use std::io::Write; use std::path::{Path, PathBuf}; use std::time::{SystemTime, UNIX_EPOCH}; @@ -132,15 +133,16 @@ pub use model_catalog::{ }; pub use net::{ Backoff, DEFAULT_LOCAL_HOST, DEFAULT_LOCAL_PORT, DOWNLOAD_MAX_ATTEMPTS, DownloadOutcome, - DownloadRequest, EndpointReadiness, EndpointReadinessOutcome, HttpResponseParts, - INFERENCE_PROBE_ATTEMPTED_STATE_KEY, INFERENCE_PROBE_RETRY_INTERVAL, INFERENCE_PROBE_TIMEOUT, - INFERENCE_VERIFIED_STATE_KEY, connect_tcp_stream, download_file_streaming, - download_file_streaming_with_progress, download_file_to_path, + DownloadRequest, EndpointIdentity, EndpointReadiness, EndpointReadinessOutcome, + HttpResponseParts, INFERENCE_PROBE_ATTEMPTED_STATE_KEY, INFERENCE_PROBE_RETRY_INTERVAL, + INFERENCE_PROBE_TIMEOUT, INFERENCE_VERIFIED_STATE_KEY, connect_tcp_stream, + download_file_streaming, download_file_streaming_with_progress, download_file_to_path, download_file_to_path_with_progress, engine_state_inference_verified, format_host_for_url, format_host_port, format_http_base_url, http_get_text, http_get_text_with_auth, - http_post_json_with_auth, managed_service_endpoint_model_ready, - managed_service_endpoint_readiness, merge_json_state_file, openai_chat_completion_probe, - openai_chat_completion_status, openai_models_endpoint_has_model, parse_http_endpoint, + http_get_with_auth, http_post_json_with_auth, managed_service_endpoint_identity, + managed_service_endpoint_model_ready, managed_service_endpoint_readiness, + merge_json_state_file, openai_chat_completion_probe, openai_chat_completion_status, + openai_models_endpoint_has_model, openai_models_endpoint_identity, parse_http_endpoint, read_http_response_bounded, write_all_tcp_stream, }; pub use paths::{AppPaths, engine_plugin_dirs}; @@ -337,61 +339,188 @@ pub fn unix_time_millis() -> u128 { .as_millis() } -/// Replace `path`'s content with `bytes`, atomically. +/// How many temp names to try before giving up on staging an atomic write. +const ATOMIC_WRITE_TEMP_ATTEMPTS: u32 = 128; + +/// A unique temp path next to `path`, preserving the full file name so a +/// multi-extension artifact keeps its extensions (`sdk.tar.gz` becomes +/// `sdk.tar.gz.tmp-`, where `with_extension` would drop `.gz`). +fn temp_sibling_path(path: &Path, suffix: &OsStr) -> Result { + let parent = path.parent().context("file path has no parent directory")?; + let mut file_name = path + .file_name() + .context("file path has no file name")? + .to_os_string(); + file_name.push(".tmp-"); + file_name.push(suffix); + Ok(parent.join(file_name)) +} + +/// Move a staged temp file onto `path`, replacing whatever is there. +/// +/// The single implementation of the publish step for the whole workspace. It +/// lives here rather than in each caller because the Windows half is not +/// something to re-derive: a plain `rename` over an existing file is not +/// reliable while another process holds the destination open (an antivirus or +/// indexer scanning a file this CLI just wrote is enough), so an existing +/// destination goes through `ReplaceFileW`. Callers that got this wrong would +/// fail in the direction that matters — `ManagedServiceRecord::write` is on the +/// uninstall stop path, where a failed publish is reported as a service that +/// could not be stopped and aborts the whole uninstall. +#[cfg(not(windows))] +pub fn publish_temp_file(tmp: &Path, path: &Path) -> std::io::Result<()> { + fs::rename(tmp, path) +} + +/// Move a staged temp file onto `path`, replacing whatever is there. +/// +/// See the non-Windows twin for why this is centralized. +#[cfg(windows)] +pub fn publish_temp_file(tmp: &Path, path: &Path) -> std::io::Result<()> { + if path.try_exists()? { + return replace_file_windows(path, tmp); + } + + match fs::rename(tmp, path) { + Ok(()) => Ok(()), + Err(rename_error) => { + // The destination can appear between the check and the rename. + if path.try_exists()? { + replace_file_windows(path, tmp) + } else { + Err(rename_error) + } + } + } +} + +#[cfg(windows)] +#[allow(unsafe_code)] // Win32 FFI +fn replace_file_windows(path: &Path, replacement: &Path) -> std::io::Result<()> { + use std::os::windows::ffi::OsStrExt; + use windows_sys::Win32::Storage::FileSystem::ReplaceFileW; + + let path_wide: Vec = path.as_os_str().encode_wide().chain(Some(0)).collect(); + let replacement_wide: Vec = replacement + .as_os_str() + .encode_wide() + .chain(Some(0)) + .collect(); + + // SAFETY: both path buffers are valid, NUL-terminated UTF-16 strings and + // remain alive for the duration of the synchronous Windows API call. The + // optional backup, exclude, and reserved pointers are intentionally null. + let replaced = unsafe { + ReplaceFileW( + path_wide.as_ptr(), + replacement_wide.as_ptr(), + std::ptr::null(), + 0, + std::ptr::null(), + std::ptr::null(), + ) + }; + if replaced == 0 { + Err(std::io::Error::last_os_error()) + } else { + Ok(()) + } +} + +/// Write `bytes` to `path` so a concurrent reader sees either the old contents +/// or the new ones, never a half-written file. /// -/// Writes a sibling temp file and renames it over the destination: an -/// interrupted write (full disk, crash, killed process) can never leave a -/// truncated file in place, and a reader sees -/// either the old content or the new content. The destination is NOT removed -/// first: `fs::rename` replaces an existing file on both supported platforms -/// (it is `MoveFileEx` with `MOVEFILE_REPLACE_EXISTING` on Windows), and -/// removing it would open a window with no file at all, which a concurrent -/// reader could see as "file doesn't exist" instead of the old or new -/// content. The temp file is removed if the write or the rename fails. +/// Staged under a unique sibling name reserved with `create_new` — so two +/// writers cannot pick the same scratch file — and published with +/// [`publish_temp_file`]. A failed publish takes the scratch file with it. /// -/// `tmp_ext` names the temp file's extension (e.g. `"json"`, `"jsonl"`). -/// Both halves of the generated suffix matter: the timestamp keeps -/// successive writes to the same path apart, and the pid keeps two -/// processes writing in the same millisecond from clobbering each other's -/// temp file and losing one of the two updates. +/// Four limits worth knowing before reaching for this: /// -/// Not guaranteed: -/// - Serializing writers: this takes no lock, so two read-modify-write -/// cycles that overlap still lose one update (last rename wins). A caller -/// whose new content depends on the old must hold a lock across the whole -/// cycle, as the JSONL proposal store does (`automation::jsonl_lock_path`). -/// Two callers in the *same process* within the same millisecond are a -/// sharper case of this, not just a milder one: they generate the exact -/// same temp path below, so concurrent `fs::write`s truncating that one -/// path can publish mixed bytes, and the losing rename then fails outright -/// (its source file already gone) rather than merely losing an update — -/// this is the same-process collision -/// `automation::tests::concurrent_appends_from_many_threads_lose_no_records`'s -/// own comment describes ("two threads of one process even shared a -/// temp-file name") — fixed for its own callers by the per-store lock -/// `automation.rs` takes before ever calling into this function. -/// - Durability: the bytes are not fsynced before the rename, so a power -/// loss can still surface the new name with unflushed contents. -/// - Replacing a destination another process holds open without delete -/// sharing: on Windows the rename then fails with a sharing violation, -/// this returns that error, and the old destination stays in place. -pub fn write_atomic(path: &Path, bytes: &[u8], tmp_ext: &str) -> Result<()> { - let tmp_path = path.with_extension(format!( - "{tmp_ext}.tmp-{}-{}", - unix_time_millis(), - std::process::id() - )); - fs::write(&tmp_path, bytes) - .with_context(|| format!("failed to write {}", tmp_path.display())) - .inspect_err(|_| { - let _ = fs::remove_file(&tmp_path); - })?; - fs::rename(&tmp_path, path) - .with_context(|| format!("failed to replace {}", path.display())) +/// * **Atomic, not durable.** The staged bytes are `sync_all`ed before the +/// publish, so a crash cannot leave a half-written or zero-length file. The +/// containing *directory* is never fsynced, so the rename itself can still be +/// lost by a power failure — the guarantee is that readers only ever see one +/// complete version, not that the newest one survives a crash. That flush is +/// held by construction, not by test: deleting it leaves every test green, +/// because reproducing what it prevents needs a real crash between the write +/// and the rename. Do not read a green suite as evidence it is still there. +/// * **Permissions are not carried over.** This replaces the target inode with +/// a freshly created file, so an existing file's mode does not survive (a +/// difference from the `fs::write` it usually replaces), and the new file gets +/// whatever the umask allows. The containing directory does not make up for +/// it: `AppPaths::ensure` creates it with plain `create_dir_all` and no mode of +/// its own. A caller with something sensitive to write must set the mode +/// explicitly here; it cannot lean on the directory. `RocmCliConfig::save` is +/// such a caller today (it persists the dashboard token) and does not yet. +/// * **No lock.** Two read-modify-write cycles that overlap still lose one +/// update (last rename wins), so a caller whose new content depends on the old +/// must hold a lock across the whole cycle, as the JSONL stores in +/// `automation` do. +/// * **A symlink at `path` is replaced, not written through.** The publish is a +/// rename onto `path` itself, so a symlink there is what gets replaced — where +/// `fs::write` would have followed it and overwritten the target. That is the +/// safer direction for state files (it cannot be aimed somewhere else by +/// planting a link) but it is a behaviour change, so a caller that means to +/// write through a link must resolve it first. +pub fn write_file_atomically(path: &Path, bytes: &[u8]) -> Result<()> { + let temp_id = format!("{}-{}", std::process::id(), unix_time_millis()); + write_file_atomically_with_temp_id(path, bytes, &temp_id) +} + +/// [`write_file_atomically`] with the scratch-file id supplied, so a test can +/// plant files under the names the attempts will try and drive the +/// already-exists retry and the exhausted-attempts failure deterministically. +fn write_file_atomically_with_temp_id(path: &Path, bytes: &[u8], temp_id: &str) -> Result<()> { + let parent = path.parent().context("file path has no parent directory")?; + fs::create_dir_all(parent).with_context(|| format!("failed to create {}", parent.display()))?; + + let mut reserved = None; + for attempt in 0..ATOMIC_WRITE_TEMP_ATTEMPTS { + let tmp = temp_sibling_path(path, &OsString::from(format!("{temp_id}-{attempt}")))?; + match fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&tmp) + { + Ok(file) => { + reserved = Some((tmp, file)); + break; + } + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => {} + Err(error) => { + return Err(error).with_context(|| format!("failed to create {}", tmp.display())); + } + } + } + let Some((tmp, mut file)) = reserved else { + bail!( + "failed to reserve a temporary file next to {} after {ATOMIC_WRITE_TEMP_ATTEMPTS} \ + attempts", + path.display() + ); + }; + + if let Err(error) = file.write_all(bytes) { + drop(file); + let _ = fs::remove_file(&tmp); + return Err(disk_space::map_write_error(error, &tmp)); + } + // Flush before publishing. Without this the rename can reach the disk while + // the bytes have not, leaving a zero-length file after a crash — which for a + // service record is not a lost update but an unparseable manifest, and the + // uninstall gate refuses to remove anything while one of those exists. + if let Err(error) = file.sync_all() { + drop(file); + let _ = fs::remove_file(&tmp); + return Err(disk_space::map_write_error(error, &tmp)); + } + drop(file); + + publish_temp_file(&tmp, path) .inspect_err(|_| { - let _ = fs::remove_file(&tmp_path); - })?; - Ok(()) + let _ = fs::remove_file(&tmp); + }) + .with_context(|| format!("failed to publish {}", path.display())) } #[cfg(test)] @@ -401,6 +530,385 @@ mod tests { use std::fs; use std::path::Path; + /// A private root for one test, removed on the way out. + fn atomic_write_root(name: &str) -> PathBuf { + let root = std::env::temp_dir().join(format!( + "rocm-core-atomic-write-{name}-{}-{}", + std::process::id(), + unix_time_millis() + )); + let _ = fs::remove_dir_all(&root); + fs::create_dir_all(&root).expect("failed to create the test root"); + root + } + + /// The scratch files `write_file_atomically` stages under, if any survived. + /// + /// `temp_sibling_path` names them `.tmp---`, so + /// a leftover is visible as a sibling containing `.tmp-`. + fn leftover_scratch_files(dir: &Path) -> Vec { + fs::read_dir(dir) + .expect("failed to read the test root") + .filter_map(|entry| { + let name = entry.ok()?.file_name().to_string_lossy().into_owned(); + name.contains(".tmp-").then_some(name) + }) + .collect() + } + + #[test] + fn a_taken_scratch_name_is_skipped_and_the_write_still_lands() { + // Another writer already holds attempt 0's name: the write must move on to + // the next attempt rather than fail or clobber the file it does not own. + let root = atomic_write_root("scratch-name-taken"); + let target = root.join("record.json"); + let taken = root.join("record.json.tmp-fixed-0"); + fs::write(&taken, b"someone else's scratch").expect("plant the taken name"); + + write_file_atomically_with_temp_id(&target, b"mine", "fixed").expect("write"); + + assert_eq!(fs::read(&target).expect("read"), b"mine"); + assert_eq!( + fs::read(&taken).expect("read"), + b"someone else's scratch", + "a scratch file that is not ours is left untouched" + ); + assert_eq!( + leftover_scratch_files(&root), + vec!["record.json.tmp-fixed-0".to_owned()], + "our own scratch file is gone once published" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn every_scratch_name_taken_is_an_error_not_a_clobber() { + let root = atomic_write_root("scratch-names-exhausted"); + let target = root.join("record.json"); + fs::write(&target, b"original").expect("seed the target"); + for attempt in 0..ATOMIC_WRITE_TEMP_ATTEMPTS { + fs::write(root.join(format!("record.json.tmp-fixed-{attempt}")), b"x") + .expect("plant a taken name"); + } + + let error = write_file_atomically_with_temp_id(&target, b"new", "fixed") + .expect_err("no scratch name is free"); + + assert!( + format!("{error:#}").contains("failed to reserve a temporary file"), + "{error:#}" + ); + assert_eq!( + fs::read(&target).expect("read"), + b"original", + "a failed reservation leaves the target as it was" + ); + let _ = fs::remove_dir_all(root); + } + + /// Linux/unix: relies on a rename replacing the directory entry while an + /// already-open descriptor keeps reading the previous inode. Windows + /// publishes through `ReplaceFileW` and does not offer the same observation. + #[cfg(unix)] + #[test] + fn a_service_record_write_never_shows_a_reader_a_truncated_manifest() { + // `ManagedServiceRecord::write` runs on the uninstall stop path, where a + // torn manifest is not a lost update but an unparseable record that makes + // the gate refuse to remove anything. + // + // Asserting "the bytes landed" would not pin that: a plain `fs::write` + // also leaves correct final bytes and creates no scratch file, so such a + // test passes either way. The property only the atomic path has is what a + // *concurrent reader* sees — `fs::write` truncates the existing inode in + // place, so a reader holding it open watches the record disappear and + // come back, while the publish swaps in a new inode and leaves the old + // one whole. Hold the manifest open across a rewrite and require the old + // view to still be a complete, parseable record. + use std::io::{Read, Seek, SeekFrom}; + + let root = atomic_write_root("service-record-reader-safety"); + let paths = AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + paths.ensure().expect("create the app directories"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-reader-safety", + "vllm", + "amd/model", + "amd/model", + "127.0.0.1", + 8000, + "managed", + 0, + None, + None, + None, + ); + record.status = "ready".to_owned(); + record.write().expect("seed the manifest"); + + // A reader that opened the manifest before the rewrite began. + let mut reader = fs::File::open(&record.manifest_path).expect("open the manifest"); + + record.status = "stopped".to_owned(); + record.write().expect("rewrite the manifest"); + + reader.seek(SeekFrom::Start(0)).expect("rewind"); + let mut seen = Vec::new(); + reader + .read_to_end(&mut seen) + .expect("read the open manifest"); + let parsed: ManagedServiceRecord = serde_json::from_slice(&seen) + .expect("a reader holding the manifest open must never see a truncated document"); + assert_eq!( + parsed.status, "ready", + "the open descriptor must still see the pre-write record, not a rewritten inode" + ); + + // And the published file is the new one. + let published: ManagedServiceRecord = + serde_json::from_slice(&fs::read(&record.manifest_path).expect("read back")) + .expect("the published manifest must parse"); + assert_eq!(published.status, "stopped", "the rewrite must have landed"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_runtime_state_write_never_shows_a_reader_a_truncated_file() { + // The uninstall gate reads this file to decide whether a daemon is live; a + // torn write makes it unparseable and aborts the run. Same property as + // the service record: a reader holding the old inode must keep seeing a + // whole document across a rewrite. + use std::io::{Read, Seek, SeekFrom}; + + let root = atomic_write_root("runtime-state-reader-safety"); + let paths = AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + paths.ensure().expect("create the app directories"); + let mut state = AutomationRuntimeState { + running: true, + automations_enabled: true, + daemon_pid: 1, + daemon_start_ticks: None, + started_at_unix_ms: 1, + last_tick_unix_ms: 1, + local_webhook_endpoint: None, + active_watchers: Vec::new(), + }; + state.write(&paths).expect("seed the state file"); + let mut reader = + fs::File::open(paths.automation_state_path()).expect("open the state file"); + + state.running = false; + state.write(&paths).expect("rewrite the state file"); + + reader.seek(SeekFrom::Start(0)).expect("rewind"); + let mut seen = Vec::new(); + reader.read_to_end(&mut seen).expect("read the open file"); + let parsed: AutomationRuntimeState = serde_json::from_slice(&seen) + .expect("a reader holding the file open must never see a truncated document"); + assert!( + parsed.running, + "the open descriptor must see the pre-write state" + ); + let published = AutomationRuntimeState::load(&paths) + .expect("load") + .expect("present"); + assert!(!published.running, "the rewrite must have landed"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn a_service_record_write_leaves_no_scratch_sibling() { + // This pins scratch-file hygiene, NOT atomic publishing: a plain + // `fs::write` creates no scratch file at all and would pass too. The + // atomicity property is pinned next door, by + // `a_service_record_write_never_shows_a_reader_a_truncated_manifest`, + // which fails against an in-place write. + // + // It still earns its place, but for a smaller reason than uninstall: + // scratch siblings are named `.json.tmp---`, + // whose extension is `tmp-…`, and both `unreadable_service_manifests` + // and the record loader filter on `extension == "json"` — so a leaked + // one is skipped, not mistaken for a corrupt record, and cannot abort + // uninstall. What this guards is directory hygiene: a publish that + // aborts between staging and rename leaves litter that accumulates + // silently, one file per failed write, in a directory an operator reads + // by hand. + let root = atomic_write_root("service-record-publish"); + let paths = AppPaths { + config_dir: root.join("config"), + data_dir: root.join("data"), + cache_dir: root.join("cache"), + }; + paths.ensure().expect("create the app directories"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-atomic-publish", + "vllm", + "amd/model", + "amd/model", + "127.0.0.1", + 8000, + "managed", + 0, + None, + None, + None, + ); + record.write().expect("first publish"); + record.status = "stopped".to_owned(); + record + .write() + .expect("republish over the existing manifest"); + + let services_dir = paths.services_dir(); + let stray = fs::read_dir(&services_dir) + .expect("read the services directory") + .filter_map(|entry| { + let name = entry.ok()?.file_name().to_string_lossy().into_owned(); + name.contains(".tmp-").then_some(name) + }) + .collect::>(); + assert!( + stray.is_empty(), + "publishing a record must leave no scratch manifest: {stray:?}" + ); + let written = fs::read(&record.manifest_path).expect("read the manifest back"); + let parsed: ManagedServiceRecord = + serde_json::from_slice(&written).expect("the published manifest must always parse"); + assert_eq!(parsed.status, "stopped", "the republish must have landed"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn write_file_atomically_replaces_contents_and_leaves_no_scratch_file() { + // The publish must land the new bytes and take its staging file with it; + // a leaked scratch sibling in `services_dir` would be a file the uninstall + // gate has to reason about. + let root = atomic_write_root("replaces"); + let target = root.join("record.json"); + fs::write(&target, b"old contents").expect("seed the file"); + + write_file_atomically(&target, b"new contents").expect("the write must succeed"); + + assert_eq!( + fs::read(&target).expect("read back"), + b"new contents", + "the published file must hold the new bytes" + ); + assert!( + leftover_scratch_files(&root).is_empty(), + "a successful publish must leave no scratch file: {:?}", + leftover_scratch_files(&root) + ); + let _ = fs::remove_dir_all(root); + } + + /// Unix: needs a real symlink. + #[cfg(unix)] + #[test] + fn write_file_atomically_replaces_a_symlink_instead_of_writing_through_it() { + // Documented behaviour, and a change from the `fs::write` this replaced: + // that followed a symlink and overwrote whatever it pointed at, while + // the publish here renames onto the path itself. Security-relevant in + // the safe direction — planting a link next to a state file can no + // longer redirect the write somewhere else — but a change, so it is + // pinned rather than left as prose. + let root = atomic_write_root("symlink-target"); + let elsewhere = root.join("elsewhere"); + fs::write(&elsewhere, b"must not be touched").expect("seed the link target"); + let target = root.join("state.json"); + std::os::unix::fs::symlink(&elsewhere, &target).expect("plant a symlink at the target"); + + write_file_atomically(&target, b"published").expect("the write must succeed"); + + assert_eq!( + fs::read(&elsewhere).expect("read the link target"), + b"must not be touched", + "the write must not follow the symlink to its target" + ); + assert_eq!( + fs::read(&target).expect("read the published file"), + b"published", + "the published bytes must land at the path itself" + ); + assert!( + !fs::symlink_metadata(&target) + .expect("stat the published path") + .file_type() + .is_symlink(), + "the symlink must have been replaced by a regular file" + ); + assert!( + leftover_scratch_files(&root).is_empty(), + "no scratch file may survive a successful publish" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn write_file_atomically_creates_the_parent_directory() { + // `ManagedServiceRecord::write` targets a services dir that may not exist + // yet on a first write. + let root = atomic_write_root("creates-parent"); + let target = root.join("nested").join("deeper").join("record.json"); + + write_file_atomically(&target, b"contents").expect("the write must succeed"); + + assert_eq!(fs::read(&target).expect("read back"), b"contents"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn write_file_atomically_cleans_up_when_publishing_fails() { + // A failed publish must not strand its staging file. The destination is a + // non-empty directory, which no platform will let a file replace, so the + // failure happens at the publish step with the scratch file already + // written — exactly the path that has to clean up after itself. + let root = atomic_write_root("publish-fails"); + let target = root.join("record.json"); + fs::create_dir_all(&target).expect("seed a directory where the file goes"); + fs::write(target.join("occupant"), b"x").expect("make it non-empty"); + + let error = write_file_atomically(&target, b"contents") + .expect_err("replacing a non-empty directory must fail"); + + assert!( + leftover_scratch_files(&root).is_empty(), + "a failed publish must remove its scratch file, got {:?} ({error:#})", + leftover_scratch_files(&root) + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn write_file_atomically_refuses_a_path_with_no_parent_or_an_unusable_parent() { + let error = write_file_atomically(Path::new("/"), b"x") + .expect_err("a root path has no directory to stage in"); + assert!(format!("{error:#}").contains("no parent"), "{error:#}"); + + // A regular file where the parent directory should be: the directory + // cannot be created, and nothing may be left behind. + let root = atomic_write_root("parent-is-a-file"); + let blocker = root.join("blocker"); + fs::write(&blocker, b"x").expect("seed the blocking file"); + let error = write_file_atomically(&blocker.join("record.json"), b"x") + .expect_err("a file cannot be a parent directory"); + assert!( + format!("{error:#}").contains("failed to create"), + "{error:#}" + ); + assert!(leftover_scratch_files(&root).is_empty()); + let _ = fs::remove_dir_all(root); + } + #[test] fn resolve_amd_smi_binary_prefers_home_rocm_venv_path() -> Result<()> { let temp_root = @@ -536,13 +1044,13 @@ mod tests { } #[test] - fn write_atomic_writes_bytes_and_leaves_no_temp_file() -> Result<()> { + fn write_file_atomically_writes_bytes_and_leaves_no_temp_file() -> Result<()> { let temp_root = std::env::temp_dir().join(format!("rocm-cli-write-atomic-ok-{}", unix_time_millis())); fs::create_dir_all(&temp_root)?; let path = temp_root.join("store.jsonl"); - write_atomic(&path, b"hello\n", "jsonl")?; + write_file_atomically(&path, b"hello\n")?; let written = fs::read(&path)?; let leftovers: Vec<_> = fs::read_dir(&temp_root)? @@ -561,17 +1069,17 @@ mod tests { } #[test] - fn write_atomic_rename_failure_leaves_destination_and_dir_untouched() -> Result<()> { + fn write_file_atomically_rename_failure_leaves_destination_and_dir_untouched() -> Result<()> { let temp_root = std::env::temp_dir().join(format!("rocm-cli-write-atomic-fail-{}", unix_time_millis())); fs::create_dir_all(&temp_root)?; let path = temp_root.join("store.jsonl"); // A file can't be renamed over an existing directory, so making the - // destination a directory forces `write_atomic`'s rename step to fail + // destination a directory forces `write_file_atomically`'s rename step to fail // after the temp file has already been written. fs::create_dir_all(&path)?; - let result = write_atomic(&path, b"hello\n", "jsonl"); + let result = write_file_atomically(&path, b"hello\n"); let destination_is_still_a_dir = path.is_dir(); let leftovers: Vec<_> = fs::read_dir(&temp_root)? .filter_map(Result::ok) diff --git a/crates/rocm-core/src/net.rs b/crates/rocm-core/src/net.rs index b60a2a7d7c..f23dd25ac6 100644 --- a/crates/rocm-core/src/net.rs +++ b/crates/rocm-core/src/net.rs @@ -521,6 +521,27 @@ pub fn http_get_text_with_auth( endpoint_api_key: Option<&str>, timeout: Duration, ) -> Result { + let parts = http_get_with_auth(endpoint_url, path, endpoint_api_key, timeout)?; + if parts.status != 200 { + bail!("HTTP endpoint returned HTTP {}", parts.status); + } + Ok(parts.body) +} + +/// GET `path` and return the response status alongside the body. +/// +/// The GET sibling of [`http_post_json_with_auth`], and it exists for the same +/// reason: a caller probing an endpoint has to tell "the server answered, with a +/// refusal" apart from "the server never answered", and only the former proves +/// something is alive on that port. [`http_get_text_with_auth`] collapses both +/// into `Err` — correct for callers that only want a 200's body, wrong for +/// anything deciding whether a port is still held. +pub fn http_get_with_auth( + endpoint_url: &str, + path: &str, + endpoint_api_key: Option<&str>, + timeout: Duration, +) -> Result { let deadline = Instant::now() + timeout; let (host, port) = parse_http_endpoint(endpoint_url) .with_context(|| format!("unsupported endpoint URL `{endpoint_url}`"))?; @@ -541,10 +562,12 @@ pub fn http_get_text_with_auth( .split_once("\r\n\r\n") .context("HTTP response was missing a body")?; let status_line = headers.lines().next().unwrap_or_default(); - if !status_line.contains(" 200 ") { - bail!("HTTP endpoint returned {status_line}"); - } - Ok(body.to_owned()) + let status = http_status_code(status_line) + .with_context(|| format!("unparsable HTTP status line `{status_line}`"))?; + Ok(HttpResponseParts { + status, + body: body.to_owned(), + }) } /// POST a JSON body and return the response status line plus body. @@ -636,13 +659,11 @@ pub fn merge_json_state_file(path: &Path, patch: &serde_json::Value) -> Result<( object.insert(key.clone(), patch_value.clone()); } } - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .with_context(|| format!("failed to create {}", parent.display()))?; - } - fs::write( + // Atomic, so `rocmd` and `rocm uninstall` reading this file while a launch + // patches it never see a torn document. It creates the parent directory itself. + crate::write_file_atomically( path, - serde_json::to_vec_pretty(&value).context("failed to serialize service state")?, + &serde_json::to_vec_pretty(&value).context("failed to serialize service state")?, ) .with_context(|| format!("failed to write {}", path.display())) } @@ -777,19 +798,59 @@ pub fn openai_models_endpoint_has_model( endpoint_api_key: Option<&str>, timeout: Duration, ) -> Result { + Ok( + openai_models_endpoint_identity(endpoint_url, expected_model, endpoint_api_key, timeout)? + == EndpointIdentity::ServesExpectedModel, + ) +} + +/// What a listener said when asked which models it serves. +/// +/// The distinction [`openai_models_endpoint_has_model`] cannot draw: it answers +/// `false` both for "this is someone else's service" and for "this endpoint +/// answered, but listed nothing". For a readiness poll those are the same — not +/// ready either way — but for a caller deciding whether something still holds a +/// port they are opposites. An engine that is up but has not populated its model +/// list yet, or is mid-unload, lists nothing while very much holding the GPU. +#[derive(Debug, Clone, Copy, Eq, PartialEq)] +pub enum EndpointIdentity { + /// Listed a model matching the one asked about (or listed models when no + /// particular one was asked for). + ServesExpectedModel, + /// Listed models, none of which match — a different service on this port. + ServesOtherModels, + /// Answered `/v1/models` with an empty list. Says nothing either way about + /// whose service this is. + ListsNoModels, +} + +/// Ask a listener which models it serves, keeping "listed nothing" distinct. +pub fn openai_models_endpoint_identity( + endpoint_url: &str, + expected_model: Option<&str>, + endpoint_api_key: Option<&str>, + timeout: Duration, +) -> Result { let body = http_get_text_with_auth(endpoint_url, "/v1/models", endpoint_api_key, timeout)?; let value = serde_json::from_str::(body.trim()) .context("failed to parse /v1/models JSON")?; let loaded_models = openai_loaded_model_ids(&value); if loaded_models.is_empty() { - return Ok(false); + return Ok(EndpointIdentity::ListsNoModels); } let Some(expected_model) = expected_model.filter(|value| !value.trim().is_empty()) else { - return Ok(true); + return Ok(EndpointIdentity::ServesExpectedModel); }; - Ok(loaded_models - .iter() - .any(|loaded| model_refs_match(loaded, expected_model))) + Ok( + if loaded_models + .iter() + .any(|loaded| model_refs_match(loaded, expected_model)) + { + EndpointIdentity::ServesExpectedModel + } else { + EndpointIdentity::ServesOtherModels + }, + ) } pub fn managed_service_endpoint_model_ready( @@ -804,6 +865,26 @@ pub fn managed_service_endpoint_model_ready( openai_models_endpoint_has_model(&record.endpoint_url, expected, endpoint_api_key, timeout) } +/// [`managed_service_endpoint_model_ready`] without collapsing "listed nothing" +/// into "not ours". +/// +/// An empty `endpoint_url` is the one case with no listener to ask about at all, +/// so it stays an error rather than being reported as an identity. +pub fn managed_service_endpoint_identity( + record: &ManagedServiceRecord, + endpoint_api_key: Option<&str>, + timeout: Duration, +) -> Result { + if record.endpoint_url.trim().is_empty() { + bail!("service record has no endpoint URL to identify"); + } + // The name the endpoint lists, which is not always the canonical id: an AIM + // service's canonical id is its Docker image reference while the container + // serves the image's own model name. + let expected = Some(record.served_model_name()).filter(|name| !name.trim().is_empty()); + openai_models_endpoint_identity(&record.endpoint_url, expected, endpoint_api_key, timeout) +} + /// How far along a managed service's endpoint is. /// /// The middle state is the one that matters: an engine lists a model within @@ -1199,6 +1280,145 @@ mod tests { server.join().expect("server thread should not panic")?; Ok(()) } + #[test] + fn an_empty_model_list_is_not_reported_as_somebody_elses_service() -> Result<()> { + // `openai_models_endpoint_has_model` answers `false` both for "these are + // someone else's models" and for "this endpoint listed nothing". For a + // readiness poll that is fine — not ready either way. For a caller + // deciding whether something still holds a port they are opposites: an + // engine that is up but has not populated its list yet, or is + // mid-unload, lists nothing while holding the port and the GPU, and + // reading that as "not ours" removes the tooling out from under it. + // + // Four connections: an empty list and a stranger's list, asked twice — + // once through the enum, once through the bool wrapper. The wrapper is + // where the three-way answer collapses back to the readiness contract + // every existing caller depends on, and that single comparison is not + // pinned by asserting on the enum alone. + let listener = std::net::TcpListener::bind("127.0.0.1:0")?; + let port = listener.local_addr()?.port(); + let server = std::thread::spawn(move || -> Result<()> { + let empty = r#"{"data":[]}"#; + let stranger = r#"{"data":[{"id":"someone/else"}]}"#; + for body in [empty, stranger, empty, stranger] { + let (mut stream, _) = listener.accept()?; + stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); + let mut buffer = [0_u8; 512]; + let _ = stream.read(&mut buffer)?; + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + )?; + } + Ok(()) + }); + let endpoint = format!("http://127.0.0.1:{port}/v1"); + + assert_eq!( + openai_models_endpoint_identity( + &endpoint, + Some("amd/ours"), + None, + Duration::from_secs(2) + )?, + EndpointIdentity::ListsNoModels, + "an empty list says nothing about whose service this is" + ); + assert_eq!( + openai_models_endpoint_identity( + &endpoint, + Some("amd/ours"), + None, + Duration::from_secs(2) + )?, + EndpointIdentity::ServesOtherModels, + "a list that names other models is the only real evidence of a stranger" + ); + + // Across the wrapper. Only `ServesExpectedModel` may read as ready: + // mapping either of the other two arms to `true` would tell every + // readiness caller that an endpoint listing nothing, or listing a + // stranger's model, is serving theirs. + assert!( + !openai_models_endpoint_has_model( + &endpoint, + Some("amd/ours"), + None, + Duration::from_secs(2) + )?, + "an endpoint listing nothing is not serving our model" + ); + assert!( + !openai_models_endpoint_has_model( + &endpoint, + Some("amd/ours"), + None, + Duration::from_secs(2) + )?, + "an endpoint serving a stranger's model is not serving ours" + ); + + server.join().expect("server thread should not panic")?; + Ok(()) + } + + #[test] + fn a_get_reports_a_refusal_as_an_answer_not_as_a_failure() -> Result<()> { + // The distinction `http_get_text_with_auth` cannot make. Its `Err` means + // "no 200", which lumps a 401 from a live server together with a dead + // port — and a caller deciding whether something still holds a port has + // to tell those apart: only the refusal proves the port is held. + // + // Two connections: the same 401 read both ways. + let listener = std::net::TcpListener::bind("127.0.0.1:0")?; + let port = listener.local_addr()?.port(); + let server = std::thread::spawn(move || -> Result<()> { + for _ in 0..2 { + let (mut stream, _) = listener.accept()?; + stream.set_read_timeout(Some(Duration::from_secs(2))).ok(); + let mut buffer = [0_u8; 512]; + let _ = stream.read(&mut buffer)?; + write!( + stream, + "HTTP/1.1 401 Unauthorized\r\nContent-Length: 0\r\nConnection: close\r\n\r\n" + )?; + } + Ok(()) + }); + let endpoint = format!("http://127.0.0.1:{port}/v1"); + + let parts = http_get_with_auth(&endpoint, "/models", None, Duration::from_secs(2))?; + assert_eq!( + parts.status, 401, + "a refusal is an answer, and its status has to survive" + ); + assert!( + http_get_text_with_auth(&endpoint, "/models", None, Duration::from_secs(2)).is_err(), + "the text helper still collapses every non-200 into an error" + ); + + server.join().expect("server thread should not panic")?; + + // And an unreachable port is an error either way — the two must not + // collapse in the other direction. + let dead = std::net::TcpListener::bind("127.0.0.1:0")?; + let dead_port = dead.local_addr()?.port(); + drop(dead); + assert!( + http_get_with_auth( + &format!("http://127.0.0.1:{dead_port}/v1"), + "/models", + None, + Duration::from_millis(500), + ) + .is_err(), + "nothing answered, so there is no status to report" + ); + Ok(()) + } + // Why readiness is gated on an inference probe (EAI-7333): a server that // lists the model on `/v1/models` but cannot yet serve // `/v1/chat/completions` still reports the model as present. This test pins diff --git a/crates/rocm-core/src/proc_lifecycle.rs b/crates/rocm-core/src/proc_lifecycle.rs index 57da4bb1fe..103e87e457 100644 --- a/crates/rocm-core/src/proc_lifecycle.rs +++ b/crates/rocm-core/src/proc_lifecycle.rs @@ -12,6 +12,22 @@ //! truthfully: a stop is only "graceful" once the recorded process is observed to //! have actually exited within a bounded grace period, escalating to `SIGKILL` //! only when the caller opts into a forced stop. +//! +//! **The recycling defence needs a readable start-time.** [`process_start_ticks`] +//! reads it from `/proc` on Linux and from the process creation time +//! (`GetProcessTimes`) on Windows, and is a compile-time `None` everywhere +//! else, so on other platforms (macOS) every record *captured there* carries no +//! identity, and [`identity_state`] degrades to best-effort +//! [`IdentityState::Matches`] — the paragraph above describes what this module +//! enforces *where the platform can answer*. (A record reconstructed with a +//! start-time recorded elsewhere is the other case: unconfirmable rather than +//! absent, so it lands on [`IdentityState::Indeterminate`] and is not signalled +//! either.) The degradation is deliberate: a +//! host that cannot tell two processes apart must not therefore refuse to stop +//! anything, since that would make every service unstoppable rather than making +//! any of them safer. What holds on those hosts is the rest of the caller's +//! gate — the port reality-check, the endpoint identity probe, and aborting +//! with the recovery tooling intact on any unconfirmed stop. use std::time::{Duration, Instant}; @@ -31,8 +47,9 @@ pub enum KillScope { /// A spawned process identified in a way that is robust to PID recycling. /// -/// `start_ticks` is the kernel start-time of the process (clock ticks since -/// boot). It is `None` on platforms without `/proc`, where identity cannot be +/// `start_ticks` is the kernel start-time of the process (Linux: clock ticks since +/// boot; Windows: the creation time). It is `None` when the platform cannot read it (or the +/// process is gone), where identity cannot be /// verified and callers fall back to best-effort behaviour. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ProcessIdentity { @@ -83,12 +100,69 @@ pub enum IdentityState { /// [`IdentityState::Indeterminate`], never a risky match. When no identity was /// recorded (legacy state files), it degrades to best-effort /// [`IdentityState::Matches`]. +/// +/// Reads the PID's current start-time itself, after its own liveness check, so +/// the reading always describes whatever holds the PID now. #[must_use] pub fn identity_state(id: &ProcessIdentity) -> IdentityState { - if !crate::process_is_running(id.pid) || process_has_exited(id.pid) { + identity_state_from_probes( + id, + || crate::process_is_running(id.pid) && !process_has_exited(id.pid), + || process_start_ticks(id.pid), + ) +} + +/// [`identity_state`] with its two observations taken lazily, so the order they +/// are taken in is a property of this body rather than of a call site. +/// +/// Liveness BEFORE the reading, and the early return is what enforces it. The +/// reading must not sit in argument position — Rust evaluates arguments before +/// entering the callee, which inverts the safe side of the race. A PID read +/// while it was still the recorded process, which then exits and is recycled +/// before the liveness check, would be compared as `expected == actual` and come +/// back [`IdentityState::Matches`]: the one verdict that authorises a kill, +/// handed out for a PID that is now somebody else. Reading only after liveness +/// means the reading always describes whatever holds the PID *now*, so a +/// recycled one disagrees and comes back [`IdentityState::Recycled`]. +/// +/// The two probes are parameters purely so that ordering is testable. Staging +/// the real race needs a process to exit and its PID to be reissued inside a +/// window of microseconds, which no test can do; two closures that record when +/// they were called pin it deterministically instead. They are `FnOnce` +/// generics, so this costs nothing at runtime — the real call above +/// monomorphizes back into the same two direct calls. +/// +/// What the tests pin is THIS function's ordering. That [`identity_state`] +/// hands it the reading as a closure, not as a value it read first, is the one +/// line `|| process_start_ticks(id.pid)` there, and no test can see it: a dead +/// PID reads `None` either way. Keep that line a closure. +fn identity_state_from_probes( + id: &ProcessIdentity, + is_live: impl FnOnce() -> bool, + read_start_ticks: impl FnOnce() -> Option, +) -> IdentityState { + if !is_live() { return IdentityState::Gone; } - match (id.start_ticks, process_start_ticks(id.pid)) { + // Compare only: re-running the liveness check just performed would cost two + // more `/proc` reads per call on Linux, paid on every 25 ms tick of the + // bounded waits, which is where this is called from in a loop. + compare_start_ticks(id, read_start_ticks()) +} + +/// The identity comparison alone, for callers that have already established +/// liveness. +/// +/// Split out so the ordering guarantee above does not have to pay for a second +/// liveness check. Deliberately not public: on its own it cannot return +/// [`IdentityState::Gone`], so a caller that had not checked liveness would get +/// [`IdentityState::Matches`] for a dead PID — the one verdict that authorises a +/// kill. The two callers that establish liveness first are in this file. +const fn compare_start_ticks( + id: &ProcessIdentity, + observed_start_ticks: Option, +) -> IdentityState { + match (id.start_ticks, observed_start_ticks) { (Some(expected), Some(actual)) => { if expected == actual { IdentityState::Matches @@ -300,8 +374,8 @@ fn send_signal(pid: u32, signal: Signal, _tree: bool) -> bool { /// Read the kernel start-time (field 22 of `/proc//stat`) for `pid`. /// -/// Returns `None` when the value cannot be read, including on non-Linux -/// platforms, where identity verification degrades to best-effort. +/// Returns `None` when the value cannot be read, including on platforms other +/// than Linux and Windows, where identity verification degrades to best-effort. #[cfg(target_os = "linux")] #[must_use] pub fn process_start_ticks(pid: u32) -> Option { @@ -309,7 +383,47 @@ pub fn process_start_ticks(pid: u32) -> Option { parse_start_ticks(&stat) } -#[cfg(not(target_os = "linux"))] +/// On Windows the start-time is the process creation `FILETIME`. +/// +/// It counts 100 ns ticks since 1601. It is opaque to the identity comparison, which only checks +/// equality, and it is what lets a recycled PID be told from the recorded one. +#[cfg(windows)] +#[must_use] +#[allow(unsafe_code)] // Win32 FFI +pub fn process_start_ticks(pid: u32) -> Option { + use windows_sys::Win32::Foundation::{CloseHandle, FILETIME}; + use windows_sys::Win32::System::Threading::{ + GetProcessTimes, OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION, + }; + + unsafe { + let handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid); + if handle.is_null() { + return None; + } + let zero = FILETIME { + dwLowDateTime: 0, + dwHighDateTime: 0, + }; + let (mut created, mut exited, mut kernel, mut user) = (zero, zero, zero, zero); + // SAFETY: `handle` is a live process handle opened above and closed below; + // the four out-pointers reference initialized FILETIMEs on this stack. + let ok = GetProcessTimes( + handle, + &raw mut created, + &raw mut exited, + &raw mut kernel, + &raw mut user, + ); + CloseHandle(handle); + if ok == 0 { + return None; + } + Some((u64::from(created.dwHighDateTime) << 32) | u64::from(created.dwLowDateTime)) + } +} + +#[cfg(not(any(target_os = "linux", windows)))] #[must_use] // Platform stub. The `cfg` sibling calls non-const code, so making only this // arm `const` would give the two platforms different signatures and push @@ -366,9 +480,111 @@ fn parse_start_ticks(stat: &str) -> Option { #[cfg(test)] mod tests { use super::*; + + #[cfg(windows)] + #[test] + fn windows_start_time_is_readable_and_stable_for_our_own_pid() { + let pid = std::process::id(); + let first = process_start_ticks(pid).expect("our own creation time is readable"); + std::thread::sleep(std::time::Duration::from_millis(50)); + assert_eq!(process_start_ticks(pid), Some(first)); + assert_ne!(first, 0); + } + + #[cfg(windows)] + #[test] + fn windows_start_time_is_none_for_an_unopenable_pid_and_differs_per_process() { + assert_eq!(process_start_ticks(u32::MAX - 1), None); + let mut child = std::process::Command::new("ping") + .args(["-n", "30", "127.0.0.1"]) + .stdout(std::process::Stdio::null()) + .spawn() + .expect("spawn a child"); + std::thread::sleep(std::time::Duration::from_millis(50)); + let ours = process_start_ticks(std::process::id()); + let theirs = process_start_ticks(child.id()); + let _ = child.kill(); + let _ = child.wait(); + assert!(theirs.is_some()); + assert_ne!( + ours, theirs, + "a different process has a different creation time" + ); + } + #[cfg(unix)] use std::process::{Child, Command, Stdio}; + #[test] + fn a_dead_process_is_never_read_for_a_start_time() { + // The ordering inside `identity_state_from_probes`, pinned rather than + // inspected: the reading is taken only after the liveness check passes. + // If it were ever read first, a PID that exits and is recycled in between + // comes back `Matches`: the one verdict that authorises a kill, for a + // process that is no longer the recorded one. This covers the helper + // only; `identity_state` passing the reading lazily is not observable + // from a test (a dead PID reads `None` either way). + // + // The real race is microseconds wide and needs a PID-space wrap, so it + // cannot be staged. This stages the observable consequence instead: a + // reading taken for a process already known dead is a reading that + // could not have informed the verdict. + let id = ProcessIdentity { + pid: 4321, + start_ticks: Some(99), + }; + let mut read_start_ticks = false; + + let state = identity_state_from_probes( + &id, + || false, + || { + read_start_ticks = true; + Some(99) + }, + ); + + assert_eq!( + state, + IdentityState::Gone, + "a process that fails the liveness check is Gone whatever its start time reads as" + ); + assert!( + !read_start_ticks, + "`identity_state_from_probes` must not read the start time once liveness has \ + failed; this pins that helper's ordering only, not how `identity_state` calls it" + ); + } + + #[test] + fn a_live_process_is_judged_on_a_reading_taken_after_its_liveness_check() { + // The other half: when liveness passes, the reading is taken, and it is + // the reading — not the record — that decides. A recycled PID reads + // differently and must come back `Recycled`. + // + // The PID is arbitrary, and nothing here establishes whether it is + // running — it does not need to. Both probes are stubbed, so this path + // never consults the real process table at all, and that is the point: + // reaching the comparison at all proves the verdict came from the + // liveness this function was handed rather than from a second check of + // its own. + let id = ProcessIdentity { + pid: 4321, + start_ticks: Some(99), + }; + + assert_eq!( + identity_state_from_probes(&id, || true, || Some(99)), + IdentityState::Matches, + "a live PID still reading as its recorded start time is the recorded process" + ); + assert_eq!( + identity_state_from_probes(&id, || true, || Some(1234)), + IdentityState::Recycled, + "a live PID reading as a different start time is somebody else" + ); + } + /// Spawn a child that prints a line to stdout once it is ready, and block /// until that line arrives. This replaces sleep-based readiness guesses with /// a deterministic signal (e.g. that a shell has installed its SIGTERM trap), diff --git a/crates/rocm-core/src/service_record.rs b/crates/rocm-core/src/service_record.rs index 37bb07d945..0f2a80381d 100644 --- a/crates/rocm-core/src/service_record.rs +++ b/crates/rocm-core/src/service_record.rs @@ -23,6 +23,7 @@ use crate::AppPaths; use crate::INFERENCE_VERIFIED_STATE_KEY; use crate::format_http_base_url; use crate::unix_time_millis; +use crate::write_file_atomically; use crate::{normalize_runtime_path_for_host, normalize_runtime_path_for_storage}; #[derive(Debug, Clone, Serialize, serde::Deserialize)] @@ -371,24 +372,25 @@ impl ManagedServiceRecord { record } + /// Persist the record, atomically. + /// + /// Staged next to the manifest and published over it, so a concurrent reader + /// sees either the old record or the new one, never a half-written file. A + /// plain overwrite left a window in which a reader (`load_managed_services`, + /// or the uninstall gate that treats an unparseable manifest as a live server + /// it cannot account for) could observe a truncated record and act on it. + /// + /// Uses the workspace's one [`write_file_atomically`] rather than a local + /// rename: this runs on every status transition, including the ones the + /// uninstall stop pass performs, where a failed publish aborts the uninstall. pub fn write(&self) -> Result<()> { let mut host_record = self.clone(); host_record.normalize_paths_for_host(); - let parent = host_record - .manifest_path - .parent() - .context("service manifest path must have a parent directory")?; - fs::create_dir_all(parent) - .with_context(|| format!("failed to create {}", parent.display()))?; let storage_record = host_record.with_storage_paths(); let bytes = serde_json::to_vec_pretty(&storage_record) .context("failed to serialize service record")?; - crate::write_atomic(&host_record.manifest_path, &bytes, "json").with_context(|| { - format!( - "failed to write service manifest {}", - host_record.manifest_path.display() - ) - }) + write_file_atomically(&host_record.manifest_path, &bytes) + .with_context(|| format!("failed to write {}", host_record.manifest_path.display())) } } diff --git a/crates/rocm-engine-protocol/src/lib.rs b/crates/rocm-engine-protocol/src/lib.rs index 6ca773ccd6..cefb13c9b2 100644 --- a/crates/rocm-engine-protocol/src/lib.rs +++ b/crates/rocm-engine-protocol/src/lib.rs @@ -48,6 +48,84 @@ pub fn is_public_bind_host(host: &str) -> bool { !matches!(host, "127.0.0.1" | "localhost" | "::1") } +/// The `rocm serve` invocation that brings back a service whose endpoint key +/// was dropped, for the refusals and the uninstall abort that tell an operator +/// to launch it again. +/// +/// One builder for every place that prints it (the `rocm serve` and `rocmd` +/// refusals and the uninstall abort, one command per stopped service), so they +/// cannot drift: a public host needs `--allow-public-bind`, and a service that +/// asked for authentication needs `--require-api-key` even on loopback. The key +/// itself is optional ([`RELAUNCH_KEY_CHOICE`]). +#[must_use] +pub fn relaunch_command(host: &str, require_api_key: bool) -> String { + let public_flag = if is_public_bind_host(host) { + " --allow-public-bind" + } else { + "" + }; + let key_flag = if require_api_key { + " --require-api-key" + } else { + "" + }; + // The flags are appended, not interpolated after ``: the advised-command + // parse test reads this literal as source, and a placeholder glued to + // `{public_flag}` would substitute the port value into it. + let mut command = format!("rocm serve --host {host} --port "); + command.push_str(public_flag); + command.push_str(key_flag); + command +} + +/// [`relaunch_command`] filled in for one concrete service. +/// +/// The template stays the single source of the flag set; this substitutes its +/// model and port placeholders and appends the engine, so the advice runs as printed. +#[must_use] +pub fn relaunch_command_for( + model: &str, + engine: &str, + host: &str, + port: u16, + require_api_key: bool, +) -> String { + let mut command = relaunch_command(host, require_api_key) + .replacen("", &shell_word(model), 1) + .replacen("", &port.to_string(), 1); + if !engine.is_empty() { + command.push_str(" --engine "); + command.push_str(&shell_word(engine)); + } + command +} + +/// The running platform's shell: PowerShell on Windows, POSIX `sh` elsewhere. +fn shell_word(value: &str) -> String { + shell_word_for(value, cfg!(windows)) +} + +/// Leave a plain word as is and single-quote anything the shell would split or +/// expand. An apostrophe inside is doubled for PowerShell and closed-escaped- +/// reopened (`'\''`) for POSIX `sh`; each is wrong in the other shell. +fn shell_word_for(value: &str, powershell: bool) -> String { + let plain = !value.is_empty() + && value + .chars() + .all(|c| c.is_ascii_alphanumeric() || "._/:@=+,-".contains(c)); + if plain { + value.to_owned() + } else if powershell { + format!("'{}'", value.replace('\'', "''")) + } else { + format!("'{}'", value.replace('\'', "'\\''")) + } +} + +/// The parenthetical that follows a [`relaunch_command`]: supplying a key is +/// optional, and the environment variable keeps it out of shell history. +pub const RELAUNCH_KEY_CHOICE: &str = "(add `--api-key `, or set ROCM_SERVE_API_KEY, to choose the key instead of generating one)"; + /// Resolve the endpoint API key an engine adapter must enforce, if any. /// /// Reads the key file named by [`ENDPOINT_API_KEY_FILE_ENV`]. Returns `None` when @@ -822,6 +900,57 @@ mod tests { std::fs::remove_dir_all(&paths.data_dir).ok(); } + #[test] + fn relaunch_command_carries_exactly_the_flags_the_service_needs() { + assert_eq!( + relaunch_command("127.0.0.1", false), + "rocm serve --host 127.0.0.1 --port " + ); + assert_eq!( + relaunch_command("127.0.0.1", true), + "rocm serve --host 127.0.0.1 --port --require-api-key" + ); + assert_eq!( + relaunch_command("0.0.0.0", false), + "rocm serve --host 0.0.0.0 --port --allow-public-bind" + ); + assert_eq!( + relaunch_command("0.0.0.0", true), + "rocm serve --host 0.0.0.0 --port --allow-public-bind --require-api-key" + ); + } + + #[test] + fn relaunch_command_for_fills_in_the_service_it_was_given() { + assert_eq!( + relaunch_command_for("amd/Model-7B", "vllm", "0.0.0.0", 8123, true), + "rocm serve amd/Model-7B --host 0.0.0.0 --port 8123 --allow-public-bind --require-api-key --engine vllm" + ); + assert_eq!( + relaunch_command_for("/models/my model.gguf", "", "127.0.0.1", 8000, false), + "rocm serve '/models/my model.gguf' --host 127.0.0.1 --port 8000" + ); + assert!(!relaunch_command_for("m", "vllm", "127.0.0.1", 1, false).contains('<')); + } + + #[test] + fn an_apostrophe_in_a_model_path_is_escaped_for_the_shell_that_will_read_it() { + let model = "/models/O'Brien's model.gguf"; + assert_eq!( + shell_word_for(model, false), + "'/models/O'\\''Brien'\\''s model.gguf'" + ); + assert_eq!( + shell_word_for(model, true), + "'/models/O''Brien''s model.gguf'" + ); + let command = relaunch_command_for(model, "vllm", "127.0.0.1", 8000, false); + assert!( + command.contains(&shell_word_for(model, cfg!(windows))), + "{command}" + ); + } + #[test] fn is_public_bind_host_recognizes_only_the_accepted_loopback_spellings() { for host in ["127.0.0.1", "localhost", "::1"] { diff --git a/docs/architecture.md b/docs/architecture.md index cf78d1cee9..413665670d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -57,7 +57,8 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`service_launch.rs`](../apps/rocm/src/service_launch.rs) — mechanical relocation, owns private types (Phase 6b, ROCMAI-91): spawning/attaching to a managed service process and streaming its log, plus the endpoint-key/auth checks that gate a public bind — `AttachOutcome`/`RawModeGuard`/`ManagedSpawn`/`AutostartClaim` are its private types; `ManagedLaunchReport` is `pub(crate)`, since [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) reads its fields directly for the interactive summary. Most of this module's functions are `pub(crate)` for the same reason — [`main.rs`](../apps/rocm/src/main.rs), [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), and [`automations.rs`](../apps/rocm/src/automations.rs) all call into it. `ServicesCommand`/`services()` stayed in [`main.rs`](../apps/rocm/src/main.rs) rather than moving alongside — the service-record-management helpers `services()` dispatches to (stop/restart/remove/prune) are entangled with each other; only stop/restart are also reachable from the chat/LLM sandbox tool-calling path, out of scope for Phase 6b. - [`storage.rs`](../apps/rocm/src/storage.rs) — full domain extraction. - [`therock.rs`](../apps/rocm/src/therock.rs) — full domain extraction; owns TheRock SDK install/update and the runtime-manifest model/loaders. The `runtimes` subcommand's own command enum, dispatch fn, and most of its domain logic — rendering, activation, and manifest import — stay in [`main.rs`](../apps/rocm/src/main.rs). -- [`uninstall.rs`](../apps/rocm/src/uninstall.rs) — mechanical relocation, owns private types: `uninstall()`, plus the installer `PATH` undo and running-executable deletion. The shared plan type and its file-removal helpers live in [`uninstall_plan.rs`](../apps/rocm/src/uninstall_plan.rs) below instead, because [`storage.rs`](../apps/rocm/src/storage.rs) reuses them too. +- [`uninstall.rs`](../apps/rocm/src/uninstall.rs) — mechanical relocation, owns private types: `uninstall()`, the installer `PATH` undo, running-executable deletion, and the stop-before-remove ordering with its partial-removal report (the stop passes are in [`uninstall_gate.rs`](../apps/rocm/src/uninstall_gate.rs), whose pre-flight is shared by the plan and the stop pass, so the plan's helper and refusal lines come from the same decision as the run; the abort's relaunch advice comes from `rocm_engine_protocol::relaunch_command`, the builder `rocm serve` and `rocmd` refusals use). The shared plan type and its file-removal helpers live in [`uninstall_plan.rs`](../apps/rocm/src/uninstall_plan.rs) below instead, because [`storage.rs`](../apps/rocm/src/storage.rs) reuses them too. +- [`uninstall_gate.rs`](../apps/rocm/src/uninstall_gate.rs) — the stop-before-remove gate behind `rocm uninstall`: the shared pre-flight, the stop passes and their failure/report types. The plan's rendering of that pre-flight lives in `build_uninstall_plan` in [`uninstall_plan.rs`](../apps/rocm/src/uninstall_plan.rs). - [`uninstall_plan.rs`](../apps/rocm/src/uninstall_plan.rs) — mechanical relocation, shares crate-visible types (Phase 6f, ROCMAI-212): the uninstall-plan types (`UninstallOptions`, `UninstallPlan`, `UninstallPlanEntry`), building the plan (`build_uninstall_plan`, `plan_installed_binaries`, and the installed-binary/shared-cache discovery helpers behind them), rendering it (`render_uninstall_dry_run`, `render_uninstall_plan`), the `[y/N]` confirmation prompt (`confirm_uninstall`), and the filesystem removal primitive both [`uninstall.rs`](../apps/rocm/src/uninstall.rs) and [`storage.rs`](../apps/rocm/src/storage.rs) call to carry a plan out (`remove_path`, and the entry-resolution helpers behind it). Kept separate from `uninstall.rs` rather than folded into it, since `uninstall.rs`'s own doc comment and name both describe `uninstall()`'s exclusive logic (the installer `PATH` undo and running-executable deletion), while this module's plan/render/removal helpers are shared with `storage.rs`. `uninstall.rs`, `storage.rs`, `runtimes_cmd.rs`, `chat.rs`, and `deletion_properties.rs` each import the subset of these `pub(crate)` types and helpers they need; see this module's own doc comment for which symbol each one uses. ### `apps/rocmd` — background daemon @@ -112,7 +113,7 @@ What else stays in [`lib.rs`](../crates/rocm-core/src/lib.rs), beside module dec - `TORCH_ALIGNMENT_DISABLED_ENV`/`torch_alignment_disabled` — the one read of the torch-alignment opt-out, shared by the CLI and the vLLM engine so the two cannot disagree. - `VLLM_GPU_MEMORY_UTILIZATION_HINT` — a single wording contract between the CLI, the vLLM engine, and this crate's own `fix-16-vllm-oom` diagnosis; `rocm-core` is the only crate all of them can reach. - **amd-smi binary resolution** (`resolve_amd_smi_binary*`) — belongs in its own `amd_smi.rs` module: locating the tool is a distinct concern that only reads [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs)'s registry records. The move is deferred to ROCMAI-723 and sequenced after the open [#447](https://github.com/ROCm/rocm-cli/pull/447), which adds amd-smi invocation code to the same stretch of `lib.rs`; moving the family first would make that PR relocate its additions mid-review. Until then `VLLM_GPU_MEMORY_UTILIZATION_HINT` stays physically inside that stretch; the extraction leaves it behind at the root. -- **`write_atomic`** — not permanent. It is the temp-then-rename writer [`config.rs`](../crates/rocm-core/src/config.rs), [`automation.rs`](../crates/rocm-core/src/automation.rs), and [`service_record.rs`](../crates/rocm-core/src/service_record.rs) use for their atomic replaces. The open [#610](https://github.com/ROCm/rocm-cli/pull/610) adds a shared, stronger atomic writer to this crate for `rocm` and `rocmd`, and the open [#541](https://github.com/ROCm/rocm-cli/pull/541) moves `RocmCliConfig::save` onto it. It is `pub`, visible to other crates, because `rocm-engine-lemonade` also writes Lemonade's `backend_versions.json` through it, in [`install.rs`](../engines/lemonade/src/install.rs). Once #610 lands, the automation and service-record callers follow; `write_atomic` is deleted only after that Lemonade caller has moved to the shared writer too (ROCMAI-724). That is a behavior change, not a pure move, so it is not done here. +- **`write_file_atomically`** — the one temp-then-rename writer for the workspace, in [`lib.rs`](../crates/rocm-core/src/lib.rs): [`config.rs`](../crates/rocm-core/src/config.rs), [`automation.rs`](../crates/rocm-core/src/automation.rs), [`service_record.rs`](../crates/rocm-core/src/service_record.rs), the engine state files, and Lemonade's `backend_versions.json` in [`install.rs`](../engines/lemonade/src/install.rs) all publish through it, and `publish_temp_file` is its Windows-aware publish step. It takes no lock: a read-modify-write caller holds its own, as the JSONL stores do. Once those two deferred moves land, [`lib.rs`](../crates/rocm-core/src/lib.rs) holds only module declarations, re-exports, and the crate-root primitives above — the end state of the modularization effort for this crate (EAI-7768, planned last: highest fan-in, since every app and engine crate depends on it, but lowest novelty, since the sibling modules already prove the pattern). diff --git a/docs/manual-testing.md b/docs/manual-testing.md index efd1b21eb7..fb187e2258 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -842,6 +842,38 @@ Expected result: - With `rocm uninstall --keep-binaries`, the review says the `PATH` change stays too, and it does. +### With a running managed server + +Start a managed server (`rocm serve `), then run: + +```powershell +rocm uninstall --yes +``` + +Expected result: + +- The plan warns the server will be stopped before removal. If the background + helper (`rocmd`) is running and the run proceeds, the plan says it will be stopped before removal (its whole process tree on Linux; only the recorded process on Windows, and the plan says so). On a run that will refuse, the plan says the helper is left running instead. +- The server (and helper) are stopped, then the files are removed and the + command exits 0. + +The refusal paths each need a fresh install and a fresh managed server, because +a successful `uninstall --yes` above removes the install. Reinstall, start a +managed server again, then: + +- **Unreadable record.** Write `{` into `/services/bad.json`, where + the data directory is the one `rocm examine` reports, and run + `rocm uninstall --yes`. It must exit non-zero, name that file's full path, + advise repairing or deleting it, and leave the install, config, and data + directories in place. +- **No recorded start time.** Edit the server's record under the same + `services` folder and delete its `supervisor_start_ticks` field (a record + written by an older version looks like this). The plan must say the stop + will be refused, and `rocm uninstall --yes` must exit non-zero naming the + pid, leave the server running, and remove nothing. +- **Keep binaries and data (removes only config and cache).** `rocm uninstall --yes --keep-binaries --keep-data` must leave + the server and the helper running and say so in the plan. + ## 9. AIM (Docker) Engine Verification Linux/WSL2 only, and requires a reachable Docker daemon (`docker info` must diff --git a/docs/testing.md b/docs/testing.md index 1ff01a513c..08fe4ebda4 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1654,7 +1654,13 @@ generated public-key PEM installer verification; verify first-install PATH setup Linux writes the shell profile); reinstall stale-manifest purge and config preservation; a Windows loopback-HTTP install; isolated installed-binary directory smoke checks (`rocm examine` must read only the isolated -config/data/cache, never the real user `.rocm` state); and uninstall: a full +config/data/cache, never the real user `.rocm` state); and uninstall: stopping the local server it manages +(lifecycle-10 and lifecycle-24, Linux and Windows), the background helper +(lifecycle-26), an uninstall that keeps the binaries and data leaving the +server running and its record in place (lifecycle-27), refusing to remove +anything while a managed service record cannot be read (lifecycle-25), while a +live server it cannot prove is ours has no recorded start time (lifecycle-28), +and while the helper's runtime state cannot be read (lifecycle-29); and a full purge that also restores the shell profile, the Windows user `PATH` taken back off with its other entries and its registry type kept, the `PATH` change kept while another program shares the install folder or with `--keep-binaries`, a `PATH` change that cannot be undone removing nothing diff --git a/engines/lemonade/src/backend_alignment.rs b/engines/lemonade/src/backend_alignment.rs index edb30aeb9a..c1c1dfe263 100644 --- a/engines/lemonade/src/backend_alignment.rs +++ b/engines/lemonade/src/backend_alignment.rs @@ -2036,8 +2036,8 @@ mod tests { #[test] fn a_failed_put_back_says_so_and_leaves_the_restored_pins_in_place() { - // The backup points at a directory that does not exist, so writing the - // previous bytes back fails. + // The backup's parent is a regular file, so writing the previous bytes + // back fails whether or not the writer creates missing directories. let flow = restore_flow( vec![ Err(anyhow!("connection reset")), @@ -2046,8 +2046,9 @@ mod tests { |pin_file, dir| { let previous = fs::read(pin_file).unwrap(); fs::write(pin_file, "PACKAGED").unwrap(); + fs::write(dir.join("not-a-dir"), "file").unwrap(); Ok(Some(PinFileBackup::new( - dir.join("no-such-dir").join("backend_versions.json"), + dir.join("not-a-dir").join("backend_versions.json"), Some(previous), ))) }, diff --git a/engines/lemonade/src/install.rs b/engines/lemonade/src/install.rs index ab50ff705b..b69ef74167 100644 --- a/engines/lemonade/src/install.rs +++ b/engines/lemonade/src/install.rs @@ -362,7 +362,7 @@ fn restore_packaged_backend_versions_with( // Missing (or not JSON): there is nothing of the user's to keep. None => packaged, }; - rocm_core::write_atomic(&target, &restored, "pins") + rocm_core::write_file_atomically(&target, &restored) .with_context(|| format!("failed to restore {}", target.display()))?; Ok(Some(PinFileBackup { target, @@ -414,7 +414,7 @@ fn set_or_remove_pin( /// What `resources/backend_versions.json` held before /// [`restore_packaged_backend_versions`] rewrote it. Written back through -/// `rocm_core::write_atomic`, like the restore itself. +/// `rocm_core::write_file_atomically`, like the restore itself. /// /// A restore is attempted after any failed install, including a transient /// network or disk error, so a restore whose retry fails too must not leave a @@ -437,7 +437,7 @@ impl PinFileBackup { /// Writes the previous bytes back, or removes the file if there were none. pub(crate) fn put_back(self) -> Result<()> { match self.previous { - Some(bytes) => rocm_core::write_atomic(&self.target, &bytes, "pins") + Some(bytes) => rocm_core::write_file_atomically(&self.target, &bytes) .with_context(|| format!("failed to put back {}", self.target.display())), None => match fs::remove_file(&self.target) { Err(err) if err.kind() != std::io::ErrorKind::NotFound => { diff --git a/engines/lemonade/src/state.rs b/engines/lemonade/src/state.rs index 6df1ded86d..e443f924d1 100644 --- a/engines/lemonade/src/state.rs +++ b/engines/lemonade/src/state.rs @@ -203,10 +203,10 @@ pub(crate) fn read_service_state(path: &Path) -> Result { } fn write_state(path: &Path, value: &Value) -> Result<()> { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent)?; - } - fs::write(path, serde_json::to_vec_pretty(value)?) + // `rocmd` and `rocm uninstall` read this file while a launch rewrites it; + // the atomic publish means they never see a torn document. It creates the + // parent directory itself. + rocm_core::write_file_atomically(path, &serde_json::to_vec_pretty(value)?) .with_context(|| format!("failed to write {}", path.display())) } diff --git a/engines/vllm/src/state.rs b/engines/vllm/src/state.rs index bca142849a..e7d8648f02 100644 --- a/engines/vllm/src/state.rs +++ b/engines/vllm/src/state.rs @@ -194,13 +194,12 @@ pub(crate) fn read_service_state(path: &Path) -> Result { /// [`inference_verified`]), which patches the file directly rather than /// routing through here. fn write_state(path: &Path, value: &Value) -> Result<()> { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .with_context(|| format!("failed to create {}", parent.display()))?; - } - fs::write( + // `rocmd` and `rocm uninstall` read this file while a launch rewrites it; + // the atomic publish means they never see a torn document. It creates the + // parent directory itself. + rocm_core::write_file_atomically( path, - serde_json::to_vec_pretty(value).context("failed to serialize vLLM state")?, + &serde_json::to_vec_pretty(value).context("failed to serialize vLLM state")?, ) .with_context(|| format!("failed to write {}", path.display())) } diff --git a/tests/e2e-cucumber/features/install_lifecycle.feature b/tests/e2e-cucumber/features/install_lifecycle.feature index 589db146e2..d0f27b0a60 100644 --- a/tests/e2e-cucumber/features/install_lifecycle.feature +++ b/tests/e2e-cucumber/features/install_lifecycle.feature @@ -136,10 +136,28 @@ Feature: Release install lifecycle And the shell profile is back to the user's own settings And the record of the installer's PATH change is gone + # EAI-8014: uninstall used to report success while a server it manages kept + # serving and holding the GPU, then delete the very tooling (`rocm services + # stop`, the service records) needed to stop it. + @id:lifecycle-linux-uninstall-stops-managed-server @lifecycle @requires-os:linux + Scenario: lifecycle-10 - Linux - uninstall stops the local server it manages + Given a freshly built release tree + And a generated signing keypair + And a signed bundle installed with the public key file + And the installed binary has isolated XDG directories with state + And a local server this machine manages is running + When the user uninstalls from the installed binary + Then the removal is reported as complete + And the uninstall plan says the managed servers will be stopped before removal + And the uninstall result says one managed service was stopped + And the local server this machine manages is no longer running + And the installed rocm and rocmd binaries are gone + And the isolated config, data, and cache folders are gone + # ── Windows user-PATH restoration, loopback HTTP install, isolated smoke ─ @id:lifecycle-windows-install-signed @lifecycle @requires-os:windows - Scenario: lifecycle-10 - Windows - a signed zip installs and verifies with native crypto + Scenario: lifecycle-11 - Windows - a signed zip installs and verifies with native crypto Given a freshly built release tree And a generated signing keypair When the release is packaged and signed with the private key file @@ -149,7 +167,7 @@ Feature: Release install lifecycle And the install manifest is present @id:lifecycle-windows-key-rotation-fallback @lifecycle @requires-os:windows - Scenario: lifecycle-11 - Windows - a malformed current trust root falls through to the valid next key + Scenario: lifecycle-12 - Windows - a malformed current trust root falls through to the valid next key Given a freshly built release tree And a generated signing keypair When the release is packaged and signed with the private key file @@ -159,7 +177,7 @@ Feature: Release install lifecycle And the installed rocm binary is present @id:lifecycle-windows-all-pinned-keys-malformed @lifecycle @requires-os:windows - Scenario: lifecycle-12 - Windows - all malformed pinned trust roots fail deterministically + Scenario: lifecycle-13 - Windows - all malformed pinned trust roots fail deterministically Given a freshly built release tree And a generated signing keypair When the release is packaged and signed with the private key file @@ -169,7 +187,7 @@ Feature: Release install lifecycle And no binaries are activated in the target directory @id:lifecycle-windows-updates-user-path @lifecycle @requires-os:windows @serial - Scenario: lifecycle-13 - Windows - a default install updates the user PATH and restores it afterwards + Scenario: lifecycle-14 - Windows - a default install updates the user PATH and restores it afterwards Given a freshly built release tree And a generated signing keypair And the user PATH is captured for restoration @@ -180,7 +198,7 @@ Feature: Release install lifecycle And the install directory is on the user PATH @id:lifecycle-windows-verifies-without-openssl @lifecycle @requires-os:windows - Scenario: lifecycle-14 - Windows - the installer verifies with native crypto when openssl is absent from PATH + Scenario: lifecycle-15 - Windows - the installer verifies with native crypto when openssl is absent from PATH Given a freshly built release tree And a generated signing keypair When the release is packaged and signed with the private key file @@ -190,7 +208,7 @@ Feature: Release install lifecycle And the install manifest is present @id:lifecycle-windows-rejects-bad-signature-without-openssl @lifecycle @requires-os:windows - Scenario: lifecycle-15 - Windows - a bad signature is rejected with native crypto when openssl is absent from PATH + Scenario: lifecycle-16 - Windows - a bad signature is rejected with native crypto when openssl is absent from PATH Given a freshly built release tree And a generated signing keypair When the release is packaged and signed with the private key file @@ -200,7 +218,7 @@ Feature: Release install lifecycle And no binaries are activated in the target directory @id:lifecycle-windows-reinstall-purges-stale @lifecycle @requires-os:windows - Scenario: lifecycle-16 - Windows - reinstalling purges a stale prior entry + Scenario: lifecycle-17 - Windows - reinstalling purges a stale prior entry Given a freshly built release tree And a generated signing keypair And a signed bundle installed with the public key file @@ -211,7 +229,7 @@ Feature: Release install lifecycle And the install manifest is present @id:lifecycle-windows-reject-untrusted-key @lifecycle @requires-os:windows - Scenario: lifecycle-17 - Windows - an install with no public key rejects an untrusted signer + Scenario: lifecycle-18 - Windows - an install with no public key rejects an untrusted signer Given a freshly built release tree And a generated signing keypair When the release is packaged and signed with the private key file @@ -220,7 +238,7 @@ Feature: Release install lifecycle And no binaries are activated in the target directory @id:lifecycle-windows-reject-bad-checksum @lifecycle @requires-os:windows - Scenario: lifecycle-18 - Windows - a tampered checksum is rejected before activation + Scenario: lifecycle-19 - Windows - a tampered checksum is rejected before activation Given a freshly built release tree And a generated signing keypair When the release is packaged and signed with the private key file @@ -230,7 +248,7 @@ Feature: Release install lifecycle And no binaries are activated in the target directory @id:lifecycle-windows-reject-bad-signature @lifecycle @requires-os:windows - Scenario: lifecycle-19 - Windows - a tampered signature is rejected before activation + Scenario: lifecycle-20 - Windows - a tampered signature is rejected before activation Given a freshly built release tree And a generated signing keypair When the release is packaged and signed with the private key file @@ -240,7 +258,7 @@ Feature: Release install lifecycle And no binaries are activated in the target directory @id:lifecycle-windows-reject-missing-signature @lifecycle @requires-os:windows - Scenario: lifecycle-20 - Windows - a missing signature is rejected when a signature is required + Scenario: lifecycle-21 - Windows - a missing signature is rejected when a signature is required Given a freshly built release tree And a generated signing keypair When the release is packaged and signed with the private key file @@ -250,7 +268,7 @@ Feature: Release install lifecycle And no binaries are activated in the target directory @id:lifecycle-windows-http-install @lifecycle @requires-os:windows - Scenario: lifecycle-21 - Windows - a signed bundle installs over a loopback HTTP download + Scenario: lifecycle-22 - Windows - a signed bundle installs over a loopback HTTP download Given a freshly built release tree And a generated signing keypair When the release is packaged and signed with the private key file @@ -260,7 +278,7 @@ Feature: Release install lifecycle And the install manifest is present @id:lifecycle-windows-installed-binary-smoke @lifecycle @requires-os:windows - Scenario: lifecycle-22 - Windows - the installed binary honors isolated directories and removes itself on uninstall + Scenario: lifecycle-23 - Windows - the installed binary honors isolated directories and removes itself on uninstall Given a freshly built release tree And a generated signing keypair And a signed bundle installed with the public key file @@ -278,10 +296,114 @@ Feature: Release install lifecycle And the non-running installed rocmd binary is gone And the install manifest is gone + # The Windows half of EAI-8014. Worth its own scenario rather than trusting the + # Linux one because the termination path differs: Windows terminates only the + # recorded process, where Linux signals the whole tree. + # + # What this pins is that PID-based termination, not the gate's port + # reality-check. The planted record carries port 0, which nothing can ever be + # serving on, so the port branch is satisfied trivially on every run. Staging + # the case the port check exists for — an engine grandchild outliving the + # recorded process and still serving while the record reads "stopped" — needs + # a real listener the scenario does not stand up. That branch is covered by + # the unit tests around `stopped_record_verdict` instead. + @id:lifecycle-windows-uninstall-stops-managed-server @lifecycle @requires-os:windows + Scenario: lifecycle-24 - Windows - uninstall stops the local server it manages + Given a freshly built release tree + And a generated signing keypair + And a signed bundle installed with the public key file + And the installed binary has isolated directories with state + And a local server this machine manages is running + When the user uninstalls from the installed binary + Then the removal is reported as complete + And the uninstall plan says the managed servers will be stopped before removal + And the uninstall result says one managed service was stopped + And the local server this machine manages is no longer running + And the non-running installed rocmd binary is gone + And the isolated config, data, and cache folders are gone + + # EAI-8014: when a managed service record cannot be read, uninstall cannot tell + # whether a server is still running, so it must refuse rather than delete the + # tooling that would be needed to stop it. + @id:lifecycle-linux-uninstall-refuses-unreadable-record @lifecycle @requires-os:linux + Scenario: lifecycle-25 - Linux - uninstall refuses and keeps everything when a service record cannot be read + Given a freshly built release tree + And a generated signing keypair + And a signed bundle installed with the public key file + And the installed binary has isolated XDG directories with state + And a managed service record that cannot be parsed + When the user uninstalls from the installed binary + Then the removal is refused with advice to repair or delete that record + And the installed rocm binary and manifest are still present + And the isolated XDG state is still present + + # EAI-8014: the background helper restarts managed services on its own, so + # uninstall stops it first. Pinned alone, with no service record, so it is the + # helper stop being proven and not the service stop. + @id:lifecycle-linux-uninstall-stops-background-helper @lifecycle @requires-os:linux + Scenario: lifecycle-26 - Linux - uninstall stops the background helper + Given a freshly built release tree + And a generated signing keypair + And a signed bundle installed with the public key file + And the installed binary has isolated XDG directories with state + And the background helper is running + When the user uninstalls from the installed binary + Then the removal is reported as complete + And the uninstall plan says the background helper will be stopped + And the uninstall result says the background helper was stopped + And the background helper is no longer running + And the isolated config, data, and cache folders are gone + + # EAI-8014: a run that keeps the binaries and data (and so removes only config and cache) keeps `rocm services stop` and the + # service records, so it must not force-stop a server it leaves stoppable. + @id:lifecycle-linux-uninstall-keep-binaries-data-leaves-server @lifecycle @requires-os:linux + Scenario: lifecycle-27 - Linux - uninstall keeping binaries and data leaves the managed server running and stoppable + Given a freshly built release tree + And a generated signing keypair + And a signed bundle installed with the public key file + And the installed binary has isolated XDG directories with state + And a local server this machine manages is running + When the user uninstalls keeping binaries and data + Then the removal is reported as complete + And the keep-binaries-and-data run says the server is left running + And the local server this machine manages is still running + And the managed service record is still present + And only the isolated config and cache folders are gone + And the installed rocm and rocmd binaries are still present + + # EAI-8014: a live server whose record has no start time cannot be proven ours, + # so uninstall refuses and kills nothing instead of signalling an unverified pid. + @id:lifecycle-linux-uninstall-refuses-tickless-server @lifecycle @requires-os:linux + Scenario: lifecycle-28 - Linux - uninstall refuses a live server it cannot prove is ours + Given a freshly built release tree + And a generated signing keypair + And a signed bundle installed with the public key file + And the installed binary has isolated XDG directories with state + And a local server this machine manages is running with no recorded start time + When the user uninstalls from the installed binary + Then the removal is refused because the server cannot be proven ours + And the installed rocm binary and manifest are still present + And the managed service record is still present + And the local server this machine manages is still running + + # EAI-8014: an unreadable helper runtime state means the helper cannot be + # confirmed stopped, so uninstall refuses and keeps everything. + @id:lifecycle-linux-uninstall-refuses-unreadable-helper-state @lifecycle @requires-os:linux + Scenario: lifecycle-29 - Linux - uninstall refuses when the helper runtime state cannot be read + Given a freshly built release tree + And a generated signing keypair + And a signed bundle installed with the public key file + And the installed binary has isolated XDG directories with state + And the background helper runtime state cannot be parsed + When the user uninstalls from the installed binary + Then the removal is refused because the helper state cannot be read + And the installed rocm binary and manifest are still present + And the isolated XDG state is still present + # ── Uninstall: the installer's PATH change ────────────────────────────── @id:lifecycle-linux-uninstall-keeps-shared-path @lifecycle @requires-os:linux - Scenario: lifecycle-23 - Linux - uninstall keeps the shell PATH change while another program shares the install directory + Scenario: lifecycle-30 - Linux - uninstall keeps the shell PATH change while another program shares the install directory Given a freshly built release tree And a generated signing keypair And a signed bundle installed with the shell profile updated @@ -295,7 +417,7 @@ Feature: Release install lifecycle And the record of the installer's PATH change is still in the install directory @id:lifecycle-windows-uninstall-restores-user-path @lifecycle @requires-os:windows @serial - Scenario: lifecycle-24 - Windows - uninstall takes the install directory back off the user PATH and keeps the rest of it + Scenario: lifecycle-31 - Windows - uninstall takes the install directory back off the user PATH and keeps the rest of it Given a freshly built release tree And a generated signing keypair And the user PATH is captured for restoration @@ -309,7 +431,7 @@ Feature: Release install lifecycle And the record of the installer's PATH change is gone @id:lifecycle-linux-uninstall-keep-binaries-keeps-path @lifecycle @requires-os:linux - Scenario: lifecycle-25 - Linux - uninstall that keeps the binaries keeps the shell PATH change + Scenario: lifecycle-32 - Linux - uninstall that keeps the binaries keeps the shell PATH change Given a freshly built release tree And a generated signing keypair And a signed bundle installed with the shell profile updated @@ -322,7 +444,7 @@ Feature: Release install lifecycle And the record of the installer's PATH change is still in the install directory @id:lifecycle-linux-uninstall-path-failure-keeps-binaries @lifecycle @requires-os:linux - Scenario: lifecycle-26 - Linux - uninstall that cannot undo the PATH change removes nothing, so it can be retried + Scenario: lifecycle-33 - Linux - uninstall that cannot undo the PATH change removes nothing, so it can be retried Given a freshly built release tree And a generated signing keypair And the shell profile already holds the user's own settings @@ -342,7 +464,7 @@ Feature: Release install lifecycle And the record of the installer's PATH change is gone @id:lifecycle-linux-uninstall-follows-the-installer-record @lifecycle @requires-os:linux - Scenario: lifecycle-27 - Linux - uninstall undoes the PATH change in a profile it finds only through the installer's record + Scenario: lifecycle-34 - Linux - uninstall undoes the PATH change in a profile it finds only through the installer's record Given a freshly built release tree And a generated signing keypair And the install directory is reached through a link diff --git a/tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs b/tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs index 5563f3a0e9..ff0e4ed610 100644 --- a/tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/lifecycle_steps.rs @@ -18,9 +18,10 @@ use std::fmt::Write as _; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::process::{Command, Stdio}; use cucumber::{given, then, when}; +use e2e_cucumber::mock_server::{ServiceRecordOptions, write_service_record_with}; use crate::E2eWorld; use crate::e2e::tui_driver::{TuiSession, default_timeout}; @@ -54,6 +55,14 @@ pub struct LifecycleState { smoke_config: Option, smoke_data: Option, smoke_cache: Option, + /// A stand-in for a local server this machine manages, planted with a + /// service record so uninstall must stop it. Killed on `Drop` if a scenario + /// fails before uninstall reaches it, so no scenario leaks a process. + managed_server: Option, + /// A stand-in for the background `rocmd` helper, planted via its runtime-state + /// file. Killed on `Drop` like `managed_server`. + background_helper: Option, + broken_record: Option, /// The shell profile contents seeded before install, which uninstall must /// restore exactly. profile_before_install: Option, @@ -81,6 +90,15 @@ fn shell_profile(st: &LifecycleState) -> PathBuf { impl Drop for LifecycleState { fn drop(&mut self) { + // Never leak the managed-server stand-in, even when a scenario fails + // before uninstall would have stopped it. + for mut child in [self.managed_server.take(), self.background_helper.take()] + .into_iter() + .flatten() + { + let _ = child.kill(); + let _ = child.wait(); + } // Restore the machine user PATH if a scenario mutated it, even on panic. #[cfg(windows)] if let Some(previous) = self.captured_user_path.take() { @@ -532,6 +550,9 @@ async fn given_release_tree(world: &mut E2eWorld) { smoke_config: None, smoke_data: None, smoke_cache: None, + managed_server: None, + background_helper: None, + broken_record: None, profile_before_install: None, custom_profile: None, }); @@ -662,6 +683,155 @@ fn seed_isolated_dirs(world: &mut E2eWorld) { st.smoke_cache = Some(cache); } +/// A single long-lived process to stand in for a managed server, on either OS. +/// +/// One process, not a shell that spawns one: the CLI terminates the recorded +/// PID, and a grandchild would leave the scenario asserting against the wrong +/// process. +fn spawn_long_lived_child() -> std::process::Child { + #[cfg(windows)] + let mut command = { + let mut command = Command::new("powershell"); + command.args(["-NoProfile", "-Command", "Start-Sleep -Seconds 600"]); + command + }; + #[cfg(not(windows))] + let mut command = { + let mut command = Command::new("sleep"); + command.arg("600"); + command + }; + command + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("failed to start the managed server stand-in") +} + +/// Start a local server this machine manages: a real long-lived child process +/// plus the on-disk service record `rocm serve --managed` would leave behind, +/// planted in the isolated data dir the installed binary reads. +/// +/// Black-box on purpose — a process the CLI can find only through its record, +/// exactly as a real managed vLLM server appears to `rocm uninstall`. The record +/// carries the supervisor start time, so the CLI's verified +/// termination verifies it against the recorded start time and stops it. +#[given("a local server this machine manages is running")] +async fn given_managed_server_running(world: &mut E2eWorld) { + plant_managed_server(world, true); +} + +/// A live server whose record carries no start time, as an older version wrote +/// it: uninstall cannot prove the pid is ours, so it must refuse. +#[given("a local server this machine manages is running with no recorded start time")] +async fn given_managed_server_running_tickless(world: &mut E2eWorld) { + plant_managed_server(world, false); +} + +fn plant_managed_server(world: &mut E2eWorld, with_start_ticks: bool) { + let services_dir = state(world) + .smoke_data + .as_ref() + .expect("isolated data dir must be seeded before planting a service record") + .join("services"); + let server = spawn_long_lived_child(); + // Port 0 is the one port nothing can ever be serving on: it resolves, so the + // CLI really does run its probe, and the connect always fails. Liveness then + // falls through to the recorded process, which is alive. Binding an + // ephemeral port and dropping it would leave a window in which something + // else on a busy runner grabs it and fails the scenario. + let free_port = 0; + write_service_record_with( + &services_dir, + "amd/test-model", + free_port, + ServiceRecordOptions { + status: "ready", + startup_phase: None, + supervisor_pid: server.id(), + engine_pid: Some(server.id()), + ..ServiceRecordOptions::default() + }, + ); + // A record written by a current `rocmd` carries the supervisor's start time; + // a tick-less live record is refused by uninstall rather than killed. + if !with_start_ticks { + state_mut(world).managed_server = Some(server); + return; + } + let record_path = services_dir.join("e2e-mock.json"); + let mut record: serde_json::Value = + serde_json::from_slice(&std::fs::read(&record_path).expect("failed to read record")) + .expect("failed to parse record"); + record["supervisor_start_ticks"] = + serde_json::json!(rocm_core::process_start_ticks(server.id())); + std::fs::write( + &record_path, + serde_json::to_vec_pretty(&record).expect("failed to serialize record"), + ) + .expect("failed to rewrite record"); + state_mut(world).managed_server = Some(server); +} + +/// The background helper `rocmd` as `rocm uninstall` finds it: only through the +/// runtime-state file a live daemon leaves, pid plus start time. +#[given("the background helper is running")] +async fn given_background_helper_running(world: &mut E2eWorld) { + let automations_dir = state(world) + .smoke_data + .as_ref() + .expect("isolated data dir must be seeded before planting helper state") + .join("automations"); + std::fs::create_dir_all(&automations_dir).expect("failed to create automations dir"); + let helper = spawn_long_lived_child(); + let runtime_state = serde_json::json!({ + "running": true, + "automations_enabled": true, + "daemon_pid": helper.id(), + "daemon_start_ticks": rocm_core::process_start_ticks(helper.id()), + "started_at_unix_ms": 0, + "last_tick_unix_ms": 0, + "active_watchers": [], + }); + std::fs::write( + automations_dir.join("runtime-state.json"), + serde_json::to_vec_pretty(&runtime_state).expect("failed to serialize helper state"), + ) + .expect("failed to write helper state"); + state_mut(world).background_helper = Some(helper); +} + +/// A runtime-state file the CLI cannot parse: it cannot tell whether a helper +/// is running, so uninstall must refuse rather than delete the tooling. +#[given("the background helper runtime state cannot be parsed")] +async fn given_unparseable_helper_state(world: &mut E2eWorld) { + let automations_dir = state(world) + .smoke_data + .as_ref() + .expect("isolated data dir must be seeded before planting helper state") + .join("automations"); + std::fs::create_dir_all(&automations_dir).expect("failed to create automations dir"); + let path = automations_dir.join("runtime-state.json"); + std::fs::write(&path, b"{ not json").expect("failed to write unparseable helper state"); + state_mut(world).broken_record = Some(path); +} + +/// An unparseable `*.json` under the isolated `services/` dir: the CLI cannot +/// tell whether it describes a running server. +#[given("a managed service record that cannot be parsed")] +async fn given_unparseable_service_record(world: &mut E2eWorld) { + let services_dir = state(world) + .smoke_data + .as_ref() + .expect("isolated data dir must be seeded before planting a service record") + .join("services"); + std::fs::create_dir_all(&services_dir).expect("failed to create services dir"); + std::fs::write(services_dir.join("broken.json"), b"{ not json") + .expect("failed to write unparseable record"); + state_mut(world).broken_record = Some(services_dir.join("broken.json")); +} + // ── When: package ────────────────────────────────────────────────────── async fn package_with_key_file(world: &mut E2eWorld) { @@ -1152,6 +1322,11 @@ fn run_uninstall(world: &mut E2eWorld, extra_args: &[&str]) { st.last_rc = output.status.code().unwrap_or(-1); } +#[when("the user uninstalls keeping binaries and data")] +async fn when_uninstall_cache_only(world: &mut E2eWorld) { + run_uninstall(world, &["--keep-binaries", "--keep-data"]); +} + /// Apply the isolated config/data/cache env for installed-binary invocations. fn apply_smoke_env(cmd: &mut Command, st: &LifecycleState) { if let Some(dir) = &st.smoke_config { @@ -1406,6 +1581,260 @@ async fn then_examine_isolated(world: &mut E2eWorld) { } } +#[then("the removal is reported as complete")] +async fn then_removal_complete(world: &mut E2eWorld) { + let out = &state(world).last_output; + assert_eq!( + state(world).last_rc, + 0, + "uninstall did not exit zero:\n{out}" + ); + assert!( + out.contains("uninstall complete"), + "uninstall did not report completion:\n{out}" + ); +} + +#[then("the removal is refused with advice to repair or delete that record")] +async fn then_removal_refused(world: &mut E2eWorld) { + let st = state(world); + let out = &st.last_output; + let record = st.broken_record.as_ref().expect("no broken record planted"); + assert_ne!( + st.last_rc, 0, + "uninstall succeeded despite an unreadable record:\n{out}" + ); + assert!( + out.contains("repair or delete"), + "refusal did not advise repairing or deleting the record:\n{out}" + ); + assert!( + out.contains(&record.display().to_string()), + "refusal did not name the unreadable record {}:\n{out}", + record.display() + ); + assert!( + !out.contains("uninstall complete"), + "refused uninstall still reported completion:\n{out}" + ); +} + +#[then("the removal is refused because the server cannot be proven ours")] +async fn then_refused_tickless(world: &mut E2eWorld) { + let out = state(world).last_output.clone(); + assert_ne!(state(world).last_rc, 0, "uninstall succeeded:\n{out}"); + assert!( + out.contains("uninstall will refuse to remove anything") + && out.contains("no start time was recorded"), + "the plan did not say the removal will be refused for want of a start time:\n{out}" + ); + assert!( + !out.contains("uninstall complete"), + "refused uninstall still reported completion:\n{out}" + ); + let server = state_mut(world) + .managed_server + .as_mut() + .expect("no managed server was started"); + assert!( + server.try_wait().expect("failed to poll").is_none(), + "uninstall killed a pid it could not prove was ours" + ); +} + +#[then("the removal is refused because the helper state cannot be read")] +async fn then_refused_helper_state(world: &mut E2eWorld) { + let st = state(world); + let out = &st.last_output; + let path = st.broken_record.as_ref().expect("no broken state planted"); + assert_ne!(st.last_rc, 0, "uninstall succeeded:\n{out}"); + assert!( + out.contains("uninstall will refuse to remove anything") + && out.contains(&path.display().to_string()), + "the plan did not name the unreadable helper state {}:\n{out}", + path.display() + ); + assert!( + !out.contains("uninstall complete"), + "refused uninstall still reported completion:\n{out}" + ); +} + +#[then("the installed rocm binary and manifest are still present")] +async fn then_install_kept(world: &mut E2eWorld) { + let st = state(world); + for path in [ + installed_binary(&st.install_dir, "rocm"), + st.install_dir.join(".rocm-cli-manifest"), + ] { + assert!( + path.exists(), + "refused uninstall removed {}", + path.display() + ); + } +} + +#[then("the isolated XDG state is still present")] +async fn then_refused_xdg_kept(world: &mut E2eWorld) { + let st = state(world); + let record = st.broken_record.as_ref().expect("no broken record planted"); + assert!( + record.exists(), + "refused uninstall removed {}", + record.display() + ); + for dir in [&st.smoke_config, &st.smoke_data, &st.smoke_cache] + .into_iter() + .flatten() + { + assert!(dir.exists(), "refused uninstall removed {}", dir.display()); + } +} + +/// Wait for our own child to exit, bounded by uninstall's stop grace. +fn child_exits_within_grace(child: &mut std::process::Child) -> bool { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + loop { + match child.try_wait().expect("failed to poll the child") { + Some(_) => return true, + None if std::time::Instant::now() >= deadline => return false, + None => std::thread::sleep(std::time::Duration::from_millis(100)), + } + } +} + +#[then("the background helper is no longer running")] +async fn then_background_helper_stopped(world: &mut E2eWorld) { + let mut helper = state_mut(world) + .background_helper + .take() + .expect("no background helper was started"); + let exited = child_exits_within_grace(&mut helper); + if !exited { + let _ = helper.kill(); + let _ = helper.wait(); + } + let out = &state(world).last_output; + assert!( + exited, + "uninstall reported success while the background helper kept running:\n{out}" + ); +} + +#[then("the uninstall plan says the background helper will be stopped")] +async fn then_plan_names_helper(world: &mut E2eWorld) { + // Asserted next to the outcome the `lifecycle-linux-uninstall-stops-background-helper` scenario checks in its following steps: + // the plan is only worth reading if the stop it promises happens. + let out = &state(world).last_output; + assert!( + out.contains("the background helper (rocmd, pid") && out.contains("will be stopped"), + "the plan did not say the helper will be stopped:\n{out}" + ); +} + +#[then("the uninstall plan says the managed servers will be stopped before removal")] +async fn then_plan_names_servers(world: &mut E2eWorld) { + let out = &state(world).last_output; + assert!( + out.contains("those servers will be stopped before removal"), + "the plan did not say the managed servers will be stopped before removal:\n{out}" + ); +} + +#[then("the uninstall result says one managed service was stopped")] +async fn then_result_names_stopped_service(world: &mut E2eWorld) { + let out = &state(world).last_output; + assert!( + out.contains("stopped 1 managed service(s) before removal"), + "the result did not report the stopped service:\n{out}" + ); +} + +#[then("the uninstall result says the background helper was stopped")] +async fn then_result_names_stopped_helper(world: &mut E2eWorld) { + let out = &state(world).last_output; + assert!( + out.contains("stopped the background helper (rocmd)"), + "the result did not report the stopped helper:\n{out}" + ); +} + +#[then("the local server this machine manages is still running")] +async fn then_managed_server_still_running(world: &mut E2eWorld) { + let server = state_mut(world) + .managed_server + .as_mut() + .expect("no managed server was started"); + // A run keeping binaries and data keeps the tooling that stops servers, so it must leave + // the server alone; give a wrongly-issued stop time to land first. + std::thread::sleep(std::time::Duration::from_secs(2)); + assert!( + server.try_wait().expect("failed to poll").is_none(), + "an uninstall keeping binaries and data stopped the managed server" + ); +} + +#[then("the keep-binaries-and-data run says the server is left running")] +async fn then_cache_only_says_left_running(world: &mut E2eWorld) { + let out = &state(world).last_output; + assert!( + out.contains("so those servers are left running"), + "the keep-binaries-and-data run did not say the server is left running:\n{out}" + ); + assert_eq!( + state(world).last_rc, + 0, + "the keep-binaries-and-data run exited non-zero:\n{out}" + ); + assert!( + !out.contains("will refuse to remove anything"), + "the keep-binaries-and-data run printed a refusal:\n{out}" + ); +} + +#[then("the managed service record is still present")] +async fn then_service_record_present(world: &mut E2eWorld) { + let record = state(world) + .smoke_data + .as_ref() + .expect("isolated data dir not seeded") + .join("services") + .join("e2e-mock.json"); + assert!(record.exists(), "the service record was removed"); +} + +#[then("the local server this machine manages is no longer running")] +async fn then_managed_server_stopped(world: &mut E2eWorld) { + let mut server = state_mut(world) + .managed_server + .take() + .expect("no managed server was started"); + // The stand-in is our own child, so its exit is observable directly rather + // than through a PID probe that a zombie would answer wrongly. Uninstall + // waits out its own bounded stop grace, so allow for that before failing. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(30); + let exited = loop { + match server + .try_wait() + .expect("failed to poll the managed server") + { + Some(_) => break true, + None if std::time::Instant::now() >= deadline => break false, + None => std::thread::sleep(std::time::Duration::from_millis(100)), + } + }; + if !exited { + let _ = server.kill(); + let _ = server.wait(); + } + let out = &state(world).last_output; + assert!( + exited, + "uninstall reported success while the server it manages kept running:\n{out}" + ); +} + #[then("the installed rocm binary that ran uninstall is gone")] async fn then_running_rocm_gone(world: &mut E2eWorld) { let st = state(world); @@ -1638,6 +2067,39 @@ async fn then_rocmd_gone(world: &mut E2eWorld) { assert_missing(&installed_binary(&st.install_dir, "rocmd")); } +/// The isolated folders are the CLI's own roots (the `ROCM_CLI_*_DIR` values), +/// so a purge removes the folder itself, not a `rocm-cli` subfolder of it. +#[then("the isolated config, data, and cache folders are gone")] +async fn then_isolated_folders_gone(world: &mut E2eWorld) { + let st = state(world); + for dir in [&st.smoke_config, &st.smoke_data, &st.smoke_cache] + .into_iter() + .flatten() + { + assert_missing(dir); + } +} + +#[then("only the isolated config and cache folders are gone")] +async fn then_only_cache_gone(world: &mut E2eWorld) { + let st = state(world); + assert_missing(st.smoke_cache.as_ref().expect("no isolated cache dir")); + assert_missing(st.smoke_config.as_ref().expect("no isolated config dir")); + let data = st.smoke_data.as_ref().expect("no isolated data dir"); + assert!( + data.exists(), + "an uninstall keeping the data removed {}", + data.display() + ); +} + +#[then("the installed rocm and rocmd binaries are still present")] +async fn then_binaries_still_present(world: &mut E2eWorld) { + let st = state(world); + assert_exists(&installed_binary(&st.install_dir, "rocm")); + assert_exists(&installed_binary(&st.install_dir, "rocmd")); +} + #[then("the isolated XDG config, data, and cache state is gone")] async fn then_xdg_gone(world: &mut E2eWorld) { let st = state(world); From 2960e90f47d75a75a6a1e5d712addd3a0510ce59 Mon Sep 17 00:00:00 2001 From: fredespi Date: Fri, 30 Oct 2026 18:08:04 +0100 Subject: [PATCH 2/4] fix(protocol): fill the port placeholder before the model so a model named like it cannot steal it Signed-off-by: fredespi --- crates/rocm-engine-protocol/src/lib.rs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/crates/rocm-engine-protocol/src/lib.rs b/crates/rocm-engine-protocol/src/lib.rs index cefb13c9b2..b020b51f10 100644 --- a/crates/rocm-engine-protocol/src/lib.rs +++ b/crates/rocm-engine-protocol/src/lib.rs @@ -82,6 +82,8 @@ pub fn relaunch_command(host: &str, require_api_key: bool) -> String { /// /// The template stays the single source of the flag set; this substitutes its /// model and port placeholders and appends the engine, so the advice runs as printed. +/// The port goes in first: it is digits and cannot collide, whereas the model is +/// arbitrary text that must never be rescanned for a placeholder. #[must_use] pub fn relaunch_command_for( model: &str, @@ -91,8 +93,8 @@ pub fn relaunch_command_for( require_api_key: bool, ) -> String { let mut command = relaunch_command(host, require_api_key) - .replacen("", &shell_word(model), 1) - .replacen("", &port.to_string(), 1); + .replacen("", &port.to_string(), 1) + .replacen("", &shell_word(model), 1); if !engine.is_empty() { command.push_str(" --engine "); command.push_str(&shell_word(engine)); @@ -933,6 +935,19 @@ mod tests { assert!(!relaunch_command_for("m", "vllm", "127.0.0.1", 1, false).contains('<')); } + #[test] + fn a_model_named_like_the_port_placeholder_does_not_steal_the_port() { + let command = relaunch_command_for("/models/.gguf", "", "127.0.0.1", 8123, false); + assert!( + command.contains("--port 8123"), + "the real port placeholder must be the one filled: {command}" + ); + assert!( + command.contains("/models/.gguf"), + "the model text must survive untouched: {command}" + ); + } + #[test] fn an_apostrophe_in_a_model_path_is_escaped_for_the_shell_that_will_read_it() { let model = "/models/O'Brien's model.gguf"; From 9d4c6ffafd68cf6710c6766d327635befc6b637a Mon Sep 17 00:00:00 2001 From: fredespi Date: Mon, 2 Nov 2026 10:59:34 +0100 Subject: [PATCH 3/4] test(uninstall): drive the stop-fails abort through the real stop plumbing The abort for a service that survives its stop was only proven with hand-built stop reports. Run the real stop path with a surviving process and assert the gate aborts and the record is not marked stopped. Signed-off-by: fredespi --- apps/rocm/src/main.rs | 44 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index de245c1e60..4896e165d0 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -12551,6 +12551,50 @@ pub(crate) mod tests { let _ = fs::remove_dir_all(root); } + #[cfg(target_os = "linux")] + #[test] + fn a_live_service_whose_processes_survive_the_real_stop_aborts_the_uninstall() { + // End to end through the real stop plumbing: the record is loaded and + // rewritten by `stop_internal_managed_service_confirmed_with`, the confirmed + // verdict is computed from what termination reported, and only the final + // signalling is replaced by one that leaves the process alive (as a + // SIGKILL-resistant child would). No hand-built report is involved. + let child = crate::test_support::bare_command("sleep") + .arg("60") + .spawn() + .expect("spawn the managed server"); + let pid = child.id(); + let (root, paths) = test_paths("uninstall-real-stop-survivor"); + let mut record = managed_record_for_pid(&paths, pid, rocm_core::process_start_ticks(pid)); + record.status = "ready".to_owned(); + record.write().expect("write service record"); + + let report = stop_managed_services_confirmed_with(&paths, |paths, id| { + crate::services_text::stop_internal_managed_service_confirmed_with(paths, id, |_| { + (vec![pid], false) + }) + }) + .expect("stop pass"); + + assert!(report.stopped.is_empty(), "{report:?}"); + assert_eq!(report.failed.len(), 1, "{report:?}"); + let error = uninstall_removal_gate(&report).expect_err("a survivor must abort"); + assert!( + format!("{error:#}").contains("svc-managed-stop"), + "{error:#}" + ); + let on_disk = load_managed_service(&paths, "svc-managed-stop").expect("record kept"); + assert!( + on_disk.stop_requested_unix_ms.is_some() && on_disk.status != "stopped", + "an unconfirmed stop must not be recorded as stopped: {on_disk:?}" + ); + let mut child = child; + assert!(child.try_wait().expect("poll").is_none(), "still alive"); + let _ = child.kill(); + let _ = child.wait(); + let _ = fs::remove_dir_all(root); + } + #[cfg(target_os = "linux")] #[test] fn a_stop_that_finds_the_record_already_stopped_still_reports_it_unconfirmed() { From b8eda02b42fe35ee5ade8e328b871c06ea3da4cc Mon Sep 17 00:00:00 2001 From: fredespi Date: Tue, 3 Nov 2026 20:48:47 +0100 Subject: [PATCH 4/4] style: cargo fmt after merging main Signed-off-by: fredespi --- apps/rocm/src/main.rs | 4 +++- apps/rocm/src/uninstall_gate.rs | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index c56f46b08f..e7002153d4 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -8624,7 +8624,9 @@ pub(crate) mod tests { if stopping.load(Ordering::SeqCst) { break; } - stream.set_read_timeout(Some(std::time::Duration::from_secs(2))).ok(); + stream + .set_read_timeout(Some(std::time::Duration::from_secs(2))) + .ok(); let mut buffer = [0_u8; 1024]; let Ok(read) = stream.read(&mut buffer) else { continue; diff --git a/apps/rocm/src/uninstall_gate.rs b/apps/rocm/src/uninstall_gate.rs index 322dfbab4e..68116a9558 100644 --- a/apps/rocm/src/uninstall_gate.rs +++ b/apps/rocm/src/uninstall_gate.rs @@ -17,9 +17,9 @@ use std::path::PathBuf; use std::time::Duration; use crate::chat::loopback_tcp_port_is_reachable; -use crate::services_text::{MANAGED_STOP_GRACE, stop_internal_managed_service_confirmed}; use crate::managed_service_status::{load_managed_services, managed_service_is_live}; use crate::service_records::managed_service_record_is_in_use; +use crate::services_text::{MANAGED_STOP_GRACE, stop_internal_managed_service_confirmed}; use crate::{endpoint_keys, host_port_resolves}; /// A managed service uninstall could not confirm stopped, and why.