From f911a5c7308b487b41ed9027e1574d171f53e6bc Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 17:38:10 +0000 Subject: [PATCH 01/30] test: pin what rocm and rocmd each do when a managed stop terminates its recorded PIDs rocm (service_control.rs) and rocmd (service.rs) each carry their own MANAGED_STOP_GRACE and terminate_recorded_service_pids, and the two have diverged. Pin what each copy does today, with no production change, so the change that merges them has to edit the tests that describe the difference: the grace value (10 s in both), the order the PIDs are visited (supervisor, then engine), the per-PID start-time pairing and the de-duplication of a PID recorded under both roles, zero PIDs, and the per-outcome signalled/forced/stopped classification. The two divergences are pinned as such: rocm drops the stopping process's own PID from the report while rocmd reports it as `self`, and the Windows `taskkill /T /F` step exists only in rocmd. The latter is a private cfg(windows) block with no injection point, so it cannot be observed from a test yet; only its portable half (an absent PID reaches terminate_verified unchanged) is pinned here. Signed-off-by: Jussi Elo --- apps/rocm/src/service_control.rs | 171 +++++++++++++++++++++++++++++ apps/rocmd/src/service.rs | 180 +++++++++++++++++++++++++++++++ 2 files changed, 351 insertions(+) diff --git a/apps/rocm/src/service_control.rs b/apps/rocm/src/service_control.rs index 2f4db4617e..c6047c8756 100644 --- a/apps/rocm/src/service_control.rs +++ b/apps/rocm/src/service_control.rs @@ -2559,4 +2559,175 @@ mod tests { ); let _ = fs::remove_dir_all(root); } + + // ---- Characterisation of this module's own managed-stop routine. ---- + // + // `rocmd` carries a second copy of `MANAGED_STOP_GRACE` and of + // `terminate_recorded_service_pids`, and the two have diverged. These pin + // what THIS copy does today so that the change that merges them is visible + // in the tests it has to edit. Every PID below is one that cannot be + // running, so nothing is signalled and the tests behave the same on every + // host. + + /// A PID no host has in use, so the stop reports it as already gone. + const ABSENT_SUPERVISOR_PID: u32 = 999_999_998; + const ABSENT_ENGINE_PID: u32 = 999_999_997; + + /// A record naming `supervisor` and `engine` with no recorded start-times. + fn record_naming( + paths: &AppPaths, + supervisor: u32, + engine: Option, + ) -> ManagedServiceRecord { + let mut record = ManagedServiceRecord::new( + paths, + "svc-characterise-stop", + "vllm", + "m", + "m", + "127.0.0.1", + 9, + "managed", + supervisor, + None, + None, + None, + ); + record.engine_pid = engine; + record + } + + /// The `(pid, role, outcome)` rows of a stop's per-PID report, in order. + fn outcome_rows( + termination: &RecordedStopTermination, + ) -> Vec<(u32, &'static str, rocm_core::TerminationOutcome)> { + termination + .pid_outcomes + .iter() + .map(|entry| (entry.pid, entry.role, entry.outcome)) + .collect() + } + + #[test] + fn managed_stop_grace_is_ten_seconds() { + // `rocmd` keeps its own copy of this value, in step only by a comment. + assert_eq!(MANAGED_STOP_GRACE, Duration::from_secs(10)); + } + + #[test] + fn managed_stop_reports_supervisor_then_engine_in_that_order() { + let (root, paths) = test_paths("managed-stop-order"); + let record = record_naming(&paths, ABSENT_SUPERVISOR_PID, Some(ABSENT_ENGINE_PID)); + + let termination = terminate_recorded_service_pids(&record); + let _ = fs::remove_dir_all(root); + + assert_eq!( + outcome_rows(&termination), + vec![ + ( + ABSENT_SUPERVISOR_PID, + "supervisor", + rocm_core::TerminationOutcome::AlreadyGone + ), + ( + ABSENT_ENGINE_PID, + "engine", + rocm_core::TerminationOutcome::AlreadyGone + ), + ] + ); + assert!( + termination.signaled_pids.is_empty(), + "nothing that is not running is signalled" + ); + assert!(termination.all_stopped); + } + + #[test] + fn managed_stop_reports_a_pid_recorded_twice_once_under_both_roles() { + let (root, paths) = test_paths("managed-stop-aliased"); + let record = record_naming(&paths, ABSENT_SUPERVISOR_PID, Some(ABSENT_SUPERVISOR_PID)); + + let termination = terminate_recorded_service_pids(&record); + let _ = fs::remove_dir_all(root); + + assert_eq!( + outcome_rows(&termination), + vec![( + ABSENT_SUPERVISOR_PID, + "supervisor_and_engine", + rocm_core::TerminationOutcome::AlreadyGone + )] + ); + } + + /// DIVERGENCE from `rocmd`: this copy drops the stopping process's own PID + /// before anything is reported, where `rocmd` reports it as `self`. + #[test] + fn managed_stop_leaves_its_own_pid_out_of_the_report() { + let (root, paths) = test_paths("managed-stop-own-pid"); + let record = record_naming(&paths, std::process::id(), Some(ABSENT_ENGINE_PID)); + + let termination = terminate_recorded_service_pids(&record); + let _ = fs::remove_dir_all(root); + + assert_eq!( + outcome_rows(&termination), + vec![( + ABSENT_ENGINE_PID, + "engine", + rocm_core::TerminationOutcome::AlreadyGone + )], + "the stopping process is absent from the per-PID report" + ); + assert!( + termination.all_stopped, + "and it does not stand in the way of a confirmed stop" + ); + assert!(termination.signaled_pids.is_empty()); + } + + #[test] + fn managed_stop_of_a_record_naming_only_its_own_pid_confirms_with_an_empty_report() { + let (root, paths) = test_paths("managed-stop-only-own-pid"); + let record = record_naming(&paths, std::process::id(), None); + + let termination = terminate_recorded_service_pids(&record); + let _ = fs::remove_dir_all(root); + + assert!(termination.pid_outcomes.is_empty()); + assert!(termination.signaled_pids.is_empty()); + assert!(termination.all_stopped); + } + + #[cfg(unix)] + #[test] + fn managed_stop_signals_the_supervisor_before_the_engine() { + let sup = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn supervisor"); + let eng = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn engine"); + let (sup_pid, eng_pid) = (sup.id(), eng.id()); + let (root, paths) = test_paths("managed-stop-signal-order"); + let mut record = + managed_record_for_pid(&paths, sup_pid, rocm_core::process_start_ticks(sup_pid)); + record.engine_pid = Some(eng_pid); + record.engine_start_ticks = rocm_core::process_start_ticks(eng_pid); + + let termination = terminate_recorded_service_pids(&record); + let (mut sup, mut eng) = (sup, eng); + let _ = sup.wait(); + let _ = eng.wait(); + let _ = fs::remove_dir_all(root); + + // Not sorted: the order is the routine's, and a copy that visited the + // engine first would fail here. + assert_eq!(termination.signaled_pids, vec![sup_pid, eng_pid]); + assert!(termination.all_stopped); + } } diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index 41e7fcf844..7aa4ab10e8 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -1080,4 +1080,184 @@ mod tests { ); Ok(()) } + + // ---- Characterisation of this module's own managed-stop routine. ---- + // + // `rocm` carries a second copy of `MANAGED_STOP_GRACE` and of + // `terminate_recorded_service_pids`, and the two have diverged. These pin + // what THIS copy does today so that the change that merges them is visible + // in the tests it has to edit. The termination is a stand-in throughout, so + // no process is signalled and the tests behave the same on every host. + + const SUPERVISOR_PID: u32 = 999_999_998; + const ENGINE_PID: u32 = 999_999_997; + + fn record_naming( + paths: &AppPaths, + supervisor: u32, + supervisor_ticks: Option, + engine: Option, + engine_ticks: Option, + ) -> ManagedServiceRecord { + let mut record = ManagedServiceRecord::new( + paths, + "svc-characterise-stop", + "vllm", + "m", + "m", + "127.0.0.1", + 9, + "managed", + supervisor, + None, + None, + None, + ); + record.supervisor_start_ticks = supervisor_ticks; + record.engine_pid = engine; + record.engine_start_ticks = engine_ticks; + record + } + + /// The identities `terminate_recorded_service_pids` handed its termination + /// function, in call order, together with what each returned as a stop. + fn terminate_with( + record: &ManagedServiceRecord, + outcome: rocm_core::TerminationOutcome, + ) -> (Vec, Vec) { + let seen = std::cell::RefCell::new(Vec::new()); + let stops = terminate_recorded_service_pids(record, |identity| { + seen.borrow_mut().push(*identity); + outcome + }); + (seen.into_inner(), stops) + } + + #[test] + fn managed_stop_grace_is_ten_seconds() { + // `rocm` keeps its own copy of this value, in step only by a comment. + assert_eq!(MANAGED_STOP_GRACE, Duration::from_secs(10)); + } + + #[test] + fn managed_stop_terminates_supervisor_then_engine_each_with_its_own_start_time() { + let (root, paths) = temp_app_paths("managed-stop-order"); + let record = record_naming(&paths, SUPERVISOR_PID, Some(11), Some(ENGINE_PID), Some(22)); + + let (seen, stops) = terminate_with(&record, rocm_core::TerminationOutcome::Graceful); + fs::remove_dir_all(root).ok(); + + assert_eq!( + seen, + vec![ + rocm_core::ProcessIdentity::new(SUPERVISOR_PID, Some(11)), + rocm_core::ProcessIdentity::new(ENGINE_PID, Some(22)), + ], + "the supervisor first, and no start-time read across from the other PID" + ); + assert_eq!( + stops + .iter() + .map(|stop| (stop.pid, stop.role)) + .collect::>(), + vec![(SUPERVISOR_PID, "supervisor"), (ENGINE_PID, "engine")] + ); + } + + #[test] + fn managed_stop_terminates_a_pid_recorded_twice_once_and_keeps_a_known_start_time() { + let (root, paths) = temp_app_paths("managed-stop-aliased-ticks"); + // The supervisor entry carries no start-time, the engine entry does: + // the one PID is stopped once, with the known token. + let record = record_naming(&paths, SUPERVISOR_PID, None, Some(SUPERVISOR_PID), Some(33)); + + let (seen, stops) = terminate_with(&record, rocm_core::TerminationOutcome::Graceful); + fs::remove_dir_all(root).ok(); + + assert_eq!( + seen, + vec![rocm_core::ProcessIdentity::new(SUPERVISOR_PID, Some(33))] + ); + assert_eq!(stops.len(), 1); + assert_eq!(stops[0].role, "supervisor_and_engine"); + } + + #[test] + fn managed_stop_skips_a_zero_pid_and_an_absent_engine() { + let (root, paths) = temp_app_paths("managed-stop-zero-pid"); + let record = record_naming(&paths, 0, None, Some(0), None); + + let (seen, stops) = terminate_with(&record, rocm_core::TerminationOutcome::Graceful); + fs::remove_dir_all(root).ok(); + + assert!(seen.is_empty()); + assert!(stops.is_empty()); + } + + /// DIVERGENCE from `rocm`: this copy reports the stopping process's own PID, + /// as `self`, where `rocm` drops it. It is never handed to the termination + /// function, and it counts as stopped. + #[test] + fn managed_stop_reports_its_own_pid_as_self_and_never_terminates_it() { + let (root, paths) = temp_app_paths("managed-stop-own-pid"); + let record = record_naming(&paths, std::process::id(), None, Some(ENGINE_PID), None); + + let (seen, stops) = terminate_with(&record, rocm_core::TerminationOutcome::Graceful); + fs::remove_dir_all(root).ok(); + + assert_eq!( + seen, + vec![rocm_core::ProcessIdentity::new(ENGINE_PID, None)], + "the stopping process is never signalled" + ); + assert_eq!(stops.len(), 2); + assert_eq!(stops[0].pid, std::process::id()); + assert_eq!(stops[0].role, "supervisor"); + assert_eq!(stops[0].outcome, "self"); + assert!(!stops[0].signaled && !stops[0].forced); + assert!(stops[0].stopped, "the launcher is not the service"); + } + + #[test] + fn managed_stop_classifies_each_outcome_as_signaled_forced_and_stopped() { + use rocm_core::TerminationOutcome::{ + AlreadyGone, Forced, Graceful, IdentityMismatch, TimedOut, Unverified, + }; + let (root, paths) = temp_app_paths("managed-stop-classes"); + let record = record_naming(&paths, SUPERVISOR_PID, None, None, None); + + // (outcome, signaled, forced, stopped) + let expected = [ + (AlreadyGone, false, false, true), + (IdentityMismatch, false, false, true), + (Unverified, false, false, false), + (Graceful, true, false, true), + (Forced, true, true, true), + (TimedOut, true, true, false), + ]; + for (outcome, signaled, forced, stopped) in expected { + let (_, stops) = terminate_with(&record, outcome); + assert_eq!(stops.len(), 1); + let stop = &stops[0]; + assert_eq!(stop.outcome, outcome.as_str()); + assert_eq!( + (stop.signaled, stop.forced, stop.stopped), + (signaled, forced, stopped), + "{outcome:?}" + ); + } + fs::remove_dir_all(root).ok(); + } + + /// The tree-kill step is Windows-only and is a private `cfg(windows)` block + /// of `terminate_recorded_pid` with no injection point, so nothing here can + /// observe it. What this does pin is the portable half: a recorded PID that + /// is not running reaches `terminate_verified` and comes back unchanged. + #[test] + fn terminate_recorded_pid_reports_an_absent_pid_as_already_gone() { + let outcome = + terminate_recorded_pid(&rocm_core::ProcessIdentity::new(SUPERVISOR_PID, None)); + + assert_eq!(outcome, rocm_core::TerminationOutcome::AlreadyGone); + } } From 0cccfc5ecc2c167a644eb7ece9224bb0950a82f2 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 18:14:55 +0000 Subject: [PATCH 02/30] refactor: stop managed services through one rocm-core routine shared by rocm and rocmd `rocm services stop` and `rocmd`'s `stop_server` each carried their own MANAGED_STOP_GRACE (10 s, kept in step only by a comment) and their own terminate_recorded_service_pids, and the copies had diverged. Move one routine and one constant into rocm-core's proc_lifecycle.rs, beside terminate_verified, and delete both private copies. The termination of each PID stays injectable. rocmd's behaviour wins, because it is the more complete one: - The stopping process's own PID is reported as `self` (never signalled, counted as stopped). rocm used to drop it, so `pid_outcomes` listed one entry fewer than rocmd's for the same record. - On Windows a verified root is preceded by `taskkill /PID /T /F`, because terminate_verified cannot walk a process tree there and the engine runs one level below the recorded PID. rocm never did this, so on Windows `rocm services stop|restart`, the sandbox `stop_server`/`restart_server` tool and the attached `rocm serve` Ctrl-C stop now take the engine's subtree down as `rocmd` already did. A root the tree kill already took is reported `forced`, not `already_gone`. The Windows step sits behind terminate_recorded_pid_with, which takes the tree-kill, the identity read and the termination as arguments, so its tests run on every host. Each caller's tests drive its own stop through the shared routine, and each carries a source check that no private copy has come back. Signed-off-by: Jussi Elo --- apps/rocm/src/service_control.rs | 544 +++++++++++------------- apps/rocm/src/service_launch.rs | 2 +- apps/rocmd/src/service.rs | 450 +++++++------------- crates/rocm-core/src/lib.rs | 5 +- crates/rocm-core/src/proc_lifecycle.rs | 564 +++++++++++++++++++++++++ docs/architecture.md | 6 +- docs/testing.md | 21 + 7 files changed, 969 insertions(+), 623 deletions(-) diff --git a/apps/rocm/src/service_control.rs b/apps/rocm/src/service_control.rs index c6047c8756..c3d9986943 100644 --- a/apps/rocm/src/service_control.rs +++ b/apps/rocm/src/service_control.rs @@ -8,9 +8,9 @@ //! Mechanically relocated from `services_text.rs` with no behavior change. //! //! This module owns: the sandbox-tool runner (`run_internal_sandbox_tool`), the -//! stop path (`terminate_recorded_service_pids`, the `MANAGED_STOP_GRACE` -//! constant, the private `RecordedPidOutcome`/`RecordedStopTermination` types -//! and the `scripted_stop_outcome` end-to-end hook, `stop_internal_managed_service`, +//! stop path (`stop_recorded_processes`, which hands the recorded PIDs to the +//! routine `rocm-core` shares with `rocmd`, the `scripted_stop_outcome` +//! end-to-end hook, `stop_internal_managed_service`, //! `stop_internal_managed_service_with`, //! `stop_internal_managed_service_confirmed_with`, //! `unload_lemonade_service_model`), and the restart path @@ -126,119 +126,14 @@ pub(crate) fn run_internal_sandbox_tool( })) } -/// How long a managed stop waits for each recorded process to actually exit -/// after `SIGTERM` before escalating to `SIGKILL` and then reporting a timeout. -const MANAGED_STOP_GRACE: Duration = Duration::from_secs(10); - -/// One recorded PID's fate in a managed stop: the role the record gave it, and -/// what [`rocm_core::terminate_verified`] did about it. Kept per PID because the -/// outcomes that leave a stop unconfirmed are reported differently: a -/// `TimedOut` PID was signalled and it or a process under it outlived the -/// deadline, while an `Unverified` one is not signalled by this stop, and the -/// report names a command to check on it. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct RecordedPidOutcome { - pid: u32, - /// `supervisor`, `engine`, or `supervisor_and_engine` when the record names - /// one PID for both — the same labels `rocmd`'s `pid_outcomes` uses. - role: &'static str, - outcome: rocm_core::TerminationOutcome, -} - -/// What a managed stop did to the processes its record names. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub(crate) struct RecordedStopTermination { - /// Every PID that received a signal, including one that then outlived the - /// deadline. - signaled_pids: Vec, - /// Whether every recorded process is confirmed no longer running. - all_stopped: bool, - pid_outcomes: Vec, -} - -impl RecordedStopTermination { - /// A stop stand-in that confirms every recorded process gone without - /// signalling any, for tests in other modules that drive the real stop path. - #[cfg(test)] - pub(crate) fn confirmed_stopped() -> Self { - Self { - all_stopped: true, - ..Self::default() - } - } -} - -/// Terminate the processes recorded for a managed service, verifying each PID's -/// identity first so a recycled PID never causes an unrelated process tree to be -/// killed. -/// -/// The supervisor (launcher) and the engine server are distinct processes, so -/// each PID is paired with **its own** start-time token (`supervisor_start_ticks` -/// vs `engine_start_ticks`). When they resolve to the same PID (e.g. before the -/// engine state has been refreshed) the entry is de-duplicated, preferring a -/// known token. Returns the PIDs actually signalled, whether every recorded -/// process is confirmed no longer running, and each PID's own outcome. -fn terminate_recorded_service_pids(record: &ManagedServiceRecord) -> RecordedStopTermination { - // Build the (pid, own-token, role) work list, de-duplicating on PID and - // preferring an entry that carries a verifiable start-time. - let mut entries: Vec<(u32, Option, &'static str)> = Vec::new(); - for (pid, ticks, role) in [ - ( - Some(record.supervisor_pid), - record.supervisor_start_ticks, - "supervisor", - ), - (record.engine_pid, record.engine_start_ticks, "engine"), - ] { - let Some(pid) = pid.filter(|pid| *pid != 0 && *pid != std::process::id()) else { - continue; - }; - if let Some(existing) = entries.iter_mut().find(|(seen, _, _)| *seen == pid) { - if existing.1.is_none() { - existing.1 = ticks; - } - existing.2 = "supervisor_and_engine"; - } else { - entries.push((pid, ticks, role)); - } - } - - let mut termination = RecordedStopTermination { - all_stopped: true, - ..RecordedStopTermination::default() - }; - for (pid, ticks, role) in entries { - let identity = rocm_core::ProcessIdentity::new(pid, ticks); - // The managed stop contract is definitive: after the adapter gets its - // graceful opportunity, this cleanup pass forces any verified survivor - // so the command does not report success while a GPU worker remains. - let outcome = scripted_stop_outcome().unwrap_or_else(|| { - rocm_core::terminate_verified( - &identity, - rocm_core::KillScope::Tree, - MANAGED_STOP_GRACE, - true, - ) - }); - if !outcome.stopped() { - termination.all_stopped = false; - } - // Report every PID that received a signal. TimedOut belongs here because - // signalling was attempted even though the process was not confirmed - // stopped; `all_stopped` separately carries the truthful completion state. - if matches!( - outcome, - rocm_core::TerminationOutcome::Graceful - | rocm_core::TerminationOutcome::Forced - | rocm_core::TerminationOutcome::TimedOut - ) { - termination.signaled_pids.push(pid); - } - termination - .pid_outcomes - .push(RecordedPidOutcome { pid, role, outcome }); - } - termination +/// Terminate the processes recorded for a managed service through the routine +/// `rocmd` shares, [`rocm_core::terminate_recorded_service_pids`], with +/// [`rocm_core::terminate_recorded_pid`] as the termination of each PID. The +/// one thing this adds is the e2e seam below. +fn stop_recorded_processes(record: &ManagedServiceRecord) -> Vec { + rocm_core::terminate_recorded_service_pids(record, |identity| { + scripted_stop_outcome().unwrap_or_else(|| rocm_core::terminate_recorded_pid(identity)) + }) } /// E2E hook: when `ROCM_E2E_STOP_OUTCOME` is `timed_out` or `unverified`, a @@ -266,12 +161,12 @@ const fn scripted_stop_outcome() -> Option { } fn stop_internal_managed_service(paths: &AppPaths, service_id: &str) -> Result { - stop_internal_managed_service_with(paths, service_id, terminate_recorded_service_pids) + stop_internal_managed_service_with(paths, service_id, stop_recorded_processes) } /// [`stop_internal_managed_service`], with the termination of the recorded /// processes supplied by the caller. Production passes -/// [`terminate_recorded_service_pids`]; a test passes a stand-in that observes +/// [`stop_recorded_processes`]; a test passes a stand-in that observes /// what is on disk at the moment the processes would be signalled. Everything /// else — the engine stop, the manifest writes, the key cleanup — runs for real. /// @@ -280,7 +175,7 @@ fn stop_internal_managed_service(paths: &AppPaths, service_id: &str) -> Result RecordedStopTermination, + terminate: impl FnOnce(&ManagedServiceRecord) -> Vec, ) -> Result { stop_internal_managed_service_confirmed_with(paths, service_id, terminate) .map(|(report, _)| report) @@ -294,7 +189,7 @@ pub(crate) fn stop_internal_managed_service_with( fn stop_internal_managed_service_confirmed_with( paths: &AppPaths, service_id: &str, - terminate: impl FnOnce(&ManagedServiceRecord) -> RecordedStopTermination, + terminate: impl FnOnce(&ManagedServiceRecord) -> Vec, ) -> Result<(serde_json::Value, bool)> { let mut stopping = load_managed_service(paths, service_id)?; // Record the request on disk before anything is stopped, not only after. @@ -322,11 +217,17 @@ fn stop_internal_managed_service_confirmed_with( }, ) }; - let RecordedStopTermination { - signaled_pids, - all_stopped: terminated_all, - pid_outcomes, - } = terminate(&stopping); + let stops = terminate(&stopping); + // Every PID that received a signal, including one that then outlived the + // deadline: signalling was attempted even though the process was not + // confirmed stopped, and `terminated_all` separately carries the truthful + // completion state. + let signaled_pids: Vec = stops + .iter() + .filter(|stop| stop.signaled()) + .map(|stop| stop.pid) + .collect(); + let terminated_all = stops.iter().all(|stop| stop.stopped()); // Re-read the record rather than write back the copy loaded before the // stop, which can be some forty seconds old by now: `write` replaces the // whole file, and a supervisor, the daemon's restart or another stop may @@ -401,15 +302,9 @@ fn stop_internal_managed_service_confirmed_with( "status": record.status, "engine_stop": engine_stop, "signaled_pids": signaled_pids, - "pid_outcomes": pid_outcomes + "pid_outcomes": stops .iter() - .map(|entry| { - serde_json::json!({ - "pid": entry.pid, - "role": entry.role, - "outcome": entry.outcome.as_str(), - }) - }) + .map(|stop| stop.report_entry()) .collect::>(), "record_changed": record_changed, // A container engine is confirmed stopped by its own `Stop`; when that @@ -517,12 +412,7 @@ pub(crate) fn restart_internal_managed_service( service_id: &str, runtime_pin: Option<&str>, ) -> Result { - restart_internal_managed_service_with( - paths, - service_id, - runtime_pin, - terminate_recorded_service_pids, - ) + restart_internal_managed_service_with(paths, service_id, runtime_pin, stop_recorded_processes) } /// [`restart_internal_managed_service`], with the termination of the recorded @@ -538,7 +428,7 @@ fn restart_internal_managed_service_with( paths: &AppPaths, service_id: &str, runtime_pin: Option<&str>, - terminate: impl FnOnce(&ManagedServiceRecord) -> RecordedStopTermination, + terminate: impl FnOnce(&ManagedServiceRecord) -> Vec, ) -> Result { restart_internal_managed_service_inner(paths, service_id, runtime_pin, terminate, None) } @@ -573,7 +463,7 @@ fn restart_internal_managed_service_inner( paths: &AppPaths, service_id: &str, runtime_pin: Option<&str>, - terminate: impl FnOnce(&ManagedServiceRecord) -> RecordedStopTermination, + terminate: impl FnOnce(&ManagedServiceRecord) -> Vec, spawn_override: Option<&RestartSpawnOverride<'_>>, ) -> Result { #[cfg(windows)] @@ -1075,7 +965,7 @@ mod tests { // Proves the *ordering*, not just the refusal: had the guard run after // `stop_internal_managed_service`, the stop would have written // "stopped". It reaches that state here because the record's only pid is - // the test's own (`engine_pid` is None and `terminate_recorded_service_pids` + // the test's own (`engine_pid` is None and the shared stop routine // skips the caller's pid), so the stop confirms termination trivially. let after = load_managed_service(&paths, service_id).unwrap(); assert_ne!( @@ -1372,27 +1262,41 @@ mod tests { assert_eq!(key.as_deref(), Some("key-123"), "the endpoint key is kept"); } - /// A stop stand-in's bare verdict, without per-PID detail. - fn bare_termination(signaled_pids: Vec, all_stopped: bool) -> RecordedStopTermination { - RecordedStopTermination { - signaled_pids, - all_stopped, - pid_outcomes: Vec::new(), + /// One recorded process's stop, for a stand-in to return. + fn stop_of(pid: u32, outcome: rocm_core::TerminationOutcome) -> rocm_core::RecordedPidStop { + rocm_core::RecordedPidStop { + pid, + role: "supervisor_and_engine", + outcome: rocm_core::RecordedPidOutcome::Terminated(outcome), + } + } + + /// A stop stand-in's bare verdict, without per-PID detail: `signaled_pids` + /// were signalled and `all_stopped` says whether they exited. A stop that + /// signalled nothing and is not confirmed has one unverifiable process. + fn bare_termination( + signaled_pids: Vec, + all_stopped: bool, + ) -> Vec { + use rocm_core::TerminationOutcome::{Graceful, TimedOut, Unverified}; + if signaled_pids.is_empty() { + return if all_stopped { + Vec::new() + } else { + vec![stop_of(999_999_999, Unverified)] + }; } + let outcome = if all_stopped { Graceful } else { TimedOut }; + signaled_pids + .into_iter() + .map(|pid| stop_of(pid, outcome)) + .collect() } /// A stop stand-in whose one recorded process (the record's shared /// supervisor/engine PID) was signalled and outlived the deadline. - fn timed_out_termination(pid: u32) -> RecordedStopTermination { - RecordedStopTermination { - signaled_pids: vec![pid], - all_stopped: false, - pid_outcomes: vec![RecordedPidOutcome { - pid, - role: "supervisor_and_engine", - outcome: rocm_core::TerminationOutcome::TimedOut, - }], - } + fn timed_out_termination(pid: u32) -> Vec { + vec![stop_of(pid, rocm_core::TerminationOutcome::TimedOut)] } /// The most recent audit event written under `paths`. @@ -1412,17 +1316,13 @@ mod tests { stop_test_record(&paths, service_id).write().unwrap(); endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); - let report = - stop_internal_managed_service_with(&paths, service_id, |_| RecordedStopTermination { - signaled_pids: Vec::new(), - all_stopped: false, - pid_outcomes: vec![RecordedPidOutcome { - pid: 999_999_999, - role: "supervisor_and_engine", - outcome: rocm_core::TerminationOutcome::Unverified, - }], - }) - .unwrap(); + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + vec![stop_of( + 999_999_999, + rocm_core::TerminationOutcome::Unverified, + )] + }) + .unwrap(); let answer = internal_stop_server_value(report); let outcome = finish_service_action(&paths, "stop_server", service_id, &answer); @@ -1485,24 +1385,19 @@ mod tests { record.engine_pid = Some(888_888_888); record.write().unwrap(); - let report = - stop_internal_managed_service_with(&paths, service_id, |_| RecordedStopTermination { - signaled_pids: vec![999_999_999], - all_stopped: false, - pid_outcomes: vec![ - RecordedPidOutcome { - pid: 999_999_999, - role: "supervisor", - outcome: rocm_core::TerminationOutcome::TimedOut, - }, - RecordedPidOutcome { - pid: 888_888_888, - role: "engine", - outcome: rocm_core::TerminationOutcome::Unverified, - }, - ], - }) - .unwrap(); + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + vec![ + rocm_core::RecordedPidStop { + role: "supervisor", + ..stop_of(999_999_999, rocm_core::TerminationOutcome::TimedOut) + }, + rocm_core::RecordedPidStop { + role: "engine", + ..stop_of(888_888_888, rocm_core::TerminationOutcome::Unverified) + }, + ] + }) + .unwrap(); let answer = internal_stop_server_value(report); let outcome = finish_service_action(&paths, "stop_server", service_id, &answer); let _ = fs::remove_dir_all(root); @@ -2385,7 +2280,7 @@ mod tests { "svc-cpu-policy", "vllm", "starting", - // Skipped by `terminate_recorded_service_pids`, which never + // Skipped by the shared stop routine, which never // signals this process — so the stop runs for real without the // test killing itself. std::process::id(), @@ -2471,15 +2366,14 @@ mod tests { let real = rocm_core::process_start_ticks(pid).expect("start-ticks"); let record = managed_record_for_pid(&paths, pid, Some(real.wrapping_add(1))); - let RecordedStopTermination { - signaled_pids: signaled, - all_stopped, - .. - } = terminate_recorded_service_pids(&record); + let stops = stop_recorded_processes(&record); - assert!(signaled.is_empty(), "must not signal a mismatched identity"); + assert!( + stops.iter().all(|stop| !stop.signaled()), + "must not signal a mismatched identity" + ); // A recycled PID means the recorded service process itself is gone. - assert!(all_stopped); + assert!(stops.iter().all(|stop| stop.stopped())); assert!( rocm_core::process_is_running(pid), "the unrelated live process must survive" @@ -2509,11 +2403,13 @@ mod tests { record.engine_pid = Some(eng_pid); record.engine_start_ticks = rocm_core::process_start_ticks(eng_pid); - let RecordedStopTermination { - signaled_pids: mut signaled, - all_stopped, - .. - } = terminate_recorded_service_pids(&record); + let stops = stop_recorded_processes(&record); + let all_stopped = stops.iter().all(|stop| stop.stopped()); + let mut signaled: Vec = stops + .iter() + .filter(|stop| stop.signaled()) + .map(|stop| stop.pid) + .collect(); signaled.sort_unstable(); let mut expected = vec![sup_pid, eng_pid]; expected.sort_unstable(); @@ -2541,14 +2437,15 @@ mod tests { let real = rocm_core::process_start_ticks(pid).expect("start-ticks"); let record = managed_record_for_pid(&paths, pid, Some(real)); - let RecordedStopTermination { - signaled_pids: signaled, - all_stopped, - .. - } = terminate_recorded_service_pids(&record); + let stops = stop_recorded_processes(&record); + let signaled: Vec = stops + .iter() + .filter(|stop| stop.signaled()) + .map(|stop| stop.pid) + .collect(); assert_eq!(signaled, vec![pid]); - assert!(all_stopped); + assert!(stops.iter().all(|stop| stop.stopped())); // Reap our own child (a detached managed process would be reaped by init) // so the liveness check does not observe a not-yet-reaped zombie. let mut child = child; @@ -2560,12 +2457,12 @@ mod tests { let _ = fs::remove_dir_all(root); } - // ---- Characterisation of this module's own managed-stop routine. ---- + // ---- This module's stop, through the routine it shares with `rocmd`. ---- // - // `rocmd` carries a second copy of `MANAGED_STOP_GRACE` and of - // `terminate_recorded_service_pids`, and the two have diverged. These pin - // what THIS copy does today so that the change that merges them is visible - // in the tests it has to edit. Every PID below is one that cannot be + // The routine itself (visiting order, start-time pairing, the `self` row, + // the per-outcome classification, the Windows tree-kill step) is tested in + // `rocm-core`'s `proc_lifecycle.rs`. These check that this caller reaches it + // and reports what it returns. Every PID below is one that cannot be // running, so nothing is signalled and the tests behave the same on every // host. @@ -2573,132 +2470,134 @@ mod tests { const ABSENT_SUPERVISOR_PID: u32 = 999_999_998; const ABSENT_ENGINE_PID: u32 = 999_999_997; - /// A record naming `supervisor` and `engine` with no recorded start-times. - fn record_naming( - paths: &AppPaths, - supervisor: u32, - engine: Option, - ) -> ManagedServiceRecord { - let mut record = ManagedServiceRecord::new( - paths, - "svc-characterise-stop", - "vllm", - "m", - "m", - "127.0.0.1", - 9, - "managed", - supervisor, - None, - None, - None, - ); + /// Run the production stop of a record naming `supervisor` and `engine`, + /// and return its report. + fn stop_report_for(name: &str, supervisor: u32, engine: Option) -> serde_json::Value { + let (root, paths) = test_paths(name); + paths.ensure().unwrap(); + let service_id = "svc-stop-through-the-shared-routine"; + let mut record = stop_test_record(&paths, service_id); + record.supervisor_pid = supervisor; record.engine_pid = engine; - record - } + record.write().unwrap(); - /// The `(pid, role, outcome)` rows of a stop's per-PID report, in order. - fn outcome_rows( - termination: &RecordedStopTermination, - ) -> Vec<(u32, &'static str, rocm_core::TerminationOutcome)> { - termination - .pid_outcomes - .iter() - .map(|entry| (entry.pid, entry.role, entry.outcome)) - .collect() + let report = + stop_internal_managed_service_with(&paths, service_id, stop_recorded_processes) + .unwrap(); + let _ = fs::remove_dir_all(root); + report } #[test] - fn managed_stop_grace_is_ten_seconds() { - // `rocmd` keeps its own copy of this value, in step only by a comment. - assert_eq!(MANAGED_STOP_GRACE, Duration::from_secs(10)); + fn a_stop_reports_the_supervisor_then_the_engine_in_that_order() { + let report = stop_report_for( + "managed-stop-order", + ABSENT_SUPERVISOR_PID, + Some(ABSENT_ENGINE_PID), + ); + + assert_eq!( + report["pid_outcomes"], + serde_json::json!([ + {"pid": ABSENT_SUPERVISOR_PID, "role": "supervisor", "outcome": "already_gone"}, + {"pid": ABSENT_ENGINE_PID, "role": "engine", "outcome": "already_gone"}, + ]) + ); + assert_eq!(report["signaled_pids"], serde_json::json!([])); + assert_eq!(report["stopped"], true); } #[test] - fn managed_stop_reports_supervisor_then_engine_in_that_order() { - let (root, paths) = test_paths("managed-stop-order"); - let record = record_naming(&paths, ABSENT_SUPERVISOR_PID, Some(ABSENT_ENGINE_PID)); - - let termination = terminate_recorded_service_pids(&record); - let _ = fs::remove_dir_all(root); + fn a_stop_reports_a_pid_recorded_twice_once_under_both_roles() { + let report = stop_report_for( + "managed-stop-aliased", + ABSENT_SUPERVISOR_PID, + Some(ABSENT_SUPERVISOR_PID), + ); assert_eq!( - outcome_rows(&termination), - vec![ - ( - ABSENT_SUPERVISOR_PID, - "supervisor", - rocm_core::TerminationOutcome::AlreadyGone - ), - ( - ABSENT_ENGINE_PID, - "engine", - rocm_core::TerminationOutcome::AlreadyGone - ), - ] - ); - assert!( - termination.signaled_pids.is_empty(), - "nothing that is not running is signalled" + report["pid_outcomes"], + serde_json::json!([ + {"pid": ABSENT_SUPERVISOR_PID, "role": "supervisor_and_engine", "outcome": "already_gone"}, + ]) ); - assert!(termination.all_stopped); } + /// Both binaries report the stopping process's own PID as `self`, never + /// signal it, and do not let it stand in the way of a confirmed stop. `rocm` + /// used to drop it from the report, so `pid_outcomes` showed one entry fewer + /// than `rocmd`'s for the same record. #[test] - fn managed_stop_reports_a_pid_recorded_twice_once_under_both_roles() { - let (root, paths) = test_paths("managed-stop-aliased"); - let record = record_naming(&paths, ABSENT_SUPERVISOR_PID, Some(ABSENT_SUPERVISOR_PID)); + fn a_stop_reports_its_own_pid_as_self() { + let own = std::process::id(); - let termination = terminate_recorded_service_pids(&record); - let _ = fs::remove_dir_all(root); + let report = stop_report_for("managed-stop-own-pid", own, Some(ABSENT_ENGINE_PID)); assert_eq!( - outcome_rows(&termination), - vec![( - ABSENT_SUPERVISOR_PID, - "supervisor_and_engine", - rocm_core::TerminationOutcome::AlreadyGone - )] + report["pid_outcomes"], + serde_json::json!([ + {"pid": own, "role": "supervisor", "outcome": "self"}, + {"pid": ABSENT_ENGINE_PID, "role": "engine", "outcome": "already_gone"}, + ]) + ); + assert_eq!( + report["signaled_pids"], + serde_json::json!([]), + "the stopping process is never signalled" ); + assert_eq!(report["stopped"], true); } - /// DIVERGENCE from `rocmd`: this copy drops the stopping process's own PID - /// before anything is reported, where `rocmd` reports it as `self`. #[test] - fn managed_stop_leaves_its_own_pid_out_of_the_report() { - let (root, paths) = test_paths("managed-stop-own-pid"); - let record = record_naming(&paths, std::process::id(), Some(ABSENT_ENGINE_PID)); + fn a_stop_of_a_record_naming_only_its_own_pid_confirms() { + let own = std::process::id(); - let termination = terminate_recorded_service_pids(&record); - let _ = fs::remove_dir_all(root); + let report = stop_report_for("managed-stop-only-own-pid", own, None); assert_eq!( - outcome_rows(&termination), - vec![( - ABSENT_ENGINE_PID, - "engine", - rocm_core::TerminationOutcome::AlreadyGone - )], - "the stopping process is absent from the per-PID report" - ); - assert!( - termination.all_stopped, - "and it does not stand in the way of a confirmed stop" + report["pid_outcomes"], + serde_json::json!([{"pid": own, "role": "supervisor", "outcome": "self"}]) ); - assert!(termination.signaled_pids.is_empty()); + assert_eq!(report["stopped"], true); } + /// The `self` row is not one of the outcomes that leave a stop unconfirmed, + /// so it never reaches the report's `still running` or `not signaled` lines. #[test] - fn managed_stop_of_a_record_naming_only_its_own_pid_confirms_with_an_empty_report() { - let (root, paths) = test_paths("managed-stop-only-own-pid"); - let record = record_naming(&paths, std::process::id(), None); + fn a_self_row_adds_no_line_to_an_unconfirmed_report() { + let (root, paths) = test_paths("managed-stop-self-row-unconfirmed"); + paths.ensure().unwrap(); + let service_id = "svc-self-row-unconfirmed"; + stop_test_record(&paths, service_id).write().unwrap(); + let own = std::process::id(); - let termination = terminate_recorded_service_pids(&record); + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + vec![ + rocm_core::RecordedPidStop { + pid: own, + role: "supervisor", + outcome: rocm_core::RecordedPidOutcome::OwnProcess, + }, + rocm_core::RecordedPidStop { + role: "engine", + ..stop_of(999_999_999, rocm_core::TerminationOutcome::TimedOut) + }, + ] + }) + .unwrap(); + let answer = internal_stop_server_value(report); + let outcome = finish_service_action(&paths, "stop_server", service_id, &answer); let _ = fs::remove_dir_all(root); + let rendered = format!("{:#}", outcome.expect_err("unconfirmed")); - assert!(termination.pid_outcomes.is_empty()); - assert!(termination.signaled_pids.is_empty()); - assert!(termination.all_stopped); + assert!( + rendered.contains("still running: process 999999999 (engine)"), + "{rendered}" + ); + assert!( + !rendered.contains(&format!("process {own}")), + "the stopping process is not a survivor: {rendered}" + ); } #[cfg(unix)] @@ -2719,7 +2618,7 @@ mod tests { record.engine_pid = Some(eng_pid); record.engine_start_ticks = rocm_core::process_start_ticks(eng_pid); - let termination = terminate_recorded_service_pids(&record); + let stops = stop_recorded_processes(&record); let (mut sup, mut eng) = (sup, eng); let _ = sup.wait(); let _ = eng.wait(); @@ -2727,7 +2626,34 @@ mod tests { // Not sorted: the order is the routine's, and a copy that visited the // engine first would fail here. - assert_eq!(termination.signaled_pids, vec![sup_pid, eng_pid]); - assert!(termination.all_stopped); + assert_eq!( + stops + .iter() + .filter(|stop| stop.signaled()) + .map(|stop| stop.pid) + .collect::>(), + vec![sup_pid, eng_pid] + ); + assert!(stops.iter().all(|stop| stop.stopped())); + } + + /// A second copy of the shared routine, or of its grace constant, is how the + /// two binaries diverged in the first place; the shared one lives in + /// `rocm-core`'s `proc_lifecycle.rs`. The needles are assembled so that this + /// test does not match itself. + #[test] + fn this_module_carries_no_private_copy_of_the_shared_stop_routine() { + let source = include_str!("service_control.rs"); + for needle in [ + ["const ", "MANAGED_STOP_GRACE"].concat(), + ["fn ", "terminate_recorded_service_pids"].concat(), + ["fn ", "terminate_recorded_pid"].concat(), + ["Command::new(\"task", "kill\")"].concat(), + ] { + assert!( + !source.contains(&needle), + "`{needle}` is defined in `rocm-core`, not here" + ); + } } } diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 53f9b5c41d..f752e48e3d 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -4425,7 +4425,7 @@ mod tests { // confirms every recorded process gone without touching the test // process itself. crate::service_control::stop_internal_managed_service_with(&paths, "lemonade-a-1", |_| { - crate::service_control::RecordedStopTermination::confirmed_stopped() + Vec::new() })?; let decision = resolve_managed_port_in_transaction( diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index 7aa4ab10e8..9a3e01192e 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -7,12 +7,9 @@ use rocm_core::{AppPaths, ManagedServiceRecord}; use rocm_engine_protocol::{EngineMethod, StopRequest, StopResponse}; use serde_json::Value; use serde_json::json; -#[cfg(windows)] -use std::process::{Command as ProcessCommand, Stdio}; -use std::time::Duration; pub(crate) fn stop_managed_service(paths: &AppPaths, service_id: &str) -> Result { - stop_managed_service_with(paths, service_id, terminate_recorded_pid) + stop_managed_service_with(paths, service_id, rocm_core::terminate_recorded_pid) } fn find_managed_service(paths: &AppPaths, service_id: &str) -> Result { @@ -23,8 +20,8 @@ fn find_managed_service(paths: &AppPaths, service_id: &str) -> Result Result { let mut stopping = find_managed_service(paths, service_id)?; // Record the request before the stop starts, not only after it ends. A stop - // can take up to `MANAGED_STOP_GRACE` twice per recorded PID, and for that + // can take up to `rocm_core::MANAGED_STOP_GRACE` twice per recorded PID, and for that // whole window the service still reads `ready`/`running` while its endpoint // stops answering — exactly what the daemon's `server-recover` watcher // restarts. The marker is what that watcher skips (`stop_requested`), so it @@ -73,32 +70,27 @@ pub(crate) fn stop_managed_service_via( // container-backed engine (`aim`) this is the only thing that stops the // container, since its recorded PIDs are only the supervisor that follows it. let engine_stop = (stopping.engine != "lemonade").then(|| engine_stop(paths, &stopping)); - let stops = terminate_recorded_service_pids(&stopping, terminate); - let pids_where = |keep: fn(&RecordedPidStop) -> bool| -> Vec { + let stops = rocm_core::terminate_recorded_service_pids(&stopping, terminate); + let pids_where = |keep: fn(rocm_core::RecordedPidStop) -> bool| -> Vec { stops .iter() - .filter(|stop| keep(stop)) - .map(|s| s.pid) + .copied() + .filter(|stop| keep(*stop)) + .map(|stop| stop.pid) .collect() }; - let signaled_pids = pids_where(|stop| stop.signaled); - let force_signaled_pids = pids_where(|stop| stop.forced); + let signaled_pids = pids_where(rocm_core::RecordedPidStop::signaled); + let force_signaled_pids = pids_where(rocm_core::RecordedPidStop::forced); // Everything that was deliberately not signalled, which is no longer one // kind of thing: the stopping process itself; a PID that was already gone; // a PID that is live but provably belongs to somebody else now; and a live // PID whose identity could not be read at all, which may still be the engine // and may still hold the device. Only `pid_outcomes` separates them, and // only `stopped` reports that the last of those is not a completed stop. - let skipped_pids = pids_where(|stop| !stop.signaled); + let skipped_pids = pids_where(|stop| !stop.signaled()); let pid_outcomes = stops .iter() - .map(|stop| { - json!({ - "pid": stop.pid, - "role": stop.role, - "outcome": stop.outcome, - }) - }) + .map(|stop| stop.report_entry()) .collect::>(); // Re-read the record rather than write back the copy loaded before the // stop: `write` replaces the whole file, and other writers — a supervisor @@ -113,7 +105,7 @@ pub(crate) fn stop_managed_service_via( // would each be false. let engine_stop_confirms = !rocm_core::engine_stop_decides_confirmation(&stopping.engine) || engine_stop.as_ref().is_some_and(Result::is_ok); - let all_stopped = stops.iter().all(|stop| stop.stopped) + let all_stopped = stops.iter().all(|stop| stop.stopped()) && stopping.names_same_processes(&record) && engine_stop_confirms; // A PID whose identity cannot be read at all — `/proc//stat` @@ -208,179 +200,6 @@ pub(crate) fn stop_report_confirmed(report: &Value) -> bool { report.get("stopped").and_then(Value::as_bool) == Some(true) } -/// How long a stop waits for a recorded process tree to exit on its own before -/// escalating to a forced kill. Matches `rocm services stop`, which terminates -/// the same records. -const MANAGED_STOP_GRACE: Duration = Duration::from_secs(10); - -/// What a stop did about one PID recorded in a service manifest. -struct RecordedPidStop { - pid: u32, - /// Which recorded PID this is: `supervisor`, `engine`, or - /// `supervisor_and_engine` when the record holds the same PID for both. - role: &'static str, - /// Stable label for what happened, from - /// [`rocm_core::TerminationOutcome::as_str`], plus `self` for the stopping - /// process's own PID. - outcome: &'static str, - signaled: bool, - forced: bool, - /// Whether the recorded process is confirmed to be no longer running. - stopped: bool, -} - -/// Terminate the processes a service manifest records, verifying identity first. -/// -/// PIDs are recycled, so a persisted PID is not by itself evidence that the -/// recorded process is still the one holding it. Each PID is therefore paired -/// with **its own** start-time token — `supervisor_start_ticks` with -/// `supervisor_pid`, `engine_start_ticks` with `engine_pid` — and routed through -/// [`rocm_core::terminate_verified`], which signals nothing when the recorded -/// identity is refuted. `rocm services stop` reads the same records through the -/// same primitive, so neither command can decide differently about one. -/// -/// Descendants are reached from the *verified* root rather than expanded out of -/// a live process listing keyed on a PID nobody checked — which, given a stale -/// root, enumerated a stranger's children and signalled those too. Where the -/// kernel start-time is readable, `terminate_verified` additionally binds each -/// child to its own identity before any forced kill; see -/// [`terminate_recorded_pid`] for what Windows can and cannot do here. -/// -/// The stopping process's own PID is reported but never signalled, and does not -/// count towards "the recorded service is gone": `rocmd` records itself as the -/// supervisor of the services it launches, and it is the launcher, not the -/// service. -/// -/// `terminate` performs the verified termination of one recorded PID; -/// production passes [`terminate_recorded_pid`]. -fn terminate_recorded_service_pids( - record: &ManagedServiceRecord, - terminate: impl Fn(&rocm_core::ProcessIdentity) -> rocm_core::TerminationOutcome, -) -> Vec { - // Build the (pid, role, own-token) work list, de-duplicating on PID and - // preferring an entry that carries a verifiable start-time. The two PIDs can - // coincide — a launcher that is also the server — and a token must never be - // read across from the PID it does not belong to. - let mut entries: Vec<(u32, &'static str, Option)> = Vec::new(); - for (pid, role, ticks) in [ - ( - Some(record.supervisor_pid), - "supervisor", - record.supervisor_start_ticks, - ), - (record.engine_pid, "engine", record.engine_start_ticks), - ] { - let Some(pid) = pid.filter(|pid| *pid != 0) else { - continue; - }; - if let Some(existing) = entries.iter_mut().find(|(seen, _, _)| *seen == pid) { - // Only the engine entry can land here, aliasing the supervisor: the - // one PID is both, and is reported as both. - existing.1 = "supervisor_and_engine"; - if existing.2.is_none() { - existing.2 = ticks; - } - } else { - entries.push((pid, role, ticks)); - } - } - - entries - .into_iter() - .map(|(pid, role, ticks)| { - if pid == std::process::id() { - return RecordedPidStop { - pid, - role, - outcome: "self", - signaled: false, - forced: false, - stopped: true, - }; - } - let outcome = terminate(&rocm_core::ProcessIdentity::new(pid, ticks)); - RecordedPidStop { - pid, - role, - outcome: outcome.as_str(), - // TimedOut belongs with the signalled: the signal went out, the - // exit was just never observed. `stopped` carries that truth. - signaled: matches!( - outcome, - rocm_core::TerminationOutcome::Graceful - | rocm_core::TerminationOutcome::Forced - | rocm_core::TerminationOutcome::TimedOut - ), - // "The stop escalated to a forced kill", not "the forced kill - // worked": a forced stop only reaches `TimedOut` by way of that - // escalation, and `stopped` is what says whether the process - // actually went away. - forced: matches!( - outcome, - rocm_core::TerminationOutcome::Forced | rocm_core::TerminationOutcome::TimedOut - ), - stopped: outcome.stopped(), - } - }) - .collect() -} - -/// Terminate one recorded process after verifying its identity. -/// -/// The stop is forced: its contract is definitive, so the command must not -/// report success while an engine worker still holds the GPU. Only a process -/// whose recorded identity is confirmed is ever reached by it. -fn terminate_recorded_pid(identity: &rocm_core::ProcessIdentity) -> rocm_core::TerminationOutcome { - // Windows needs the descendants taken separately. `rocm_core` has no way to - // walk a process tree there — `process_tree_pids` returns just the root — and - // the engine runs one level below the recorded PID, because `rocmd` launches - // it through an `__engine-serve-http` process. A root-only kill would leave - // the engine running and still holding the device. - // - // Identity verification does not apply on Windows. `process_start_ticks` - // has no `/proc` to read there and returns `None`, so no recorded identity - // can be refuted and the gate below only ever turns away a PID that is not - // running at all. A recorded PID that has been recycled by an unrelated - // process still passes it, and `taskkill /T /F` then takes that process and - // its whole subtree: the stale-PID hazard is unchanged on Windows, exactly - // as it was before this gate existed. Closing it needs `rocm_core` to read a - // Windows process creation time. - #[cfg(windows)] - if matches!( - rocm_core::identity_state(identity), - rocm_core::IdentityState::Matches - ) { - // Taken while the root is still alive: `taskkill /T` resolves children - // through the live parent, so it cannot reach them once the root is gone. - let _ = ProcessCommand::new("taskkill") - .args(["/PID", &identity.pid.to_string(), "/T", "/F"]) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - // `terminate_verified` still owns the bounded wait and the verdict, but - // a root the tree kill already took now reads back as `AlreadyGone` — - // which would deny a stop this command did perform. - let outcome = rocm_core::terminate_verified( - identity, - rocm_core::KillScope::Tree, - MANAGED_STOP_GRACE, - true, - ); - return if matches!(outcome, rocm_core::TerminationOutcome::AlreadyGone) { - rocm_core::TerminationOutcome::Forced - } else { - outcome - }; - } - rocm_core::terminate_verified( - identity, - rocm_core::KillScope::Tree, - MANAGED_STOP_GRACE, - true, - ) -} - #[cfg(test)] mod tests { use super::*; @@ -1081,27 +900,32 @@ mod tests { Ok(()) } - // ---- Characterisation of this module's own managed-stop routine. ---- + // ---- This module's stop, through the routine it shares with `rocm`. ---- // - // `rocm` carries a second copy of `MANAGED_STOP_GRACE` and of - // `terminate_recorded_service_pids`, and the two have diverged. These pin - // what THIS copy does today so that the change that merges them is visible - // in the tests it has to edit. The termination is a stand-in throughout, so + // The routine itself (visiting order, start-time pairing, the `self` row, + // the per-outcome classification, the Windows tree-kill step) is tested in + // `rocm-core`'s `proc_lifecycle.rs`. These check that this caller reaches it + // and reports what it returns. The termination is a stand-in throughout, so // no process is signalled and the tests behave the same on every host. const SUPERVISOR_PID: u32 = 999_999_998; const ENGINE_PID: u32 = 999_999_997; - fn record_naming( - paths: &AppPaths, + /// Plant a record naming `supervisor` and `engine`, with the given start-time + /// tokens, and return the paths it lives under. + fn plant_record_naming( + name: &str, supervisor: u32, supervisor_ticks: Option, engine: Option, engine_ticks: Option, - ) -> ManagedServiceRecord { + ) -> Result<(std::path::PathBuf, AppPaths, &'static str)> { + let (root, paths) = temp_app_paths(name); + paths.ensure()?; + let service_id = "svc-stop-through-the-shared-routine"; let mut record = ManagedServiceRecord::new( - paths, - "svc-characterise-stop", + &paths, + service_id, "vllm", "m", "m", @@ -1116,148 +940,158 @@ mod tests { record.supervisor_start_ticks = supervisor_ticks; record.engine_pid = engine; record.engine_start_ticks = engine_ticks; - record - } - - /// The identities `terminate_recorded_service_pids` handed its termination - /// function, in call order, together with what each returned as a stop. - fn terminate_with( - record: &ManagedServiceRecord, - outcome: rocm_core::TerminationOutcome, - ) -> (Vec, Vec) { - let seen = std::cell::RefCell::new(Vec::new()); - let stops = terminate_recorded_service_pids(record, |identity| { - seen.borrow_mut().push(*identity); - outcome - }); - (seen.into_inner(), stops) + record.status = "ready".to_owned(); + record.write()?; + Ok((root, paths, service_id)) } - #[test] - fn managed_stop_grace_is_ten_seconds() { - // `rocm` keeps its own copy of this value, in step only by a comment. - assert_eq!(MANAGED_STOP_GRACE, Duration::from_secs(10)); + fn pids_listed(report: &Value, key: &str) -> Vec { + report[key] + .as_array() + .unwrap_or_else(|| panic!("no `{key}` in {report}")) + .iter() + .filter_map(Value::as_u64) + .collect() } #[test] - fn managed_stop_terminates_supervisor_then_engine_each_with_its_own_start_time() { - let (root, paths) = temp_app_paths("managed-stop-order"); - let record = record_naming(&paths, SUPERVISOR_PID, Some(11), Some(ENGINE_PID), Some(22)); + fn a_stop_terminates_the_supervisor_then_the_engine_each_with_its_own_start_time() -> Result<()> + { + let (root, paths, service_id) = plant_record_naming( + "stop-order", + SUPERVISOR_PID, + Some(11), + Some(ENGINE_PID), + Some(22), + )?; + let seen = std::cell::RefCell::new(Vec::new()); - let (seen, stops) = terminate_with(&record, rocm_core::TerminationOutcome::Graceful); + let report = stop_managed_service_with(&paths, service_id, |identity| { + seen.borrow_mut().push(*identity); + rocm_core::TerminationOutcome::Graceful + }); fs::remove_dir_all(root).ok(); + let report = report?; assert_eq!( - seen, + seen.into_inner(), vec![ rocm_core::ProcessIdentity::new(SUPERVISOR_PID, Some(11)), rocm_core::ProcessIdentity::new(ENGINE_PID, Some(22)), - ], - "the supervisor first, and no start-time read across from the other PID" + ] ); assert_eq!( - stops - .iter() - .map(|stop| (stop.pid, stop.role)) - .collect::>(), - vec![(SUPERVISOR_PID, "supervisor"), (ENGINE_PID, "engine")] + report["pid_outcomes"], + json!([ + {"pid": SUPERVISOR_PID, "role": "supervisor", "outcome": "graceful"}, + {"pid": ENGINE_PID, "role": "engine", "outcome": "graceful"}, + ]) ); - } - - #[test] - fn managed_stop_terminates_a_pid_recorded_twice_once_and_keeps_a_known_start_time() { - let (root, paths) = temp_app_paths("managed-stop-aliased-ticks"); - // The supervisor entry carries no start-time, the engine entry does: - // the one PID is stopped once, with the known token. - let record = record_naming(&paths, SUPERVISOR_PID, None, Some(SUPERVISOR_PID), Some(33)); - - let (seen, stops) = terminate_with(&record, rocm_core::TerminationOutcome::Graceful); - fs::remove_dir_all(root).ok(); - assert_eq!( - seen, - vec![rocm_core::ProcessIdentity::new(SUPERVISOR_PID, Some(33))] + pids_listed(&report, "signaled_pids"), + vec![u64::from(SUPERVISOR_PID), u64::from(ENGINE_PID)] ); - assert_eq!(stops.len(), 1); - assert_eq!(stops[0].role, "supervisor_and_engine"); - } - - #[test] - fn managed_stop_skips_a_zero_pid_and_an_absent_engine() { - let (root, paths) = temp_app_paths("managed-stop-zero-pid"); - let record = record_naming(&paths, 0, None, Some(0), None); - - let (seen, stops) = terminate_with(&record, rocm_core::TerminationOutcome::Graceful); - fs::remove_dir_all(root).ok(); - - assert!(seen.is_empty()); - assert!(stops.is_empty()); + Ok(()) } - /// DIVERGENCE from `rocm`: this copy reports the stopping process's own PID, - /// as `self`, where `rocm` drops it. It is never handed to the termination - /// function, and it counts as stopped. + /// The stopping process's own PID is reported as `self`, never handed to the + /// termination, and does not stand in the way of a confirmed stop. #[test] - fn managed_stop_reports_its_own_pid_as_self_and_never_terminates_it() { - let (root, paths) = temp_app_paths("managed-stop-own-pid"); - let record = record_naming(&paths, std::process::id(), None, Some(ENGINE_PID), None); + fn a_stop_reports_its_own_pid_as_self_and_skips_it() -> Result<()> { + let own = std::process::id(); + let (root, paths, service_id) = + plant_record_naming("stop-own-pid", own, None, Some(ENGINE_PID), None)?; + let seen = std::cell::RefCell::new(Vec::new()); - let (seen, stops) = terminate_with(&record, rocm_core::TerminationOutcome::Graceful); + let report = stop_managed_service_with(&paths, service_id, |identity| { + seen.borrow_mut().push(identity.pid); + rocm_core::TerminationOutcome::Graceful + }); fs::remove_dir_all(root).ok(); + let report = report?; + assert_eq!(seen.into_inner(), vec![ENGINE_PID]); assert_eq!( - seen, - vec![rocm_core::ProcessIdentity::new(ENGINE_PID, None)], - "the stopping process is never signalled" + report["pid_outcomes"][0], + json!({"pid": own, "role": "supervisor", "outcome": "self"}) ); - assert_eq!(stops.len(), 2); - assert_eq!(stops[0].pid, std::process::id()); - assert_eq!(stops[0].role, "supervisor"); - assert_eq!(stops[0].outcome, "self"); - assert!(!stops[0].signaled && !stops[0].forced); - assert!(stops[0].stopped, "the launcher is not the service"); + assert_eq!(pids_listed(&report, "skipped_pids"), vec![u64::from(own)]); + assert_eq!( + pids_listed(&report, "signaled_pids"), + vec![u64::from(ENGINE_PID)] + ); + assert_eq!(report["stopped"], true); + Ok(()) } #[test] - fn managed_stop_classifies_each_outcome_as_signaled_forced_and_stopped() { + fn a_stop_sorts_each_outcome_into_the_signaled_forced_and_skipped_lists() -> Result<()> { use rocm_core::TerminationOutcome::{ AlreadyGone, Forced, Graceful, IdentityMismatch, TimedOut, Unverified, }; - let (root, paths) = temp_app_paths("managed-stop-classes"); - let record = record_naming(&paths, SUPERVISOR_PID, None, None, None); - - // (outcome, signaled, forced, stopped) + // (outcome, signaled, forced, skipped, stopped) let expected = [ - (AlreadyGone, false, false, true), - (IdentityMismatch, false, false, true), - (Unverified, false, false, false), - (Graceful, true, false, true), - (Forced, true, true, true), - (TimedOut, true, true, false), + (AlreadyGone, false, false, true, true), + (IdentityMismatch, false, false, true, true), + (Unverified, false, false, true, false), + (Graceful, true, false, false, true), + (Forced, true, true, false, true), + (TimedOut, true, true, false, false), ]; - for (outcome, signaled, forced, stopped) in expected { - let (_, stops) = terminate_with(&record, outcome); - assert_eq!(stops.len(), 1); - let stop = &stops[0]; - assert_eq!(stop.outcome, outcome.as_str()); + for (outcome, signaled, forced, skipped, stopped) in expected { + let (root, paths, service_id) = + plant_record_naming("stop-classes", SUPERVISOR_PID, None, None, None)?; + + let report = stop_managed_service_with(&paths, service_id, |_| outcome); + fs::remove_dir_all(root).ok(); + + let report = report?; + let listed = |key| !pids_listed(&report, key).is_empty(); assert_eq!( - (stop.signaled, stop.forced, stop.stopped), - (signaled, forced, stopped), - "{outcome:?}" + ( + listed("signaled_pids"), + listed("force_signaled_pids"), + listed("skipped_pids"), + report["stopped"].as_bool() + ), + (signaled, forced, skipped, Some(stopped)), + "{outcome:?}: {report}" ); } - fs::remove_dir_all(root).ok(); + Ok(()) } - /// The tree-kill step is Windows-only and is a private `cfg(windows)` block - /// of `terminate_recorded_pid` with no injection point, so nothing here can - /// observe it. What this does pin is the portable half: a recorded PID that - /// is not running reaches `terminate_verified` and comes back unchanged. #[test] - fn terminate_recorded_pid_reports_an_absent_pid_as_already_gone() { - let outcome = - terminate_recorded_pid(&rocm_core::ProcessIdentity::new(SUPERVISOR_PID, None)); + fn the_production_stop_reports_an_absent_pid_as_already_gone() -> Result<()> { + let (root, paths, service_id) = + plant_record_naming("stop-production-absent", SUPERVISOR_PID, None, None, None)?; - assert_eq!(outcome, rocm_core::TerminationOutcome::AlreadyGone); + let report = stop_managed_service(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let report = report?; + assert_eq!(report["pid_outcomes"][0]["outcome"], "already_gone"); + assert_eq!(report["stopped"], true); + Ok(()) + } + + /// A second copy of the shared routine, or of its grace constant, is how the + /// two binaries diverged in the first place; the shared one lives in + /// `rocm-core`'s `proc_lifecycle.rs`. The needles are assembled so that this + /// test does not match itself. + #[test] + fn this_module_carries_no_private_copy_of_the_shared_stop_routine() { + let source = include_str!("service.rs"); + for needle in [ + ["const ", "MANAGED_STOP_GRACE"].concat(), + ["fn ", "terminate_recorded_service_pids"].concat(), + ["fn ", "terminate_recorded_pid"].concat(), + ["Command::new(\"task", "kill\")"].concat(), + ] { + assert!( + !source.contains(&needle), + "`{needle}` is defined in `rocm-core`, not here" + ); + } } } diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index aad2dda627..0884c70fb2 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -169,8 +169,9 @@ pub use net::{ }; pub use paths::{AppPaths, engine_plugin_dirs}; pub use proc_lifecycle::{ - IdentityState, KillScope, ProcessIdentity, TerminationOutcome, identity_state, - process_start_ticks, terminate_verified, + IdentityState, KillScope, MANAGED_STOP_GRACE, ProcessIdentity, RecordedPidOutcome, + RecordedPidStop, TerminationOutcome, identity_state, process_start_ticks, + terminate_recorded_pid, terminate_recorded_service_pids, terminate_verified, }; pub use process::{ FileLock, LoopbackPortLease, MANAGED_LAUNCH_LOCK_TIMEOUT, detach_command_session, diff --git a/crates/rocm-core/src/proc_lifecycle.rs b/crates/rocm-core/src/proc_lifecycle.rs index 57da4bb1fe..bfa14c1989 100644 --- a/crates/rocm-core/src/proc_lifecycle.rs +++ b/crates/rocm-core/src/proc_lifecycle.rs @@ -261,6 +261,245 @@ pub fn terminate_verified( } } +/// How long a managed stop waits on a recorded process before it escalates. +/// +/// Each recorded process, tree included, gets this long to exit after `SIGTERM` +/// before the stop escalates to `SIGKILL`, and then this long again before it +/// reports a timeout. The one value behind `rocm services stop` and `rocmd`'s +/// `stop_server`: both stop the same records, so neither may wait for a +/// different time. +pub const MANAGED_STOP_GRACE: Duration = Duration::from_secs(10); + +/// What a managed stop did about one PID a service record names. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RecordedPidOutcome { + /// The PID is the stopping process's own. It is reported but never + /// signalled, and it counts as stopped: `rocmd` and an attached `rocm + /// serve` record themselves as the supervisor of the service they launched, + /// and the launcher is not the service. + OwnProcess, + /// The PID was handed to the verified termination, with this verdict. + Terminated(TerminationOutcome), +} + +impl RecordedPidOutcome { + /// The label a stop report carries: `self` for the stopping process's own + /// PID, otherwise [`TerminationOutcome::as_str`]. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::OwnProcess => "self", + Self::Terminated(outcome) => outcome.as_str(), + } + } +} + +/// One recorded PID's fate in a managed stop. +/// +/// Kept per PID because the outcomes that leave a stop unconfirmed are reported +/// differently: a `timed_out` PID was signalled and it, or a process under it, +/// outlived the deadline, while an `unverified` one was not signalled at all. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RecordedPidStop { + pub pid: u32, + /// Which recorded PID this is: `supervisor`, `engine`, or + /// `supervisor_and_engine` when the record holds the same PID for both. + pub role: &'static str, + pub outcome: RecordedPidOutcome, +} + +impl RecordedPidStop { + /// Whether a signal went out to this PID. A `timed_out` PID counts: the + /// signal was sent and only the exit was never observed, which + /// [`stopped`](Self::stopped) reports separately. + #[must_use] + pub const fn signaled(self) -> bool { + matches!( + self.outcome, + RecordedPidOutcome::Terminated( + TerminationOutcome::Graceful + | TerminationOutcome::Forced + | TerminationOutcome::TimedOut + ) + ) + } + + /// Whether the stop escalated to a forced kill for this PID. That is not + /// "the forced kill worked": a forced stop only reaches `timed_out` by way + /// of the escalation, and [`stopped`](Self::stopped) says whether the + /// process actually went away. + #[must_use] + pub const fn forced(self) -> bool { + matches!( + self.outcome, + RecordedPidOutcome::Terminated( + TerminationOutcome::Forced | TerminationOutcome::TimedOut + ) + ) + } + + /// Whether the recorded process is confirmed no longer running. + #[must_use] + pub const fn stopped(self) -> bool { + match self.outcome { + RecordedPidOutcome::OwnProcess => true, + RecordedPidOutcome::Terminated(outcome) => outcome.stopped(), + } + } + + /// The entry a stop report lists for this PID under `pid_outcomes`. + #[must_use] + pub fn report_entry(self) -> serde_json::Value { + serde_json::json!({ + "pid": self.pid, + "role": self.role, + "outcome": self.outcome.as_str(), + }) + } +} + +/// Terminate the processes a service record names, verifying each PID's +/// identity first, and report what happened to each. +/// +/// PIDs are recycled, so a persisted PID is not by itself evidence that the +/// recorded process still holds it. Each PID is paired with **its own** +/// start-time token (`supervisor_start_ticks` with `supervisor_pid`, +/// `engine_start_ticks` with `engine_pid`) and handed to `terminate`, which +/// production supplies as [`terminate_recorded_pid`]. When the two PIDs +/// coincide, the entry is reported once, under the role `supervisor_and_engine`, +/// and keeps a known token in preference to a missing one: a token is never read +/// across from the PID it does not belong to. The supervisor is visited first. +/// +/// A zero PID names nothing and is skipped. The stopping process's own PID is +/// reported as [`RecordedPidOutcome::OwnProcess`] and `terminate` is not called +/// for it. +/// +/// `terminate` is injectable so a caller's test can supply the verdicts it +/// cannot construct on demand (a process that survives `SIGKILL`, one whose +/// identity cannot be read). +pub fn terminate_recorded_service_pids( + record: &crate::ManagedServiceRecord, + terminate: impl Fn(&ProcessIdentity) -> TerminationOutcome, +) -> Vec { + let mut entries: Vec<(u32, &'static str, Option)> = Vec::new(); + for (pid, role, ticks) in [ + ( + Some(record.supervisor_pid), + "supervisor", + record.supervisor_start_ticks, + ), + (record.engine_pid, "engine", record.engine_start_ticks), + ] { + let Some(pid) = pid.filter(|pid| *pid != 0) else { + continue; + }; + if let Some(existing) = entries.iter_mut().find(|(seen, _, _)| *seen == pid) { + // Only the engine entry can land here, aliasing the supervisor: the + // one PID is both, and is reported as both. + existing.1 = "supervisor_and_engine"; + if existing.2.is_none() { + existing.2 = ticks; + } + } else { + entries.push((pid, role, ticks)); + } + } + + entries + .into_iter() + .map(|(pid, role, ticks)| { + let outcome = if pid == std::process::id() { + RecordedPidOutcome::OwnProcess + } else { + RecordedPidOutcome::Terminated(terminate(&ProcessIdentity::new(pid, ticks))) + }; + RecordedPidStop { pid, role, outcome } + }) + .collect() +} + +/// Terminate one recorded process, and its descendants, after verifying its +/// identity. The production `terminate` of [`terminate_recorded_service_pids`]. +/// +/// The stop is forced: a managed stop is definitive, so it must not report +/// success while an engine worker still holds the GPU. Only a process whose +/// recorded identity is confirmed is ever reached by it. +/// +/// On Windows the descendants are taken separately, first. [`terminate_verified`] +/// cannot walk a process tree there (`process_tree_pids` returns just the root), +/// and the engine runs one level below the recorded PID, because the daemon +/// launches it through an `__engine-serve-http` process: a root-only kill would +/// leave the engine running and still holding the device. +/// +/// Identity verification does not reach that far on Windows. +/// [`process_start_ticks`] has no `/proc` to read there and returns `None`, so +/// no recorded identity can be refuted and the [`IdentityState::Matches`] gate +/// only ever turns away a PID that is not running at all. A recorded PID that +/// has been recycled by an unrelated process still passes it, and `taskkill /T +/// /F` then takes that process and its whole subtree. Closing that needs a +/// Windows process creation time in [`process_start_ticks`]. +pub fn terminate_recorded_pid(identity: &ProcessIdentity) -> TerminationOutcome { + terminate_recorded_pid_with(identity, platform_tree_kill(), identity_state, |identity| { + terminate_verified(identity, KillScope::Tree, MANAGED_STOP_GRACE, true) + }) +} + +/// [`terminate_recorded_pid`] with its three platform and system touchpoints +/// supplied by the caller, so a test drives the tree-kill step on every host: +/// `tree_kill` takes one PID's descendants down (`None` where the platform +/// needs no separate step), `state_of` reads the PID's identity, and `terminate` +/// is the verified termination of the root. +fn terminate_recorded_pid_with( + identity: &ProcessIdentity, + tree_kill: Option<&dyn Fn(u32)>, + state_of: impl Fn(&ProcessIdentity) -> IdentityState, + terminate: impl Fn(&ProcessIdentity) -> TerminationOutcome, +) -> TerminationOutcome { + if let Some(tree_kill) = tree_kill + && matches!(state_of(identity), IdentityState::Matches) + { + // Taken while the root is still alive: `taskkill /T` resolves children + // through the live parent, so it cannot reach them once the root is gone. + tree_kill(identity.pid); + // `terminate` still owns the bounded wait and the verdict, but a root + // the tree kill already took now reads back as `AlreadyGone` — which + // would deny a stop this command did perform. + return match terminate(identity) { + TerminationOutcome::AlreadyGone => TerminationOutcome::Forced, + outcome => outcome, + }; + } + terminate(identity) +} + +/// The separate descendant kill this platform needs before the verified +/// termination of a root, if any: `taskkill /T /F` on Windows, none elsewhere, +/// where [`terminate_verified`] walks the tree itself. +// The `cfg` arms differ in what they return, so the non-Windows arm cannot be a +// `const fn` without giving the two platforms different signatures. +#[allow(clippy::missing_const_for_fn)] +fn platform_tree_kill() -> Option<&'static dyn Fn(u32)> { + #[cfg(windows)] + { + Some(&taskkill_tree) + } + #[cfg(not(windows))] + { + None + } +} + +#[cfg(windows)] +fn taskkill_tree(pid: u32) { + use std::process::{Command, Stdio}; + let _ = Command::new("taskkill") + .args(["/PID", &pid.to_string(), "/T", "/F"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status(); +} + /// Poll until every process in `members` has exited, or `grace` elapses. /// /// A member is exited once its PID is gone or now belongs to a different process @@ -614,4 +853,329 @@ mod tests { ); reap(child); } + + // ---- The managed-stop routine `rocm` and `rocmd` share. ---- + + use std::cell::RefCell; + + /// A PID no host has in use. + const SUPERVISOR_PID: u32 = 999_999_998; + const ENGINE_PID: u32 = 999_999_997; + + fn record_naming( + supervisor: u32, + supervisor_ticks: Option, + engine: Option, + engine_ticks: Option, + ) -> crate::ManagedServiceRecord { + let paths = crate::AppPaths { + config_dir: std::path::PathBuf::from("config"), + data_dir: std::path::PathBuf::from("data"), + cache_dir: std::path::PathBuf::from("cache"), + }; + let mut record = crate::ManagedServiceRecord::new( + &paths, + "svc-managed-stop", + "vllm", + "m", + "m", + "127.0.0.1", + 9, + "managed", + supervisor, + None, + None, + None, + ); + record.supervisor_start_ticks = supervisor_ticks; + record.engine_pid = engine; + record.engine_start_ticks = engine_ticks; + record + } + + /// The identities `terminate_recorded_service_pids` handed its termination + /// function, in call order, and the stops it reported. + fn stop_with( + record: &crate::ManagedServiceRecord, + outcome: TerminationOutcome, + ) -> (Vec, Vec) { + let seen = RefCell::new(Vec::new()); + let stops = terminate_recorded_service_pids(record, |identity| { + seen.borrow_mut().push(*identity); + outcome + }); + (seen.into_inner(), stops) + } + + #[test] + fn managed_stop_grace_is_ten_seconds() { + assert_eq!(MANAGED_STOP_GRACE, Duration::from_secs(10)); + } + + #[test] + fn managed_stop_terminates_supervisor_then_engine_each_with_its_own_start_time() { + let record = record_naming(SUPERVISOR_PID, Some(11), Some(ENGINE_PID), Some(22)); + + let (seen, stops) = stop_with(&record, TerminationOutcome::Graceful); + + assert_eq!( + seen, + vec![ + ProcessIdentity::new(SUPERVISOR_PID, Some(11)), + ProcessIdentity::new(ENGINE_PID, Some(22)), + ], + "the supervisor first, and no start-time read across from the other PID" + ); + assert_eq!( + stops + .iter() + .map(|stop| (stop.pid, stop.role)) + .collect::>(), + vec![(SUPERVISOR_PID, "supervisor"), (ENGINE_PID, "engine")] + ); + } + + #[test] + fn managed_stop_terminates_a_pid_recorded_twice_once_and_keeps_a_known_start_time() { + // Either entry may be the one that carries the token. + for (supervisor_ticks, engine_ticks, expected) in [ + (None, Some(33), Some(33)), + (Some(44), None, Some(44)), + (Some(44), Some(33), Some(44)), + (None, None, None), + ] { + let record = record_naming( + SUPERVISOR_PID, + supervisor_ticks, + Some(SUPERVISOR_PID), + engine_ticks, + ); + + let (seen, stops) = stop_with(&record, TerminationOutcome::Graceful); + + assert_eq!( + seen, + vec![ProcessIdentity::new(SUPERVISOR_PID, expected)], + "{supervisor_ticks:?} / {engine_ticks:?}" + ); + assert_eq!(stops.len(), 1); + assert_eq!(stops[0].role, "supervisor_and_engine"); + } + } + + #[test] + fn managed_stop_skips_a_zero_pid_and_an_absent_engine() { + let record = record_naming(0, None, Some(0), None); + + let (seen, stops) = stop_with(&record, TerminationOutcome::Graceful); + + assert!(seen.is_empty()); + assert!(stops.is_empty()); + } + + #[test] + fn managed_stop_reports_its_own_pid_as_self_and_never_terminates_it() { + let own = std::process::id(); + let record = record_naming(own, None, Some(ENGINE_PID), None); + + let (seen, stops) = stop_with(&record, TerminationOutcome::Graceful); + + assert_eq!( + seen, + vec![ProcessIdentity::new(ENGINE_PID, None)], + "the stopping process is never signalled" + ); + assert_eq!(stops.len(), 2); + assert_eq!(stops[0].pid, own); + assert_eq!(stops[0].role, "supervisor"); + assert_eq!(stops[0].outcome, RecordedPidOutcome::OwnProcess); + assert_eq!(stops[0].outcome.as_str(), "self"); + assert!(!stops[0].signaled() && !stops[0].forced()); + assert!(stops[0].stopped(), "the launcher is not the service"); + assert_eq!( + stops[0].report_entry(), + serde_json::json!({"pid": own, "role": "supervisor", "outcome": "self"}) + ); + } + + #[test] + fn managed_stop_reports_a_pid_that_is_both_roles_and_its_own_as_self() { + let own = std::process::id(); + let record = record_naming(own, None, Some(own), None); + + let (seen, stops) = stop_with(&record, TerminationOutcome::Graceful); + + assert!(seen.is_empty()); + assert_eq!(stops.len(), 1); + assert_eq!(stops[0].role, "supervisor_and_engine"); + assert_eq!(stops[0].outcome, RecordedPidOutcome::OwnProcess); + } + + #[test] + fn managed_stop_classifies_each_outcome_as_signaled_forced_and_stopped() { + use TerminationOutcome::{ + AlreadyGone, Forced, Graceful, IdentityMismatch, TimedOut, Unverified, + }; + let record = record_naming(SUPERVISOR_PID, None, None, None); + + // (outcome, signaled, forced, stopped) + let expected = [ + (AlreadyGone, false, false, true), + (IdentityMismatch, false, false, true), + (Unverified, false, false, false), + (Graceful, true, false, true), + (Forced, true, true, true), + (TimedOut, true, true, false), + ]; + for (outcome, signaled, forced, stopped) in expected { + let (_, stops) = stop_with(&record, outcome); + assert_eq!(stops.len(), 1); + let stop = stops[0]; + assert_eq!(stop.outcome, RecordedPidOutcome::Terminated(outcome)); + assert_eq!(stop.outcome.as_str(), outcome.as_str()); + assert_eq!( + (stop.signaled(), stop.forced(), stop.stopped()), + (signaled, forced, stopped), + "{outcome:?}" + ); + } + } + + /// What one `terminate_recorded_pid_with` call did, in order. + #[derive(Debug, PartialEq, Eq)] + enum Step { + TreeKill(u32), + Terminate(u32), + } + + /// Run `terminate_recorded_pid_with` for `pid`, with a tree-kill step + /// present or not, the identity in `state`, and the root's termination + /// answering `outcome`. Returns the verdict and the steps taken. + fn tree_kill_run( + has_tree_kill: bool, + state: IdentityState, + outcome: TerminationOutcome, + ) -> (TerminationOutcome, Vec) { + let steps = RefCell::new(Vec::new()); + let tree_kill = |pid: u32| steps.borrow_mut().push(Step::TreeKill(pid)); + let verdict = terminate_recorded_pid_with( + &ProcessIdentity::new(SUPERVISOR_PID, None), + has_tree_kill.then_some(&tree_kill as &dyn Fn(u32)), + |_| state, + |identity| { + steps.borrow_mut().push(Step::Terminate(identity.pid)); + outcome + }, + ); + (verdict, steps.into_inner()) + } + + #[test] + fn the_tree_is_killed_before_the_root_is_terminated() { + let (_, steps) = tree_kill_run(true, IdentityState::Matches, TerminationOutcome::Forced); + + assert_eq!( + steps, + vec![ + Step::TreeKill(SUPERVISOR_PID), + Step::Terminate(SUPERVISOR_PID) + ], + "`taskkill /T` resolves children through the live parent, so the \ + tree kill has to come first" + ); + } + + #[test] + fn a_root_the_tree_kill_already_took_is_reported_forced_not_already_gone() { + let (verdict, _) = tree_kill_run( + true, + IdentityState::Matches, + TerminationOutcome::AlreadyGone, + ); + + assert_eq!( + verdict, + TerminationOutcome::Forced, + "the stop did perform the kill; reading the root back as gone must not deny it" + ); + } + + #[test] + fn the_verdict_after_a_tree_kill_is_the_terminations_own_for_every_other_outcome() { + use TerminationOutcome::{Forced, Graceful, IdentityMismatch, TimedOut, Unverified}; + for outcome in [Graceful, Forced, TimedOut, Unverified, IdentityMismatch] { + let (verdict, _) = tree_kill_run(true, IdentityState::Matches, outcome); + + assert_eq!(verdict, outcome); + } + } + + #[test] + fn no_tree_kill_is_run_for_a_pid_that_is_gone_recycled_or_unverifiable() { + for state in [ + IdentityState::Gone, + IdentityState::Recycled, + IdentityState::Indeterminate, + ] { + let (verdict, steps) = tree_kill_run(true, state, TerminationOutcome::AlreadyGone); + + assert_eq!( + steps, + vec![Step::Terminate(SUPERVISOR_PID)], + "{state:?}: a stranger's process tree must not be taken" + ); + assert_eq!( + verdict, + TerminationOutcome::AlreadyGone, + "{state:?}: and the verdict is the termination's, unmapped" + ); + } + } + + #[test] + fn without_a_tree_kill_step_the_root_is_terminated_and_its_verdict_unmapped() { + let (verdict, steps) = tree_kill_run( + false, + IdentityState::Matches, + TerminationOutcome::AlreadyGone, + ); + + assert_eq!(steps, vec![Step::Terminate(SUPERVISOR_PID)]); + assert_eq!(verdict, TerminationOutcome::AlreadyGone); + } + + #[test] + fn only_windows_has_a_separate_tree_kill_step() { + // Ties the injected tests above to what production actually runs: the + // step they exercise is the one `terminate_recorded_pid` has on Windows + // and does without elsewhere, where `terminate_verified` walks the tree. + assert_eq!(platform_tree_kill().is_some(), cfg!(windows)); + } + + #[test] + fn terminate_recorded_pid_reports_an_absent_pid_as_already_gone() { + let outcome = terminate_recorded_pid(&ProcessIdentity::new(SUPERVISOR_PID, None)); + + assert_eq!(outcome, TerminationOutcome::AlreadyGone); + } + + #[cfg(unix)] + #[test] + fn terminate_recorded_pid_stops_a_running_verified_process() { + let child = Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn child"); + let id = ProcessIdentity::capture(child.id()); + + let outcome = terminate_recorded_pid(&id); + + assert!(outcome.stopped(), "{outcome:?}"); + assert!(matches!( + outcome, + TerminationOutcome::Graceful | TerminationOutcome::Forced + )); + reap(child); + assert!(!crate::process_is_running(id.pid)); + } } diff --git a/docs/architecture.md b/docs/architecture.md index ad51f2aaf6..823630e6d3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -90,7 +90,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) — mechanical relocation, owns types (Phase 6a): `serve()`, plus the engine selection that runs ahead of it (`select_serve_engine`, `recipe_preferred_engine`, `serve_model_ref_for_engine`) and the AIM pre-flight that runs inside it: the Hugging Face token check, the GPU-family profile check and pin, the multi-GPU refusal (`aim_gpu_preflight`), and the board note. The engine-recipe overrides `serve()` applies to the launch hint live in [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs); this module imports them by name. It also holds `collect_serve_notes`, the GPU/device warnings folded into the interactive summary, which was relocated here from `main.rs` with its two tests. Its tests cover engine selection, the AIM pre-flight and the reuse pre-gate; the AIM tests build their hint with `protocol_engine_recipe_hint` and `engine_recipe_with_extra_env` from `serve_recipe.rs`, which `serve()` also calls. - [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs) — mechanical relocation, owns a type (split out of [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), which imports the entry points by name): the pure transforms `serve()` applies to the selected engine's launch recipe hint. `protocol_engine_recipe_hint` maps the catalog's engine record onto the protocol hint, and `render_serve_engine_recipe_lines` prints it for the serve plan. The explicit controls are then layered on top: `--tool-call-parser` (`engine_recipe_with_tool_call_override`), `--gpu-memory-utilization` (`parse_gpu_memory_utilization`, `engine_recipe_with_gpu_memory_utilization_override`), `--temperature`/`--top-p`/`--max-tokens` (`ServeGenerationDefaults`, `engine_recipe_with_generation_defaults`) and `--engine-env` (`engine_recipe_with_extra_env`, `engine_env_ignored_note`, with the env names it refuses to override in `RESERVED_ENGINE_ENV_KEYS`); `engine_recipe_enables_tool_choice` reads the result back. The per-flag rewriters (`set_vllm_tool_call_parser`, `set_vllm_gpu_memory_utilization`, `set_vllm_override_generation_config`, `set_lemonade_generation_defaults`) are private. The entry points above, and `ServeGenerationDefaults`, are `pub(crate)` for `serve()` and for the AIM tests in `serve_cmd.rs`; `RESERVED_ENGINE_ENV_KEYS` is private. Among first-party crates the module depends only on `rocm_core` and `rocm_engine_protocol`, and nothing in it calls back into `serve_cmd.rs`. Its tests cover each override on its own. - [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs) — full domain extraction. -- [`service_control.rs`](../apps/rocm/src/service_control.rs) — mechanical relocation, owns private types: stopping and restarting a managed local server, the process control behind `rocm services stop|restart`, the chat/LLM sandbox tool-calling path and `runtime_services.rs`'s moves onto a new runtime. It holds the sandbox-tool runner (`run_internal_sandbox_tool`), the stop path (`terminate_recorded_service_pids`, the `MANAGED_STOP_GRACE` constant, the `RecordedPidOutcome` and `RecordedStopTermination` types, `scripted_stop_outcome`, the end-to-end hook behind the `e2e-test-hooks` feature, `stop_internal_managed_service`, `stop_internal_managed_service_with`, `stop_internal_managed_service_confirmed_with`, `unload_lemonade_service_model`) and the restart path (`apply_restart_runtime_pin`, `load_service_for_restart`, `restart_child_argv`, `restart_internal_managed_service`, `restart_internal_managed_service_with`, `update_restarting_record`, `record_restarted_child`, `record_restart_readiness`). Only `run_internal_sandbox_tool` and `restart_internal_managed_service` are `pub(crate)`: the first is read from `dispatch()` in [`main.rs`](../apps/rocm/src/main.rs), from [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs)'s `run_approved_service_action` and from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), which passes it to [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s `run_attached_service` as the stop for a Ctrl-C, the second from [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs); each importer reaches them through `crate::service_control` rather than a re-export. Every other item has no caller outside this module and stays private — see [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs)'s and [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s entries above for how those two modules' own doc comments describe this module as the caller of their helpers. The module reads managed-service records through [`services_text.rs`](../apps/rocm/src/services_text.rs) (`load_managed_service`, `read_managed_service_manifest`) and shapes the `stop_server` answer with its `internal_stop_server_value`; `services_text.rs` does not call back into it. This module takes `ensure_public_service_has_endpoint_key` and the spawn helpers from [`service_launch.rs`](../apps/rocm/src/service_launch.rs), which does not import it back. The cluster's regression tests live in this module's own `mod tests`, reaching shared fixtures (`test_paths`, `ScopedTestEnv`, `plant_service_record_on_runtime`, `OLD_RUNTIME_KEY`, `NEW_RUNTIME_KEY`) through `crate::tests::` the same way other mechanically relocated modules do, and `runtime_services::` through its own `use crate::runtime_services;` for the handful of tests that drive a restart through `runtime_services`'s public entry points rather than calling it directly. +- [`service_control.rs`](../apps/rocm/src/service_control.rs) — mechanical relocation, owns private types: stopping and restarting a managed local server, the process control behind `rocm services stop|restart`, the chat/LLM sandbox tool-calling path and `runtime_services.rs`'s moves onto a new runtime. It holds the sandbox-tool runner (`run_internal_sandbox_tool`), the stop path (`stop_recorded_processes`, which hands the recorded PIDs to the routine [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) shares with `rocmd`, `scripted_stop_outcome`, the end-to-end hook behind the `e2e-test-hooks` feature, `stop_internal_managed_service`, `stop_internal_managed_service_with`, `stop_internal_managed_service_confirmed_with`, `unload_lemonade_service_model`) and the restart path (`apply_restart_runtime_pin`, `load_service_for_restart`, `restart_child_argv`, `restart_internal_managed_service`, `restart_internal_managed_service_with`, `update_restarting_record`, `record_restarted_child`, `record_restart_readiness`). Only `run_internal_sandbox_tool` and `restart_internal_managed_service` are `pub(crate)`: the first is read from `dispatch()` in [`main.rs`](../apps/rocm/src/main.rs), from [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs)'s `run_approved_service_action` and from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), which passes it to [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s `run_attached_service` as the stop for a Ctrl-C, the second from [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs); each importer reaches them through `crate::service_control` rather than a re-export. Every other item has no caller outside this module and stays private — see [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs)'s and [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s entries above for how those two modules' own doc comments describe this module as the caller of their helpers. The module reads managed-service records through [`services_text.rs`](../apps/rocm/src/services_text.rs) (`load_managed_service`, `read_managed_service_manifest`) and shapes the `stop_server` answer with its `internal_stop_server_value`; `services_text.rs` does not call back into it. This module takes `ensure_public_service_has_endpoint_key` and the spawn helpers from [`service_launch.rs`](../apps/rocm/src/service_launch.rs), which does not import it back. The cluster's regression tests live in this module's own `mod tests`, reaching shared fixtures (`test_paths`, `ScopedTestEnv`, `plant_service_record_on_runtime`, `OLD_RUNTIME_KEY`, `NEW_RUNTIME_KEY`) through `crate::tests::` the same way other mechanically relocated modules do, and `runtime_services::` through its own `use crate::runtime_services;` for the handful of tests that drive a restart through `runtime_services`'s public entry points rather than calling it directly. - [`serve_port.rs`](../apps/rocm/src/serve_port.rs) — full domain extraction: the central `rocm serve` port policy (automatic vs explicit request, reservation by live managed services, OS preflight lease), used by [`service_launch.rs`](../apps/rocm/src/service_launch.rs); the CLI restart path shares the `rocm-core` lease primitives (`lease_loopback_port_or_bind_error`, `is_canonical_loopback_host`) rather than this module. - [`service_hints.rs`](../apps/rocm/src/service_hints.rs) — shared CLI-output component: the `rocm services stop|restart|remove --yes` hint text, shared by [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs), [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs), [`service_launch.rs`](../apps/rocm/src/service_launch.rs), [`service_records.rs`](../apps/rocm/src/service_records.rs), [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs), and [`services_text.rs`](../apps/rocm/src/services_text.rs) so they cannot go out of sync with each other, as `serve_summary.rs`'s stop row did from the day it was added — #417. It also holds the helpers that map a service tool name to the command word, verb and past tense those hints and messages use (`service_action_command`, `service_action_verb`, `service_action_past_tense`) and to its `SandboxToolArg` (`sandbox_tool_arg_from_service_tool`); [`services_text.rs`](../apps/rocm/src/services_text.rs) reads `service_action_past_tense` from here rather than through the crate root. It quotes the service id with `shell_quote`, so a hint names exactly the argv it runs. - [`service_launch.rs`](../apps/rocm/src/service_launch.rs) — mechanical relocation, owns types (Phase 6b, ROCMAI-91): spawning/attaching to a managed service process and streaming its log, the port allocation transaction and failed-launch cleanup (under the shared launch lock), plus the endpoint-key/auth checks that gate a public bind — `AttachOutcome`/`RawModeGuard`/`ManagedSpawn`/`AutostartClaim` are its private types; `ManagedLaunchReport` is `pub(crate)`, since [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) reads its fields directly for the interactive summary. Most of this module's functions are `pub(crate)` for the same reason — other modules, including [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`automations.rs`](../apps/rocm/src/automations.rs) and [`service_control.rs`](../apps/rocm/src/service_control.rs), call into it; `main.rs` does not. `ServicesCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs) rather than moving alongside, and `services()` lives in [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs); the record removal and pruning it dispatches to now lives in [`service_records.rs`](../apps/rocm/src/service_records.rs), and only stop/restart are also reachable from the chat/LLM sandbox tool-calling path, out of scope for Phase 6b. This module does not import [`service_control.rs`](../apps/rocm/src/service_control.rs), which imports it: `run_attached_service` takes the stop for a Ctrl-C as a parameter, and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) passes `run_internal_sandbox_tool`. @@ -152,7 +152,7 @@ Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): - [`persistence.rs`](../apps/rocmd/src/persistence.rs) — `record_event`/`load_managed_services`/`load_service_record`, the automation-event/audit-log and managed-service-registry I/O shared across the daemon's sandbox, MCP, service-lifecycle, and watcher code. - [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) — bubblewrap/native sandbox execution, atomic-write helpers, artifact prefetch policy gating, and the sandbox-tool result shaping for `check_updates`/`driver_plan`/`stop_server`. - [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs) — the sandbox-tool vocabulary: the `SandboxToolArg` value enum naming each sandbox tool, and `SandboxToolPolicy`, the artifact-download policy built from the `--allow-*` flags (including the Hugging Face token read from the environment when authenticated downloads are allowed). It uses no other module of the crate, so modules that need the vocabulary do not import [`cli.rs`](../apps/rocmd/src/cli.rs). -- [`service.rs`](../apps/rocmd/src/service.rs) — managed-service stop: terminating the recorded processes of a service after verifying their identity, and the stop report (`stop_managed_service`, `stop_report_confirmed`) that the sandbox and MCP tools read. +- [`service.rs`](../apps/rocmd/src/service.rs) — managed-service stop: the engine `Stop`, the manifest and endpoint-key writes, and the stop report (`stop_managed_service`, `stop_report_confirmed`) that the sandbox and MCP tools read. The termination of the recorded processes is the routine shared with `rocm services stop`, in [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs); this module passes it `terminate_recorded_pid` and keeps no copy of it or of its grace constant. - [`supervise.rs`](../apps/rocmd/src/supervise.rs) — launching and re-launching a managed service's supervisor: `supervise_service` (the `rocmd supervise` entry point: launch the engine child, record its identity, wait for readiness, record the exit) with the serve-log startup-phase polling and readiness wait it uses, and `restart_managed_service` (the recovery respawn: re-launch `rocmd supervise` detached for a service and record the new supervisor). Writes after a supervisor's first go through the private `update_supervised_record`, which re-reads the record and leaves a stop marker or another supervisor's record alone. Its production code uses only [`common.rs`](../apps/rocmd/src/common.rs), [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs) and [`persistence.rs`](../apps/rocmd/src/persistence.rs). [`cli.rs`](../apps/rocmd/src/cli.rs) calls `supervise_service`; [`watchers/server_recover.rs`](../apps/rocmd/src/watchers/server_recover.rs) (the server-recover handler) and [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) (the `restart_server` tool) call `restart_managed_service`. - [`watchers.rs`](../apps/rocmd/src/watchers.rs) — the module root of the built-in watchers: the registry (`reconcile_watcher_snapshots`) and the event collector and dispatcher (`evaluate_watchers`, `collect_automation_events_with_gpu_snapshot`, which takes the GPU snapshot as a closure so tests can inject one, and `evaluate_watchers_for_events`), which route each event to its watcher's handler. `find_recoverable_service` and `stop_requested` are re-exported from here because [`supervise.rs`](../apps/rocmd/src/supervise.rs)'s and [`service.rs`](../apps/rocmd/src/service.rs)'s tests reach them by the `crate::watchers::` path. Each watcher's handler lives in the child below that carries its name, together with the policy they share, and a child reaches only [`policy.rs`](../apps/rocmd/src/watchers/policy.rs) and, for the thermal-protect watcher, [`gpu_metrics.rs`](../apps/rocmd/src/watchers/gpu_metrics.rs) among the others. - [`watchers/policy.rs`](../apps/rocmd/src/watchers/policy.rs) — what a watcher does in each mode and the proposals it queues: `WatcherPolicyAction` and `watcher_policy_action` (observe, queue a proposal or run contained, chosen by the watcher mode), `queue_proposal` and `queue_proposal_with_arguments` with the action-to-tool and action-to-arguments tables behind them, and `pending_proposal_exists`, the check the thermal-protect and server-recover handlers use so they do not queue a duplicate of a proposal that is still pending. It uses no other watcher module. @@ -231,7 +231,7 @@ Further `rocm-core` modules, each with its own doc comment except where noted: - [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) — OpenMPI, libatomic and libnuma detection, the `libmpi_cxx` compatibility shim, and the per-library install plans (`build_openmpi_install_plan`, `build_libatomic_install_plan`, `build_libnuma_install_plan`) and hints, needed by vLLM's ROCm wheels. The distro-generic half of that planning lives in [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs); [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) re-exports its public items, so `rocm_core::openmpi::running_as_root`, `plan_for_os_release_read` and the rest keep resolving, and `pub(crate)` re-exports `PackageManager` and `resolve_package_manager` for [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs). - [`os_release.rs`](../crates/rocm-core/src/os_release.rs) — the one `/etc/os-release` parser every reader of that file uses. - [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs) — the planning half of a system-package install that does not depend on which library is installed: `PackageManager` and `resolve_package_manager` (from an os-release `ID`/`ID_LIKE`), `SystemPackageInstallPlan` and `InstallCommand`, `plan_for_os_release_at`/`plan_for_os_release_read` (which take the per-library builder as an argument), `InstallHint`, the root and passwordless-`sudo` checks (`running_as_root`, `can_autoinstall`) and the `apt-get -s` helpers (`apt_simulate_argv`, `parse_apt_simulate_removals`, `is_protected_rocm_package`). It never runs anything and depends on no other module of this crate except [`os_release.rs`](../crates/rocm-core/src/os_release.rs) and [`hardware_root.rs`](../crates/rocm-core/src/hardware_root.rs), which it reads os-release through. The module is private; its public items are re-exported from [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs). It is a different module from [`system_packages.rs`](../apps/rocm/src/system_packages.rs) in `apps/rocm`, which runs these plans with consent and `sudo`. -- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches. +- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches (`terminate_verified`), and the one managed-stop routine built on it that `rocm services stop` and `rocmd`'s `stop_server` both call: `terminate_recorded_service_pids` walks the processes a service record names (supervisor, then engine, each with its own start-time, the stopping process's own PID reported as `self` and never signalled) and returns one `RecordedPidStop` per PID; `terminate_recorded_pid` terminates one of them, adding on Windows a `taskkill /T /F` of its descendants first, because `terminate_verified` cannot walk a process tree there; `MANAGED_STOP_GRACE` is the one grace period. The termination is injectable, so a caller's test supplies outcomes it cannot construct and a test of the Windows step runs on every host. - [`report.rs`](../crates/rocm-core/src/report.rs) — the content of a Doctor report and the rule deciding whether one may exist; it sends nothing. - [`report_delivery.rs`](../crates/rocm-core/src/report_delivery.rs) — how a report leaves the machine: building its link and deciding whether a browser may be started. - [`runtime.rs`](../crates/rocm-core/src/runtime.rs) — the host-platform layer: `RuntimePlatform`/`RuntimeHost`, OS-specific executable and Python-environment names (including `runtime_python_env_root_for_executable`, the one rule for finding an environment's folder from its interpreter's path: only an interpreter directly inside a `bin` or `Scripts` folder, in any case, has one, and everything else gets `None`; the adopt command, the ComfyUI runtime environment and the vLLM launch all call it rather than keeping a copy, since the engine crate cannot reach `apps/rocm`), default config/data/cache directories, path-text and path-list helpers, and `install_root_has_payload`, the one definition of a non-empty install root (dot-entries such as the runtime marker do not count; a path that does not exist holds nothing; a root that cannot be listed is an error) that tarball extraction, update repair detection and activation validation in `apps/rocm` all ask (no module doc comment). diff --git a/docs/testing.md b/docs/testing.md index b3369520b6..03ae2e5cf5 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1772,6 +1772,27 @@ directly, exits 0, and records whether that signal worked, adding a second stderr `note:` when it did not (`cargo test -p rocm --bin rocm services_stop_` and `cargo test -p rocm --bin rocm attached_ctrl_c_`). +`rocm services stop` and `rocmd`'s `stop_server` terminate the processes a +record names through one routine, +[`terminate_recorded_service_pids`](../crates/rocm-core/src/proc_lifecycle.rs), +and wait `MANAGED_STOP_GRACE` for each; neither binary keeps a copy. The +routine is tested in `rocm-core` (`cargo test -p rocm-core proc_lifecycle::`): +the supervisor is visited before the engine, each PID with its own start-time, +a PID recorded under both roles once, and the stopping process's own PID is +reported as `self` in `pid_outcomes` and never signalled, in both commands. +On Windows the verified root is preceded by `taskkill /PID /T /F`, because +`terminate_verified` cannot walk a process tree there; that step is tested +through injected functions that run on every host (`the_tree_is_killed_before_the_root_is_terminated` +and its neighbours), not by killing a real tree. The callers' own tests check +that each reaches the routine and reports what it returns +(`cargo test -p rocm --bin rocm service_control::tests::a_stop_` and +`cargo test -p rocmd service::tests::a_stop_`), and each carries a source check +that no private copy of the routine or its constant has come back. No scenario +kills a real process tree on Windows: `service-stop-01` plants PID 0 and +`service-stop-02` is Linux-only, so the Windows tree kill has no live coverage +beyond compiling there and the injected unit tests, which the `windows-debug` +lane runs with `cargo test --workspace --all-targets`. + No scenario drives the attached Ctrl-C stop end to end. Attaching needs a real `rocm serve` launch — an engine and a runtime the mock lane does not have, which is why the chat scenarios there plant a record behind a mock server From 9a62ecef97eec3d2ace5202e38132aa1edc27fdf Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 06:28:59 +0000 Subject: [PATCH 03/30] fix(services): clear the recorded processes after a confirmed rocm stop `rocmd`'s stop clears the supervisor and engine PIDs and their start-time tokens from the record once every recorded process is confirmed gone; `rocm services stop` did not. Its record kept naming the processes it had just stopped, so a later stop of the same record reached the termination again. A PID recorded without a start-time token (every one on Windows) cannot be refuted once the OS has recycled it, and the Windows tree kill that the shared routine added for the CLI would then take the unrelated process and everything under it. The clearing becomes one `ManagedServiceRecord::clear_recorded_processes` in `rocm-core`, called from the confirmed branch of both stops; `rocmd`'s four field assignments are replaced by the call, so the two leave the same record behind. An unconfirmed stop still keeps the PIDs, since the record is the only thing that names a survivor. `a_confirmed_stop_clears_the_recorded_processes_so_a_second_stop_signals_nothing` runs the real termination routine with the per-PID signal injected: after the first stop the persisted record names no process, and a second stop signals nothing and still reports stopped. Before the change the record kept PID 4242 and the second stop visited it again. The unconfirmed direction and the clearing method have their own tests. Signed-off-by: Jussi Elo --- apps/rocm/src/service_control.rs | 117 ++++++++++++++++++++++++- apps/rocmd/src/service.rs | 8 +- crates/rocm-core/src/service_record.rs | 48 ++++++++++ docs/architecture.md | 2 +- docs/testing.md | 12 ++- 5 files changed, 179 insertions(+), 8 deletions(-) diff --git a/apps/rocm/src/service_control.rs b/apps/rocm/src/service_control.rs index c3d9986943..e81823817b 100644 --- a/apps/rocm/src/service_control.rs +++ b/apps/rocm/src/service_control.rs @@ -256,6 +256,14 @@ fn stop_internal_managed_service_confirmed_with( if all_stopped { record.status = "stopped".to_owned(); record.stop_requested_unix_ms = None; + // The processes this stop confirmed gone are not the service's any more, + // and their PIDs are stale the instant they exited. Left in the record, + // a later stop would signal them again: a PID recorded without a + // start-time token (every one on Windows) cannot be refuted, so if the + // OS had given it to another process the Windows tree kill would take + // that process and its subtree. The same call, on the same confirmed + // path, as `rocmd`'s stop. + record.clear_recorded_processes(); } else { // The marker records that a stop was *asked for* even though it could // not be confirmed. `refresh_managed_service_runtime_liveness` needs it @@ -1505,6 +1513,110 @@ mod tests { assert_eq!(key, None, "a confirmed stop drops the key"); } + /// A confirmed stop must not leave the processes it stopped named in the + /// record. Their PIDs are stale the moment they exit, and a recorded PID + /// with no start-time token is signalled on trust alone once the OS has + /// handed it to another process; on Windows the tree kill then takes that + /// process and everything under it. `rocmd`'s stop clears them for the same + /// reason (`stop_managed_service_clears_recorded_pids_once_the_processes_are_gone`). + #[test] + fn a_confirmed_stop_clears_the_recorded_processes_so_a_second_stop_signals_nothing() { + use rocm_core::TerminationOutcome::Graceful; + let (root, paths) = test_paths("services-stop-clears-pids"); + paths.ensure().unwrap(); + let service_id = "svc-services-stop-clears-pids"; + let mut record = stop_test_record(&paths, service_id); + // The dangerous shape: a supervisor recorded without a start-time token, + // and an engine with one. + record.supervisor_pid = 4242; + record.supervisor_start_ticks = None; + record.engine_pid = Some(4243); + record.engine_start_ticks = Some(77); + record.write().unwrap(); + + // The termination of each recorded PID, observed: the real routine runs + // (which PIDs it visits, in which order), only the signalling is replaced. + let signalled = std::cell::RefCell::new(Vec::new()); + let stop = |record: &ManagedServiceRecord| { + rocm_core::terminate_recorded_service_pids(record, |identity| { + signalled.borrow_mut().push(*identity); + Graceful + }) + }; + + let first = stop_internal_managed_service_with(&paths, service_id, stop).unwrap(); + let after_first = raw_manifest(&paths, service_id); + let signalled_by_first = signalled.borrow().clone(); + + let second = stop_internal_managed_service_with(&paths, service_id, stop).unwrap(); + let after_second = raw_manifest(&paths, service_id); + let signalled_in_all = signalled.borrow().clone(); + let _ = fs::remove_dir_all(root); + + assert_eq!(first["stopped"], true, "{first}"); + assert_eq!( + signalled_by_first, + vec![ + rocm_core::ProcessIdentity::new(4242, None), + rocm_core::ProcessIdentity::new(4243, Some(77)), + ] + ); + // The persisted state after the confirmed stop names no process. + assert_eq!(after_first.status, "stopped"); + assert_eq!(after_first.stop_requested_unix_ms, None); + assert_eq!(after_first.supervisor_pid, 0); + assert_eq!(after_first.supervisor_start_ticks, None); + assert_eq!(after_first.engine_pid, None); + assert_eq!(after_first.engine_start_ticks, None); + // So a stop of the same record signals nothing, and still reads stopped. + assert_eq!( + signalled_in_all, signalled_by_first, + "the second stop reached a PID the first one had already stopped" + ); + assert_eq!(second["stopped"], true, "{second}"); + assert_eq!(second["signaled_pids"], serde_json::json!([]), "{second}"); + assert_eq!(second["pid_outcomes"], serde_json::json!([]), "{second}"); + assert_eq!(after_second.status, "stopped"); + assert_eq!(after_second.supervisor_pid, 0); + assert_eq!(after_second.engine_pid, None); + } + + #[test] + fn an_unconfirmed_stop_keeps_the_recorded_processes_for_the_next_stop() { + use rocm_core::TerminationOutcome::TimedOut; + let (root, paths) = test_paths("services-stop-keeps-pids"); + paths.ensure().unwrap(); + let service_id = "svc-services-stop-keeps-pids"; + let mut record = stop_test_record(&paths, service_id); + record.supervisor_pid = 4242; + record.supervisor_start_ticks = Some(5); + record.engine_pid = Some(4243); + record.engine_start_ticks = Some(77); + record.write().unwrap(); + + let signalled = std::cell::Cell::new(0_usize); + let stop = |record: &ManagedServiceRecord| { + rocm_core::terminate_recorded_service_pids(record, |_| { + signalled.set(signalled.get() + 1); + TimedOut + }) + }; + let first = stop_internal_managed_service_with(&paths, service_id, stop).unwrap(); + let after_first = raw_manifest(&paths, service_id); + let _ = stop_internal_managed_service_with(&paths, service_id, stop).unwrap(); + let signalled_in_all = signalled.get(); + let _ = fs::remove_dir_all(root); + + // Nothing was confirmed gone, so the record still names what survived: + // that is the only thing a later stop can find them by. + assert_eq!(first["stopped"], false, "{first}"); + assert_eq!(after_first.supervisor_pid, 4242); + assert_eq!(after_first.supervisor_start_ticks, Some(5)); + assert_eq!(after_first.engine_pid, Some(4243)); + assert_eq!(after_first.engine_start_ticks, Some(77)); + assert_eq!(signalled_in_all, 4, "each stop tried both processes"); + } + #[test] fn restart_does_not_start_an_engine_over_an_unconfirmed_stop() { // A restart used to discard its stop's result and start the new engine @@ -1804,8 +1916,11 @@ mod tests { // The claim the message makes: the service really is stopped, and no // replacement child was started behind the failed wait. assert_eq!(record.status, "stopped"); + // The confirmed stop cleared the recorded processes (the record names + // none), so a supervisor recorded behind the failed wait would show as a + // nonzero PID. assert_eq!( - record.supervisor_pid, 999_999_999, + record.supervisor_pid, 0, "no new supervisor may be recorded behind the failed wait" ); Ok(()) diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index 9a3e01192e..467ad5c189 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -127,11 +127,9 @@ pub(crate) fn stop_managed_service_via( record.stop_requested_unix_ms = None; // Recorded PIDs go stale the instant their processes exit, and a stale // PID is exactly what the identity check exists to catch — so do not - // leave one behind for the next stop to find. - record.supervisor_pid = 0; - record.supervisor_start_ticks = None; - record.engine_pid = None; - record.engine_start_ticks = None; + // leave one behind for the next stop to find. The same call + // `rocm services stop` makes on its confirmed path. + record.clear_recorded_processes(); } else { // The recorded PIDs stay: an unconfirmed survivor is still the // service's, and the record is what names it. diff --git a/crates/rocm-core/src/service_record.rs b/crates/rocm-core/src/service_record.rs index aecab92d8c..bf405c3728 100644 --- a/crates/rocm-core/src/service_record.rs +++ b/crates/rocm-core/src/service_record.rs @@ -322,6 +322,25 @@ impl ManagedServiceRecord { self.engine_start_ticks = crate::process_start_ticks(pid); } + /// Forget the processes this record names: the supervisor and the engine, + /// each with its start-time token. + /// + /// For a stop that has confirmed every one of them gone, and for nothing + /// else. A recorded PID goes stale the instant its process exits, and a + /// later stop of the same record would signal whatever the OS has since + /// handed that number to; a PID recorded without a token cannot be + /// refuted, and on Windows none can, so the tree kill would take that + /// process and everything under it. `rocm services stop` and `rocmd`'s stop + /// both call this on the confirmed path, so the two leave the same record + /// behind. An unconfirmed stop keeps the PIDs: the record is the only thing + /// that names a survivor. + pub const fn clear_recorded_processes(&mut self) { + self.supervisor_pid = 0; + self.supervisor_start_ticks = None; + self.engine_pid = None; + self.engine_start_ticks = None; + } + pub fn normalize_paths_for_host(&mut self) { self.manifest_path = normalize_runtime_path_for_host(&self.manifest_path); self.log_path = normalize_runtime_path_for_host(&self.log_path); @@ -745,6 +764,35 @@ mod tests { assert_eq!(record.engine_start_ticks, expected); } + /// What a stop that confirmed its processes gone leaves behind: no PID and + /// no token for either role, and nothing else touched. + #[test] + fn clearing_the_recorded_processes_clears_each_pid_and_its_token_and_nothing_else() { + let mut record = probe_test_record(11437); + record.supervisor_pid = 4242; + record.supervisor_start_ticks = Some(5); + record.engine_pid = Some(4243); + record.engine_start_ticks = Some(77); + record.status = "ready".to_owned(); + record.restart_count = 3; + let before = record.clone(); + + record.clear_recorded_processes(); + + assert_eq!(record.supervisor_pid, 0); + assert_eq!(record.supervisor_start_ticks, None); + assert_eq!(record.engine_pid, None); + assert_eq!(record.engine_start_ticks, None); + assert_eq!(record.service_id, before.service_id); + assert_eq!(record.status, "ready"); + assert_eq!(record.restart_count, 3); + assert_eq!(record.port, before.port); + // A cleared record no longer names what it named, so a stop that read it + // before and finds this after does not claim the same processes. + assert!(!before.names_same_processes(&record)); + assert!(record.names_same_processes(&record.clone())); + } + #[test] fn engine_state_refresh_keeps_the_runtime_selector_the_launch_requested() -> Result<()> { let (root, paths) = temp_app_paths("record-runtime-pin"); diff --git a/docs/architecture.md b/docs/architecture.md index 823630e6d3..4679d09d12 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -231,7 +231,7 @@ Further `rocm-core` modules, each with its own doc comment except where noted: - [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) — OpenMPI, libatomic and libnuma detection, the `libmpi_cxx` compatibility shim, and the per-library install plans (`build_openmpi_install_plan`, `build_libatomic_install_plan`, `build_libnuma_install_plan`) and hints, needed by vLLM's ROCm wheels. The distro-generic half of that planning lives in [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs); [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) re-exports its public items, so `rocm_core::openmpi::running_as_root`, `plan_for_os_release_read` and the rest keep resolving, and `pub(crate)` re-exports `PackageManager` and `resolve_package_manager` for [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs). - [`os_release.rs`](../crates/rocm-core/src/os_release.rs) — the one `/etc/os-release` parser every reader of that file uses. - [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs) — the planning half of a system-package install that does not depend on which library is installed: `PackageManager` and `resolve_package_manager` (from an os-release `ID`/`ID_LIKE`), `SystemPackageInstallPlan` and `InstallCommand`, `plan_for_os_release_at`/`plan_for_os_release_read` (which take the per-library builder as an argument), `InstallHint`, the root and passwordless-`sudo` checks (`running_as_root`, `can_autoinstall`) and the `apt-get -s` helpers (`apt_simulate_argv`, `parse_apt_simulate_removals`, `is_protected_rocm_package`). It never runs anything and depends on no other module of this crate except [`os_release.rs`](../crates/rocm-core/src/os_release.rs) and [`hardware_root.rs`](../crates/rocm-core/src/hardware_root.rs), which it reads os-release through. The module is private; its public items are re-exported from [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs). It is a different module from [`system_packages.rs`](../apps/rocm/src/system_packages.rs) in `apps/rocm`, which runs these plans with consent and `sudo`. -- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches (`terminate_verified`), and the one managed-stop routine built on it that `rocm services stop` and `rocmd`'s `stop_server` both call: `terminate_recorded_service_pids` walks the processes a service record names (supervisor, then engine, each with its own start-time, the stopping process's own PID reported as `self` and never signalled) and returns one `RecordedPidStop` per PID; `terminate_recorded_pid` terminates one of them, adding on Windows a `taskkill /T /F` of its descendants first, because `terminate_verified` cannot walk a process tree there; `MANAGED_STOP_GRACE` is the one grace period. The termination is injectable, so a caller's test supplies outcomes it cannot construct and a test of the Windows step runs on every host. +- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches (`terminate_verified`), and the one managed-stop routine built on it that `rocm services stop` and `rocmd`'s `stop_server` both call: `terminate_recorded_service_pids` walks the processes a service record names (supervisor, then engine, each with its own start-time, the stopping process's own PID reported as `self` and never signalled) and returns one `RecordedPidStop` per PID (each caller, on a stop that confirms every one gone, then clears the record's recorded processes with `ManagedServiceRecord::clear_recorded_processes` in [`service_record.rs`](../crates/rocm-core/src/service_record.rs), so a repeated stop reaches no stale PID; an unconfirmed stop keeps them); `terminate_recorded_pid` terminates one of them, adding on Windows a `taskkill /T /F` of its descendants first, because `terminate_verified` cannot walk a process tree there; `MANAGED_STOP_GRACE` is the one grace period. The termination is injectable, so a caller's test supplies outcomes it cannot construct and a test of the Windows step runs on every host. - [`report.rs`](../crates/rocm-core/src/report.rs) — the content of a Doctor report and the rule deciding whether one may exist; it sends nothing. - [`report_delivery.rs`](../crates/rocm-core/src/report_delivery.rs) — how a report leaves the machine: building its link and deciding whether a browser may be started. - [`runtime.rs`](../crates/rocm-core/src/runtime.rs) — the host-platform layer: `RuntimePlatform`/`RuntimeHost`, OS-specific executable and Python-environment names (including `runtime_python_env_root_for_executable`, the one rule for finding an environment's folder from its interpreter's path: only an interpreter directly inside a `bin` or `Scripts` folder, in any case, has one, and everything else gets `None`; the adopt command, the ComfyUI runtime environment and the vLLM launch all call it rather than keeping a copy, since the engine crate cannot reach `apps/rocm`), default config/data/cache directories, path-text and path-list helpers, and `install_root_has_payload`, the one definition of a non-empty install root (dot-entries such as the runtime marker do not count; a path that does not exist holds nothing; a root that cannot be listed is an error) that tarball extraction, update repair detection and activation validation in `apps/rocm` all ask (no module doc comment). diff --git a/docs/testing.md b/docs/testing.md index 03ae2e5cf5..88cb3af491 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1787,7 +1787,17 @@ and its neighbours), not by killing a real tree. The callers' own tests check that each reaches the routine and reports what it returns (`cargo test -p rocm --bin rocm service_control::tests::a_stop_` and `cargo test -p rocmd service::tests::a_stop_`), and each carries a source check -that no private copy of the routine or its constant has come back. No scenario +that no private copy of the routine or its constant has come back. A confirmed +stop also clears the PIDs and start-time tokens it stopped from the record +(`ManagedServiceRecord::clear_recorded_processes`, one call from both +binaries), and an unconfirmed one keeps them; a PID recorded without a token is +signalled on trust alone once the OS has recycled it, so a stale one must not +survive for a second stop to find. +`a_confirmed_stop_clears_the_recorded_processes_so_a_second_stop_signals_nothing` +runs the real routine with the per-PID signal injected, and asserts the +persisted record after the first stop and that a second stop signals nothing; +`an_unconfirmed_stop_keeps_the_recorded_processes_for_the_next_stop` is the +other direction. No scenario kills a real process tree on Windows: `service-stop-01` plants PID 0 and `service-stop-02` is Linux-only, so the Windows tree kill has no live coverage beyond compiling there and the injected unit tests, which the `windows-debug` From b8670898e201bdfaf7196044fad1550a1d78c527 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 06:38:58 +0000 Subject: [PATCH 04/30] test(e2e): stop a real managed server and its child through rocm services stop No scenario ran `rocm services stop` against a real recorded process. service-stop-01 plants PID 0, so it never signals anything, and service-stop-02 drives the daemon's tool, not the CLI's stop; the Windows tree kill the CLI now shares with the daemon (`taskkill /PID /T /F` before the verified termination) had no live coverage at all, only unit tests with the signalling injected. service-stop-06 (Linux) runs `rocm services stop --yes` against a recorded process the scenario starts, and checks the persisted state with the report: the process exited, the record reads stopped and names no process, and the endpoint key file is gone. It needs no new step: it reuses the steps of -02 and -03. service-stop-07 (`@requires-os:windows`, no GPU) records a PowerShell parent that has started a `ping` child, with the PID alone and no start-time token as Windows records it, runs `rocm services stop --yes`, and asserts that both processes are gone together with the report (`Local server stopped`, `stopped processes: 1`) and the record. Without the tree-kill step the stop terminates only the recorded process, the child keeps running, and the scenario fails on it. The Given checks both processes are running first, so the Then cannot pass for the wrong reason. The scenario was written on Linux and has not been run on Windows; its tag keeps it off every other host (the suite's host filter skips it here, which a run without a name filter confirms). A name filter bypasses that filter, so the documented command for the stop scenarios is now `-n "service-stop-0[1-6]"`. Signed-off-by: Jussi Elo --- docs/testing.md | 27 +++- .../features/service_stop.feature | 49 ++++++ .../tests/e2e/service_stop_steps.rs | 140 +++++++++++++++++- 3 files changed, 208 insertions(+), 8 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 88cb3af491..2491854c90 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1783,7 +1783,8 @@ reported as `self` in `pid_outcomes` and never signalled, in both commands. On Windows the verified root is preceded by `taskkill /PID /T /F`, because `terminate_verified` cannot walk a process tree there; that step is tested through injected functions that run on every host (`the_tree_is_killed_before_the_root_is_terminated` -and its neighbours), not by killing a real tree. The callers' own tests check +and its neighbours) and, against a real tree, by a Windows-only scenario +(`service-stop-07`, below). The callers' own tests check that each reaches the routine and reports what it returns (`cargo test -p rocm --bin rocm service_control::tests::a_stop_` and `cargo test -p rocmd service::tests::a_stop_`), and each carries a source check @@ -1797,11 +1798,21 @@ survive for a second stop to find. runs the real routine with the per-PID signal injected, and asserts the persisted record after the first stop and that a second stop signals nothing; `an_unconfirmed_stop_keeps_the_recorded_processes_for_the_next_stop` is the -other direction. No scenario -kills a real process tree on Windows: `service-stop-01` plants PID 0 and -`service-stop-02` is Linux-only, so the Windows tree kill has no live coverage -beyond compiling there and the injected unit tests, which the `windows-debug` -lane runs with `cargo test --workspace --all-targets`. +other direction. `service-stop-07` +(`@requires-os:windows`, no GPU) is the live coverage of the Windows tree kill: +it records a PowerShell parent that has started a `ping` child, runs `rocm +services stop --yes`, and asserts that both processes are gone together with the +report (`Local server stopped`, `stopped processes: 1`) and the record (stopped, +no process, no stop request, key file gone). The child is the tree kill's own +work: with the `taskkill /T` step removed the stop ends only the recorded +process and the scenario fails on the surviving child. `service-stop-01` plants +PID 0 and `service-stop-02` is Linux-only, so neither reaches that step; the +injected unit tests above cover its ordering and verdicts on every host. +`service-stop-06` (Linux) is the same stop of a real recorded process through +`rocm services stop`, checking the persisted state: the process exited and the +record names none. Run the Windows scenario on a Windows host with +`cargo test -p e2e-cucumber --test e2e -- -n "service-stop-07"` +and a prebuilt `ROCM_CLI_BINARY`. No scenario drives the attached Ctrl-C stop end to end. Attaching needs a real `rocm serve` launch — an engine and a runtime the mock lane does not have, @@ -1841,7 +1852,9 @@ checks that the line omits it and no key file exists. Because the seam replaces `terminate_verified`, they say nothing about what is signalled. They need a hooks build, as `cargo xtask e2e` makes: `cargo build -p rocm -p rocmd --features rocm/e2e-test-hooks`, then the command above with `-n -"service-stop"`. +"service-stop-0[1-6]"`. Not `-n "service-stop"`: a name filter bypasses the +suite's own host filter, so it would also run the Windows-only +`service-stop-07` on a Linux host, where it fails for want of PowerShell. Manual restricted-tool smoke: diff --git a/tests/e2e-cucumber/features/service_stop.feature b/tests/e2e-cucumber/features/service_stop.feature index cae0e9859a..dfbc60b11d 100644 --- a/tests/e2e-cucumber/features/service_stop.feature +++ b/tests/e2e-cucumber/features/service_stop.feature @@ -119,3 +119,52 @@ Feature: Stopping a managed service Then the CLI reports the stop as not confirmed and names no stop command And the server's record still names its process and the stop request And the report says only the record was kept, and there is no key file + + # `rocm services stop`'s confirmed half with a real process. service-stop-01 + # plants PID 0, so it never signals anything and cannot show what the stop + # leaves in the record; service-stop-02 shows it for the daemon's tool. This + # runs the CLI's own stop against a real recorded process and checks the + # persisted state it claims: the process exited, and the record names no + # process any more, so a second stop of that record would signal nothing (a + # PID recorded without a start-time token cannot be refuted once the OS + # recycles it). The unit test + # `a_confirmed_stop_clears_the_recorded_processes_so_a_second_stop_signals_nothing` + # covers the second stop itself. + # + # Linux-only for the same reason as service-stop-02: the recorded process + # carries the start-time token `/proc` gives it. + + @id:service-stop-cli-confirms-a-stop-and-clears-the-record + @requires-os:linux + Scenario: service-stop-06 - `rocm services stop` of a running managed server confirms the stop and leaves no process in its record + Given a managed server whose recorded process is running + When the user stops it with --yes + Then the CLI reports the service as stopped + And the recorded process is no longer running + And the server's record reads stopped and names no process + And the server's endpoint key file is gone + + # The Windows half of the stop. A managed engine runs one level below the + # recorded process (the daemon launches it through an `__engine-serve-http` + # process), and `terminate_verified` cannot walk a process tree on Windows, so + # the stop first runs `taskkill /PID /T /F` on the recorded process. + # service-stop-01 plants PID 0 and never reaches that step, and the scenarios + # above need Linux's start-time token, so none of them kills a real tree. + # + # This records a real parent that has started a real child, stops it, and + # checks that BOTH are gone together with the report and the record. Without + # the tree-kill step the stop ends only the recorded process, the child keeps + # running, and the scenario fails on it. No GPU, no engine, no network. The + # record carries no start-time token, because Windows has none: the case the + # tree kill and the cleared record exist for. + + @id:service-stop-takes-down-the-whole-process-tree-on-windows + @requires-os:windows + Scenario: service-stop-07 - Stopping a managed server on Windows takes down its recorded process together with the process it started + Given a managed server whose recorded process has started another process + When the user stops it with --yes + Then the CLI reports the service as stopped + And the report counts one stopped process + And the recorded process and the process it started are no longer running + And the server's record reads stopped and names no process + And the server's endpoint key file is gone diff --git a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs index 31df329d2b..273ce046cc 100644 --- a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs @@ -166,9 +166,113 @@ fn plant_server_with_running_process(world: &mut E2eWorld, host: &str, with_key: .stderr(Stdio::null()) .spawn() .expect("failed to start the process to record"); + let ticks = start_ticks(process.id()); + plant_server_record(world, host, with_key, process, Some(ticks)); +} + +/// Where the parent the Windows scenario records writes the PID of the process +/// it started, for the scenario to read. +fn descendant_pid_path(world: &E2eWorld) -> PathBuf { + services_dir(world).join("descendant.pid") +} + +/// A ready, publicly bound server whose recorded supervisor is a parent process +/// that has started a child of its own, recorded the way Windows records it: +/// the PID alone, with no start-time token (there is none to read there). +/// +/// The parent is PowerShell, the child `ping` for five minutes: both ship with +/// Windows, need nothing installed, and are disposable. PowerShell starts the +/// child itself, so the child's parent is the recorded PID, which is the +/// relationship `taskkill /T` follows; it writes the child's PID where the +/// scenario can read it and then stays alive. Both outlive the scenario only if +/// the stop fails to take them, and the World's teardown reaps the parent. +#[given("a managed server whose recorded process has started another process")] +async fn managed_server_whose_process_started_another(world: &mut E2eWorld) { + let services = services_dir(world); + std::fs::create_dir_all(&services).expect("failed to create services dir"); + let pid_file = descendant_pid_path(world); + let process = spawn_parent_with_descendant(&pid_file); + let pid = process.id(); + plant_server_record(world, "0.0.0.0", true, process, None); + + // The premise, so the Then step cannot pass for the wrong reason: the + // descendant exists and is running, and so is the parent. + let descendant = wait_for_descendant_pid(&pid_file); + assert_ne!(descendant, pid, "the child must be a different process"); + assert!( + windows_process_is_running(descendant), + "premise: the process the recorded one started must be running" + ); + assert!( + world + .recorded_process + .as_mut() + .expect("scenario recorded no process") + .try_wait() + .expect("failed to check the recorded process") + .is_none(), + "premise: the recorded process must be running" + ); +} + +fn spawn_parent_with_descendant(pid_file: &std::path::Path) -> std::process::Child { + // A single-quoted PowerShell string: an apostrophe in the path is doubled. + let pid_file = pid_file.display().to_string().replace('\'', "''"); + let script = format!( + "$child = Start-Process -FilePath ping.exe -ArgumentList '-n','300','127.0.0.1' \ + -WindowStyle Hidden -PassThru; \ + Set-Content -LiteralPath '{pid_file}' -Value $child.Id; \ + Start-Sleep -Seconds 300" + ); + Command::new("powershell") + .args(["-NoProfile", "-NonInteractive", "-Command", &script]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("failed to start the process to record") +} + +/// The child's PID, once the parent has written it. +fn wait_for_descendant_pid(pid_file: &std::path::Path) -> u32 { + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60); + loop { + if let Some(pid) = std::fs::read_to_string(pid_file) + .ok() + .and_then(|text| text.trim().parse().ok()) + { + return pid; + } + assert!( + std::time::Instant::now() < deadline, + "the recorded process never wrote the PID of the process it started" + ); + std::thread::sleep(std::time::Duration::from_millis(200)); + } +} + +/// Whether Windows has a process `pid`, by `tasklist`: a match prints a CSV +/// row naming it, and no match prints an INFO line without one. +fn windows_process_is_running(pid: u32) -> bool { + let output = Command::new("tasklist") + .args(["/FI", &format!("PID eq {pid}"), "/FO", "CSV", "/NH"]) + .output() + .expect("failed to run tasklist"); + String::from_utf8_lossy(&output.stdout).contains(&format!("\"{pid}\"")) +} + +/// Write the record for a ready server whose supervisor is `process`, with +/// `ticks` as its start-time token when there is one, and take `process` into +/// the World so the scenario (and its teardown) can check and reap it. +fn plant_server_record( + world: &mut E2eWorld, + host: &str, + with_key: bool, + process: std::process::Child, + ticks: Option, +) { let pid = process.id(); world.recorded_process = Some(process); - let ticks = start_ticks(pid); let services = services_dir(world); std::fs::create_dir_all(&services).expect("failed to create services dir"); @@ -313,6 +417,40 @@ async fn tool_reports_stopped(world: &mut E2eWorld) { #[then("the recorded process is no longer running")] async fn recorded_process_gone(world: &mut E2eWorld) { + assert_recorded_process_exited(world); +} + +/// The Windows tree kill, observed: the recorded process AND the one it started +/// are gone after the stop. Without the `taskkill /T` step the stop ends only +/// the recorded process (`TerminateProcess` on the root), and the child keeps +/// running, which is what the second check catches. +#[then("the recorded process and the process it started are no longer running")] +async fn recorded_process_and_its_child_gone(world: &mut E2eWorld) { + assert_recorded_process_exited(world); + let pid = wait_for_descendant_pid(&descendant_pid_path(world)); + // `taskkill` returns once it has asked for the termination; give the exit + // a moment to land before calling the process a survivor. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(15); + while windows_process_is_running(pid) { + assert!( + std::time::Instant::now() < deadline, + "the stop took the recorded process down but left the process it \ + started (PID {pid}) running: the tree kill did not reach it" + ); + std::thread::sleep(std::time::Duration::from_millis(250)); + } +} + +/// The report says one process was stopped (the recorded one: a tree kill that +/// already took the root is reported as forced, so it counts), checked next to +/// the processes themselves rather than instead of them. +#[then("the report counts one stopped process")] +async fn report_counts_one_stopped_process(world: &mut E2eWorld) { + let combined = combined_output(world); + assert!(combined.contains("stopped processes: 1"), "{combined}"); +} + +fn assert_recorded_process_exited(world: &mut E2eWorld) { let process = world .recorded_process .as_mut() From fa60b69a2e3d4ff4ebebec939e88617c18201821 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 07:40:39 +0000 Subject: [PATCH 05/30] test(e2e): write the service-stop wait in minutes for clippy Rust 1.96 clippy::duration_suboptimal_units flags Duration::from_secs(60) in the service-stop step. The deadline stays 60 seconds; only the unit it is spelled in changes. The e2e harness target sets test = false, so the workspace clippy run skips it and only the dedicated e2e-cucumber --test e2e clippy step lints it. Signed-off-by: Jussi Elo --- tests/e2e-cucumber/tests/e2e/service_stop_steps.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs index 273ce046cc..798354e5e1 100644 --- a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs @@ -235,7 +235,7 @@ fn spawn_parent_with_descendant(pid_file: &std::path::Path) -> std::process::Chi /// The child's PID, once the parent has written it. fn wait_for_descendant_pid(pid_file: &std::path::Path) -> u32 { - let deadline = std::time::Instant::now() + std::time::Duration::from_secs(60); + let deadline = std::time::Instant::now() + std::time::Duration::from_mins(1); loop { if let Some(pid) = std::fs::read_to_string(pid_file) .ok() From de8ee04ab4b42238569efb759420ed32841f8d52 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 15:28:42 +0000 Subject: [PATCH 06/30] fix(core): record a process creation time as the start identity on Windows, so a recycled PID is refused before the tree kill process_start_ticks read /proc and was None on Windows, so no recorded identity could be refuted there and the identity gate in front of `taskkill /T /F` only turned away a PID that was not running at all. A service that died by itself, whose PID an unrelated process then took, still passed the gate, and the stop took that process and its whole subtree. Clearing the record on a confirmed stop closed the repeat-stop route but not this one. On Windows the token is now the process creation time from GetProcessTimes, a FILETIME that only has to be equal to the value read when the PID was recorded. A record carrying one refutes a recycled PID exactly as on Linux: terminate_recorded_pid reads it back as Recycled, skips the tree kill and reports identity_mismatch. A record written before this has no token and stays best-effort. A recorded token that can no longer be read (the process is more privileged) reads as Indeterminate, and the stop reports it unverified instead of signalling it. That outcome is now reachable on Windows, and its advice named `ps -p`, which Windows does not have; the check it names is now `tasklist /FI "PID eq "` there. The FILETIME join is a pure function tested on every host. The refusal runs against the test process's own PID with a wrong token on Linux and Windows, and against a bystander process on Windows, where a stop that went ahead would run taskkill. Scenario service-stop-09 stops a server whose recorded PID belongs to a running bystander under a token no live process has and checks the bystander is still running; it runs on every OS. Signed-off-by: Jussi Elo --- apps/rocm/src/service_control.rs | 5 +- apps/rocm/src/service_launch.rs | 8 +- apps/rocm/src/services_text.rs | 39 ++++- apps/rocmd/src/service.rs | 9 +- crates/rocm-core/src/proc_lifecycle.rs | 161 ++++++++++++++++-- docs/architecture.md | 2 +- docs/testing.md | 21 ++- .../features/service_stop.feature | 25 ++- .../tests/e2e/service_stop_steps.rs | 44 +++++ 9 files changed, 284 insertions(+), 30 deletions(-) diff --git a/apps/rocm/src/service_control.rs b/apps/rocm/src/service_control.rs index e81823817b..2dd52108da 100644 --- a/apps/rocm/src/service_control.rs +++ b/apps/rocm/src/service_control.rs @@ -259,7 +259,8 @@ fn stop_internal_managed_service_confirmed_with( // The processes this stop confirmed gone are not the service's any more, // and their PIDs are stale the instant they exited. Left in the record, // a later stop would signal them again: a PID recorded without a - // start-time token (every one on Windows) cannot be refuted, so if the + // start-time token (every one a Windows record had before it recorded + // a creation time) cannot be refuted, so if the // OS had given it to another process the Windows tree kill would take // that process and its subtree. The same call, on the same confirmed // path, as `rocmd`'s stop. @@ -1412,7 +1413,7 @@ mod tests { let rendered = format!("{:#}", outcome.expect_err("unconfirmed")); assert_no_retry_line(&rendered); assert!( - rendered.contains("check: `ps -p 888888888 -o pid,args`"), + rendered.contains(&format!("check: `{}`", process_check_command(888_888_888))), "{rendered}" ); assert!(!rendered.contains("the record changed"), "{rendered}"); diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index f752e48e3d..0ae29477d8 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -2678,7 +2678,13 @@ mod tests { finish("attached-unverified", Ok(stop_answer(false, "unverified"))); let error = format!("{:#}", result.expect_err("an unverified stop fails")); assert!(error.contains("not signaled: process 4242"), "{error}"); - assert!(error.contains("check: `ps -p 4242 -o pid,args`"), "{error}"); + assert!( + error.contains(&format!( + "check: `{}`", + crate::services_text::process_check_command(4242) + )), + "{error}" + ); crate::services_text::tests::assert_no_retry_line(&error); assert!(out.is_empty(), "{out}"); assert_eq!(signalled, None); diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index f1bee48f29..9fa4f1063e 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -1140,11 +1140,22 @@ fn render_unconfirmed_service_stop(output: &serde_json::Value) -> String { /// stop left alone because it could not verify it. Not a `rocm` command: the /// CLI deliberately does not act on that process, so the user decides. /// -/// A Linux command because only Linux can produce that outcome: it needs a -/// recorded start-time that can no longer be read back, and -/// `rocm_core::process_start_ticks` records one only on Linux. +/// Linux and Windows can both produce that outcome: it needs a recorded +/// start-time that can no longer be read back, and `rocm_core::process_start_ticks` +/// records one on both. So the command is the one the host has: `ps` on Linux, +/// `tasklist` on Windows. pub(crate) fn process_check_command(pid: u64) -> String { - format!("ps -p {pid} -o pid,args") + process_check_command_for(cfg!(windows), pid) +} + +/// [`process_check_command`] for a chosen platform, so a test reads both forms +/// on every host. +fn process_check_command_for(windows: bool, pid: u64) -> String { + if windows { + format!("tasklist /FI \"PID eq {pid}\"") + } else { + format!("ps -p {pid} -o pid,args") + } } /// A managed service's record exactly as it is on disk, without the engine-state @@ -2812,4 +2823,24 @@ pub(crate) mod tests { })); assert_eq!(answer["status"], "stop_unconfirmed", "{answer}"); } + + /// The check a stop that could not verify a process tells the user to run + /// is a command the host has: `ps` on Linux, `tasklist` on Windows, where a + /// Windows record now carries a creation time that can fail to read back. + /// The wording is pinned here; that the Windows form shows whether a PID is + /// running is what the `service-stop-07` steps rely on, which look a PID up + /// with the same `tasklist /FI "PID eq "` filter. + #[test] + fn the_process_check_command_is_the_one_the_host_has() { + assert_eq!(process_check_command_for(false, 42), "ps -p 42 -o pid,args"); + assert_eq!( + process_check_command_for(true, 42), + "tasklist /FI \"PID eq 42\"" + ); + assert_eq!( + process_check_command(42), + process_check_command_for(cfg!(windows), 42), + "production picks by the host it runs on" + ); + } } diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index 467ad5c189..441c0b696d 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -327,10 +327,11 @@ mod tests { /// a different process: the recorded service already exited and its PID was /// recycled. The stop must refuse it rather than signal a stranger. /// - /// Linux-only: a refutable identity needs a readable start-time, and - /// `rocm_core::process_start_ticks` reads it from `/proc`. On Windows it is - /// always `None`, so no identity can be refuted there and the state this - /// asserts on cannot be constructed. + /// Linux-only: it stages the bystander with `sleep` and offsets its token + /// read from `/proc`. The same refusal on Windows, where the token is the + /// process creation time, is asserted in `rocm-core`'s `proc_lifecycle` + /// tests (`terminate_recorded_pid_refuses_a_pid_whose_recorded_identity_no_longer_matches` + /// and its Windows-only neighbour) and by scenario `service-stop-09`. #[cfg(target_os = "linux")] #[test] fn stop_managed_service_refuses_a_pid_whose_recorded_identity_no_longer_matches() -> Result<()> diff --git a/crates/rocm-core/src/proc_lifecycle.rs b/crates/rocm-core/src/proc_lifecycle.rs index bfa14c1989..feb183a081 100644 --- a/crates/rocm-core/src/proc_lifecycle.rs +++ b/crates/rocm-core/src/proc_lifecycle.rs @@ -32,8 +32,10 @@ pub enum KillScope { /// A spawned process identified in a way that is robust to PID recycling. /// /// `start_ticks` is the kernel start-time of the process (clock ticks since -/// boot). It is `None` on platforms without `/proc`, where identity cannot be -/// verified and callers fall back to best-effort behaviour. +/// boot) on Linux, and its creation time as a `FILETIME` on Windows. It is +/// `None` on other platforms, and for a record written before Windows recorded +/// one, where identity cannot be verified and callers fall back to best-effort +/// behaviour. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ProcessIdentity { pub pid: u32, @@ -431,13 +433,16 @@ pub fn terminate_recorded_service_pids( /// launches it through an `__engine-serve-http` process: a root-only kill would /// leave the engine running and still holding the device. /// -/// Identity verification does not reach that far on Windows. -/// [`process_start_ticks`] has no `/proc` to read there and returns `None`, so -/// no recorded identity can be refuted and the [`IdentityState::Matches`] gate -/// only ever turns away a PID that is not running at all. A recorded PID that -/// has been recycled by an unrelated process still passes it, and `taskkill /T -/// /F` then takes that process and its whole subtree. Closing that needs a -/// Windows process creation time in [`process_start_ticks`]. +/// The tree kill runs only for a PID that passes the same identity gate as the +/// rest of the stop. A record written with a start-time token carries the +/// process's creation time on Windows ([`process_start_ticks`]), so a PID that +/// an unrelated process has since taken reads back [`IdentityState::Recycled`] +/// and neither it nor its subtree is touched. A record written before Windows +/// recorded a token has none to refute, and for it the gate only turns away a +/// PID that is not running at all: that old record can still name a recycled +/// PID, and `taskkill /T /F` then takes that process and its subtree. A stop +/// that confirms clears the record's PIDs, so that exposure shrinks to the +/// records written before the token existed and not yet stopped. pub fn terminate_recorded_pid(identity: &ProcessIdentity) -> TerminationOutcome { terminate_recorded_pid_with(identity, platform_tree_kill(), identity_state, |identity| { terminate_verified(identity, KillScope::Tree, MANAGED_STOP_GRACE, true) @@ -539,8 +544,9 @@ fn send_signal(pid: u32, signal: Signal, _tree: bool) -> bool { /// Read the kernel start-time (field 22 of `/proc//stat`) for `pid`. /// -/// Returns `None` when the value cannot be read, including on non-Linux -/// platforms, where identity verification degrades to best-effort. +/// Returns `None` when the value cannot be read, including on platforms with +/// neither `/proc` nor a creation time to read (neither Linux nor Windows), +/// where identity verification degrades to best-effort. #[cfg(target_os = "linux")] #[must_use] pub fn process_start_ticks(pid: u32) -> Option { @@ -548,7 +554,56 @@ pub fn process_start_ticks(pid: u32) -> Option { parse_start_ticks(&stat) } -#[cfg(not(target_os = "linux"))] +/// Read the creation time of `pid` (`GetProcessTimes`) as the token that tells +/// the process from a later one that reuses its PID. +/// +/// The value is the creation time as a Windows `FILETIME` (100 ns intervals +/// since 1601), not clock ticks since boot: only equality with a value read +/// earlier from the same PID means anything. `None` when the process cannot be +/// opened for querying (it is gone, or belongs to a more privileged account). +#[cfg(windows)] +#[must_use] +#[allow(unsafe_code)] // Win32 FFI +pub fn process_start_ticks(pid: u32) -> Option { + use windows_sys::Win32::Foundation::{CloseHandle, FILETIME}; + use windows_sys::Win32::System::Threading::{ + GetProcessTimes, OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION, + }; + + if pid == 0 { + return None; + } + // SAFETY: plain Win32 calls. The handle is checked before use and closed on + // every path, and the four out-parameters are valid for the one call. + unsafe { + let handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid); + if handle.is_null() { + return None; + } + let mut creation = FILETIME::default(); + let mut exit = FILETIME::default(); + let mut kernel = FILETIME::default(); + let mut user = FILETIME::default(); + let read = GetProcessTimes( + handle, + &raw mut creation, + &raw mut exit, + &raw mut kernel, + &raw mut user, + ); + CloseHandle(handle); + (read != 0).then(|| filetime_ticks(creation.dwLowDateTime, creation.dwHighDateTime)) + } +} + +/// The 64-bit value of a Windows `FILETIME`, which a Win32 call returns as two +/// 32-bit halves. Kept apart from the FFI so a test can run it on every host. +#[cfg(any(windows, test))] +fn filetime_ticks(low: u32, high: u32) -> u64 { + (u64::from(high) << 32) | u64::from(low) +} + +#[cfg(not(any(target_os = "linux", windows)))] #[must_use] // Platform stub. The `cfg` sibling calls non-const code, so making only this // arm `const` would give the two platforms different signatures and push @@ -684,7 +739,7 @@ mod tests { let _ = child.wait(); } - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", windows))] #[test] fn identity_matches_our_own_process() { let pid = std::process::id(); @@ -693,7 +748,7 @@ mod tests { assert_eq!(identity_state(&id), IdentityState::Matches); } - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", windows))] #[test] fn identity_mismatch_when_start_ticks_differ() { // Same live PID (ours), but a different recorded start-time: a recycled @@ -713,7 +768,7 @@ mod tests { assert_eq!(identity_state(&id), IdentityState::Gone); } - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", windows))] #[test] fn refuses_to_signal_on_identity_mismatch() { // Our own PID with a wrong start-time. A forced stop must NOT kill us: @@ -1178,4 +1233,80 @@ mod tests { reap(child); assert!(!crate::process_is_running(id.pid)); } + + #[test] + fn a_filetime_is_its_high_word_above_its_low_word() { + // The Windows creation time arrives as two 32-bit halves; swapping them + // or dropping the shift would make every process read as a different one. + assert_eq!( + filetime_ticks(0x89AB_CDEF, 0x0123_4567), + 0x0123_4567_89AB_CDEF + ); + assert_eq!(filetime_ticks(1, 0), 1); + assert_eq!(filetime_ticks(0, 1), 1 << 32); + assert_eq!(filetime_ticks(u32::MAX, u32::MAX), u64::MAX); + } + + /// The managed stop's own entry point refuses a PID whose recorded identity no + /// longer matches, and reaches neither it nor its tree. Our own PID with a + /// wrong start-time is what a recycled PID looks like; if the stop went + /// ahead, `taskkill /T /F` on Windows (a plain forced kill elsewhere) would + /// end this test process instead of asserting. On Windows this runs the real + /// tree-kill gate: the verdict has to come from the identity check, before + /// any `taskkill`. + #[cfg(any(target_os = "linux", windows))] + #[test] + fn terminate_recorded_pid_refuses_a_pid_whose_recorded_identity_no_longer_matches() { + let pid = std::process::id(); + let real = process_start_ticks(pid).expect("own start-time"); + let stale = ProcessIdentity::new(pid, Some(real.wrapping_add(1))); + + let outcome = terminate_recorded_pid(&stale); + + assert_eq!(outcome, TerminationOutcome::IdentityMismatch); + assert!(crate::process_is_running(pid), "nothing was signalled"); + } + + /// The same refusal against a bystander that is a different process holding + /// the recorded PID (the recycled case), then the stop of that process when + /// its identity does match. Windows only: that is where a tree kill by PID + /// follows the identity gate, and where a creation-time token is read. + #[cfg(windows)] + #[test] + fn terminate_recorded_pid_leaves_a_recycled_pid_alone_and_stops_the_verified_one() { + use std::process::{Command, Stdio}; + + let mut child = Command::new("ping") + .args(["-n", "120", "127.0.0.1"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn test child"); + let pid = child.id(); + let real = process_start_ticks(pid).expect("a live process has a creation time"); + assert_eq!( + process_start_ticks(pid), + Some(real), + "the creation time of one process does not change" + ); + assert_ne!( + Some(real), + process_start_ticks(std::process::id()), + "two processes do not share one" + ); + + let outcome = + terminate_recorded_pid(&ProcessIdentity::new(pid, Some(real.wrapping_add(1)))); + assert_eq!(outcome, TerminationOutcome::IdentityMismatch); + assert!( + child.try_wait().expect("check the bystander").is_none(), + "a PID whose recorded creation time differs is a stranger and must keep running" + ); + + let outcome = terminate_recorded_pid(&ProcessIdentity::capture(pid)); + assert!(outcome.stopped(), "{outcome:?}"); + let _ = child.wait(); + assert!(!crate::process_is_running(pid)); + } } diff --git a/docs/architecture.md b/docs/architecture.md index 4679d09d12..f428fb722b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -231,7 +231,7 @@ Further `rocm-core` modules, each with its own doc comment except where noted: - [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) — OpenMPI, libatomic and libnuma detection, the `libmpi_cxx` compatibility shim, and the per-library install plans (`build_openmpi_install_plan`, `build_libatomic_install_plan`, `build_libnuma_install_plan`) and hints, needed by vLLM's ROCm wheels. The distro-generic half of that planning lives in [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs); [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) re-exports its public items, so `rocm_core::openmpi::running_as_root`, `plan_for_os_release_read` and the rest keep resolving, and `pub(crate)` re-exports `PackageManager` and `resolve_package_manager` for [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs). - [`os_release.rs`](../crates/rocm-core/src/os_release.rs) — the one `/etc/os-release` parser every reader of that file uses. - [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs) — the planning half of a system-package install that does not depend on which library is installed: `PackageManager` and `resolve_package_manager` (from an os-release `ID`/`ID_LIKE`), `SystemPackageInstallPlan` and `InstallCommand`, `plan_for_os_release_at`/`plan_for_os_release_read` (which take the per-library builder as an argument), `InstallHint`, the root and passwordless-`sudo` checks (`running_as_root`, `can_autoinstall`) and the `apt-get -s` helpers (`apt_simulate_argv`, `parse_apt_simulate_removals`, `is_protected_rocm_package`). It never runs anything and depends on no other module of this crate except [`os_release.rs`](../crates/rocm-core/src/os_release.rs) and [`hardware_root.rs`](../crates/rocm-core/src/hardware_root.rs), which it reads os-release through. The module is private; its public items are re-exported from [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs). It is a different module from [`system_packages.rs`](../apps/rocm/src/system_packages.rs) in `apps/rocm`, which runs these plans with consent and `sudo`. -- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches (`terminate_verified`), and the one managed-stop routine built on it that `rocm services stop` and `rocmd`'s `stop_server` both call: `terminate_recorded_service_pids` walks the processes a service record names (supervisor, then engine, each with its own start-time, the stopping process's own PID reported as `self` and never signalled) and returns one `RecordedPidStop` per PID (each caller, on a stop that confirms every one gone, then clears the record's recorded processes with `ManagedServiceRecord::clear_recorded_processes` in [`service_record.rs`](../crates/rocm-core/src/service_record.rs), so a repeated stop reaches no stale PID; an unconfirmed stop keeps them); `terminate_recorded_pid` terminates one of them, adding on Windows a `taskkill /T /F` of its descendants first, because `terminate_verified` cannot walk a process tree there; `MANAGED_STOP_GRACE` is the one grace period. The termination is injectable, so a caller's test supplies outcomes it cannot construct and a test of the Windows step runs on every host. +- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches (`terminate_verified`), and the one managed-stop routine built on it that `rocm services stop` and `rocmd`'s `stop_server` both call: `terminate_recorded_service_pids` walks the processes a service record names (supervisor, then engine, each with its own start-time, the stopping process's own PID reported as `self` and never signalled) and returns one `RecordedPidStop` per PID (each caller, on a stop that confirms every one gone, then clears the record's recorded processes with `ManagedServiceRecord::clear_recorded_processes` in [`service_record.rs`](../crates/rocm-core/src/service_record.rs), so a repeated stop reaches no stale PID; an unconfirmed stop keeps them); `terminate_recorded_pid` terminates one of them, adding on Windows a `taskkill /T /F` of its descendants first, because `terminate_verified` cannot walk a process tree there, and only for a PID that passes the same identity check (`process_start_ticks` is the kernel start-time on Linux and the process creation time on Windows, so a recycled PID is refused before any `taskkill`; a record written before Windows recorded one cannot be refuted); `MANAGED_STOP_GRACE` is the one grace period. The termination is injectable, so a caller's test supplies outcomes it cannot construct and a test of the Windows step runs on every host. - [`report.rs`](../crates/rocm-core/src/report.rs) — the content of a Doctor report and the rule deciding whether one may exist; it sends nothing. - [`report_delivery.rs`](../crates/rocm-core/src/report_delivery.rs) — how a report leaves the machine: building its link and deciding whether a browser may be started. - [`runtime.rs`](../crates/rocm-core/src/runtime.rs) — the host-platform layer: `RuntimePlatform`/`RuntimeHost`, OS-specific executable and Python-environment names (including `runtime_python_env_root_for_executable`, the one rule for finding an environment's folder from its interpreter's path: only an interpreter directly inside a `bin` or `Scripts` folder, in any case, has one, and everything else gets `None`; the adopt command, the ComfyUI runtime environment and the vLLM launch all call it rather than keeping a copy, since the engine crate cannot reach `apps/rocm`), default config/data/cache directories, path-text and path-list helpers, and `install_root_has_payload`, the one definition of a non-empty install root (dot-entries such as the runtime marker do not count; a path that does not exist holds nothing; a root that cannot be listed is an error) that tarball extraction, update repair detection and activation validation in `apps/rocm` all ask (no module doc comment). diff --git a/docs/testing.md b/docs/testing.md index 2491854c90..ea563e7be1 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1808,6 +1808,25 @@ work: with the `taskkill /T` step removed the stop ends only the recorded process and the scenario fails on the surviving child. `service-stop-01` plants PID 0 and `service-stop-02` is Linux-only, so neither reaches that step; the injected unit tests above cover its ordering and verdicts on every host. + +The tree kill is behind the same identity gate as the rest of the stop. On +Windows `process_start_ticks` is the process creation time (`GetProcessTimes`, a +`FILETIME`), so a record written by this version refutes a PID that an unrelated +process has taken, and the stop reports `identity_mismatch` without running +`taskkill` on it; only a record written before Windows recorded a token has +nothing to refute. `a_filetime_is_its_high_word_above_its_low_word` runs the +conversion on every host. `terminate_recorded_pid_refuses_a_pid_whose_recorded_identity_no_longer_matches` +runs the real stop against this test's own PID with a wrong token on Linux and +Windows (a stop that went ahead would end the test process), and +`terminate_recorded_pid_leaves_a_recycled_pid_alone_and_stops_the_verified_one` +repeats it against a bystander on Windows only and then stops that bystander +with its true token. `service-stop-09` is the live version on every OS: it +records a running process under a token no live process has, stops the server, +and asserts that the process is still running and the record names no process. +Neither Windows-only test nor the Windows half of `service-stop-09` ran where +this was written; the Windows CI jobs are what exercise them. A stop that cannot +verify a process names `ps -p -o pid,args` on Linux and `tasklist /FI "PID +eq "` on Windows (`process_check_command`). `service-stop-06` (Linux) is the same stop of a real recorded process through `rocm services stop`, checking the persisted state: the process exited and the record names none. Run the Windows scenario on a Windows host with @@ -1852,7 +1871,7 @@ checks that the line omits it and no key file exists. Because the seam replaces `terminate_verified`, they say nothing about what is signalled. They need a hooks build, as `cargo xtask e2e` makes: `cargo build -p rocm -p rocmd --features rocm/e2e-test-hooks`, then the command above with `-n -"service-stop-0[1-6]"`. Not `-n "service-stop"`: a name filter bypasses the +"service-stop-0[1-69]"`. Not `-n "service-stop"`: a name filter bypasses the suite's own host filter, so it would also run the Windows-only `service-stop-07` on a Linux host, where it fails for want of PowerShell. diff --git a/tests/e2e-cucumber/features/service_stop.feature b/tests/e2e-cucumber/features/service_stop.feature index dfbc60b11d..9a0a9b8882 100644 --- a/tests/e2e-cucumber/features/service_stop.feature +++ b/tests/e2e-cucumber/features/service_stop.feature @@ -155,8 +155,10 @@ Feature: Stopping a managed service # checks that BOTH are gone together with the report and the record. Without # the tree-kill step the stop ends only the recorded process, the child keeps # running, and the scenario fails on it. No GPU, no engine, no network. The - # record carries no start-time token, because Windows has none: the case the - # tree kill and the cleared record exist for. + # record carries no start-time token, like a Windows record written before the + # token was recorded: the case where the PID cannot be refuted, which the tree + # kill and the cleared record exist for. service-stop-09 below is the same stop + # with a token that does not match. @id:service-stop-takes-down-the-whole-process-tree-on-windows @requires-os:windows @@ -168,3 +170,22 @@ Feature: Stopping a managed service And the recorded process and the process it started are no longer running And the server's record reads stopped and names no process And the server's endpoint key file is gone + + # A recorded PID is not proof that the recorded process still holds it: the + # server can exit by itself and the OS hand its PID to an unrelated program. + # The record pairs the PID with the process's start-time token (Linux's + # `/proc` start time, Windows' creation time), and a stop whose token no longer + # matches treats the recorded process as already gone: it reports the stop, + # clears the record and leaves the process that now holds the PID running, + # instead of signalling a stranger (on Windows, with `taskkill /T /F`, its + # whole subtree). The stranger here is a real process, so "left alone" is + # checked on the process itself. Runs on every OS: the planted token is one + # no live process has. + @id:service-stop-leaves-a-process-that-reused-the-recorded-pid-alone + Scenario: service-stop-09 - Stopping a managed server whose recorded PID now belongs to another process leaves that process running + Given a managed server whose recorded process id now belongs to a different process + When the user stops it with --yes + Then the CLI reports the service as stopped + And the process now holding the recorded process id is still running + And the server's record reads stopped and names no process + And the server's endpoint key file is gone diff --git a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs index 798354e5e1..0bca2e0fdf 100644 --- a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs @@ -170,6 +170,50 @@ fn plant_server_with_running_process(world: &mut E2eWorld, host: &str, with_key: plant_server_record(world, host, with_key, process, Some(ticks)); } +/// A ready, publicly bound server whose recorded PID is a live process that is +/// not the server: what a recycled PID looks like. The record's start-time token +/// is not the process's own, so the stop has to refuse the PID rather than signal +/// it. The token is the process's real one offset by one on Linux, and `1` (a +/// `FILETIME` in 1601, which no live process has) on Windows. +#[given("a managed server whose recorded process id now belongs to a different process")] +async fn managed_server_with_a_recycled_pid(world: &mut E2eWorld) { + let (program, args): (&str, &[&str]) = if cfg!(windows) { + ("ping", &["-n", "300", "127.0.0.1"]) + } else { + ("sleep", &["300"]) + }; + let bystander = Command::new(program) + .args(args) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("failed to start the process that holds the recorded PID"); + let stale_ticks = if cfg!(windows) { + 1 + } else { + start_ticks(bystander.id()).wrapping_add(1) + }; + plant_server_record(world, "0.0.0.0", true, bystander, Some(stale_ticks)); +} + +/// The stop's promise about the process that holds the recorded PID, checked on +/// the process: it was not signalled, so it is still running. +#[then("the process now holding the recorded process id is still running")] +async fn process_holding_the_pid_still_runs(world: &mut E2eWorld) { + let process = world + .recorded_process + .as_mut() + .expect("scenario recorded no process"); + assert!( + process + .try_wait() + .expect("failed to check the recorded process") + .is_none(), + "the stop signalled a process that is not the recorded server" + ); +} + /// Where the parent the Windows scenario records writes the PID of the process /// it started, for the scenario to read. fn descendant_pid_path(world: &E2eWorld) -> PathBuf { From f01721fcc10f5fb7a1e5317c353c5dcc7aeb10f8 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 20:33:41 +0000 Subject: [PATCH 07/30] fix(core): decide a stop's identity verdict in one place that a table covers for every recorded and live token A Windows lane run showed service-stop-07, which records a PowerShell parent with no start-time token, ending with the recorded process still running after a stop that reported success. Reading the stop path, a record with no token already gets the whole stop (identity_state reads it as a match, so the tree kill and the verified termination run); I could not run Windows code here and did not find a path in the creation-time change that skips it. What the change did leave was that this decision was spread over identity_state and terminate_verified with the system reads mixed in. The decision now lives in two pure functions, classify_live_identity (a process is refused only when a token was recorded and the live one is provably different; no recorded token is a match; a recorded token that cannot be read back is indeterminate) and refusal_before_signalling, which identity_state and terminate_verified use. A table test drives terminate_recorded_pid_with through them for no token (the shape service-stop-07 plants), a matching token, a differing token and an unreadable one, asserting whether the tree kill runs and the verdict label together; each row was shown red by a targeted mutation of the classification. A Windows-only test stops a real process recorded without a token. The step that checks the recorded process has exited now allows its handle a few seconds to signal, since the stop confirms an exit from the OS exit code; a survivor still fails it. Signed-off-by: Jussi Elo --- crates/rocm-core/src/proc_lifecycle.rs | 179 +++++++++++++++++- docs/testing.md | 7 +- .../tests/e2e/service_stop_steps.rs | 26 ++- 3 files changed, 197 insertions(+), 15 deletions(-) diff --git a/crates/rocm-core/src/proc_lifecycle.rs b/crates/rocm-core/src/proc_lifecycle.rs index feb183a081..831c9ef5a0 100644 --- a/crates/rocm-core/src/proc_lifecycle.rs +++ b/crates/rocm-core/src/proc_lifecycle.rs @@ -90,7 +90,21 @@ pub fn identity_state(id: &ProcessIdentity) -> IdentityState { if !crate::process_is_running(id.pid) || process_has_exited(id.pid) { return IdentityState::Gone; } - match (id.start_ticks, process_start_ticks(id.pid)) { + classify_live_identity(id.start_ticks, process_start_ticks(id.pid)) +} + +/// The identity verdict for a PID that is running: what its recorded token +/// (`recorded`) says about the token read from the live process now (`live`). +/// +/// The one decision every stop path turns on, apart from the system reads, so +/// a test covers each case on every host. A process is refused only when a +/// token was recorded and the live one is provably different. A record with no +/// token (an older record, or one written where no token could be read) cannot +/// be refuted and is a match whatever the live process reads, so its stop and +/// its tree kill go ahead as they always did. A recorded token that cannot be +/// confirmed now is neither signalled nor reported stopped. +const fn classify_live_identity(recorded: Option, live: Option) -> IdentityState { + match (recorded, live) { (Some(expected), Some(actual)) => { if expected == actual { IdentityState::Matches @@ -106,6 +120,18 @@ pub fn identity_state(id: &ProcessIdentity) -> IdentityState { } } +/// The verdict that ends a termination before anything is signalled, when the +/// recorded identity says not to: `None` for a process that matches and so goes +/// on to be stopped. +const fn refusal_before_signalling(state: IdentityState) -> Option { + match state { + IdentityState::Gone => Some(TerminationOutcome::AlreadyGone), + IdentityState::Recycled => Some(TerminationOutcome::IdentityMismatch), + IdentityState::Indeterminate => Some(TerminationOutcome::Unverified), + IdentityState::Matches => None, + } +} + /// Whether `state` means the recorded process is definitively no longer running. /// /// [`IdentityState::Indeterminate`] is intentionally excluded: a process we can @@ -196,11 +222,8 @@ pub fn terminate_verified( grace: Duration, force: bool, ) -> TerminationOutcome { - match identity_state(id) { - IdentityState::Gone => return TerminationOutcome::AlreadyGone, - IdentityState::Recycled => return TerminationOutcome::IdentityMismatch, - IdentityState::Indeterminate => return TerminationOutcome::Unverified, - IdentityState::Matches => {} + if let Some(verdict) = refusal_before_signalling(identity_state(id)) { + return verdict; } let tree = matches!(scope, KillScope::Tree); @@ -1199,6 +1222,123 @@ mod tests { assert_eq!(verdict, TerminationOutcome::AlreadyGone); } + /// Every way a recorded token and the live process's token can stand, with + /// what the stop does about each: whether the Windows tree kill runs, and the + /// verdict the stop reports (the label and whether it counts as stopped). + /// + /// The live process is running in every row; the tokens go through the same + /// decision production uses ([`classify_live_identity`] and + /// [`refusal_before_signalling`]), with the system reads and the signalling + /// supplied, so each row runs on every host. A process is refused only when + /// a token was recorded and the live one is provably different. + #[test] + fn what_a_stop_does_for_each_recorded_and_live_token() { + const LIVE: u64 = 0x01DC_0000_0000_0001; + struct Row { + name: &'static str, + recorded: Option, + live: Option, + tree_kill_runs: bool, + label: &'static str, + stopped: bool, + } + let row = |name, recorded, live, tree_kill_runs, label, stopped| Row { + name, + recorded, + live, + tree_kill_runs, + label, + stopped, + }; + let rows = [ + // The shape `service-stop-07` records, and any record older than the + // token: nothing to refute, so the whole tree is taken as before. + row("no token recorded", None, Some(LIVE), true, "forced", true), + row( + "no token recorded, none readable", + None, + None, + true, + "forced", + true, + ), + row( + "token matches", + Some(LIVE), + Some(LIVE), + true, + "forced", + true, + ), + // A recycled PID: the process is a stranger, left alone. + row( + "token differs", + Some(LIVE + 1), + Some(LIVE), + false, + "identity_mismatch", + true, + ), + // A recorded token that cannot be read back is neither signalled nor + // reported stopped. + row( + "token unreadable", + Some(LIVE), + None, + false, + "unverified", + false, + ), + ]; + for Row { + name: row, + recorded, + live, + tree_kill_runs, + label, + stopped, + } in rows + { + let steps = RefCell::new(Vec::new()); + let tree_kill = |pid: u32| steps.borrow_mut().push(Step::TreeKill(pid)); + let verdict = terminate_recorded_pid_with( + &ProcessIdentity::new(SUPERVISOR_PID, recorded), + Some(&tree_kill as &dyn Fn(u32)), + |id| classify_live_identity(id.start_ticks, live), + |id| { + steps.borrow_mut().push(Step::Terminate(id.pid)); + // What the verified termination answers: its refusal for a + // process the identity rules out, otherwise the tree kill + // has already taken the root and it reads back as gone. + refusal_before_signalling(classify_live_identity(id.start_ticks, live)) + .unwrap_or(TerminationOutcome::AlreadyGone) + }, + ); + let expected_steps = if tree_kill_runs { + vec![ + Step::TreeKill(SUPERVISOR_PID), + Step::Terminate(SUPERVISOR_PID), + ] + } else { + vec![Step::Terminate(SUPERVISOR_PID)] + }; + assert_eq!(steps.into_inner(), expected_steps, "{row}: steps"); + assert_eq!(verdict.as_str(), label, "{row}: verdict"); + assert_eq!(verdict.stopped(), stopped, "{row}: counts as stopped"); + } + } + + #[test] + fn a_recorded_token_refutes_a_live_one_only_when_both_are_known_and_differ() { + use IdentityState::{Indeterminate, Matches, Recycled}; + assert_eq!(classify_live_identity(None, None), Matches); + assert_eq!(classify_live_identity(None, Some(7)), Matches); + assert_eq!(classify_live_identity(Some(7), Some(7)), Matches); + assert_eq!(classify_live_identity(Some(7), Some(8)), Recycled); + assert_eq!(classify_live_identity(Some(7), None), Indeterminate); + assert_eq!(refusal_before_signalling(Matches), None); + } + #[test] fn only_windows_has_a_separate_tree_kill_step() { // Ties the injected tests above to what production actually runs: the @@ -1309,4 +1449,31 @@ mod tests { let _ = child.wait(); assert!(!crate::process_is_running(pid)); } + + /// A record with no token (written before Windows recorded one, and the shape + /// `service-stop-07` plants) cannot be refuted and still gets the whole stop: + /// the process ends. Windows only, where a real process is stopped by PID. + #[cfg(windows)] + #[test] + fn terminate_recorded_pid_stops_a_process_recorded_without_a_token() { + use std::process::{Command, Stdio}; + + let mut child = Command::new("ping") + .args(["-n", "120", "127.0.0.1"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn test child"); + let pid = child.id(); + + let outcome = terminate_recorded_pid(&ProcessIdentity::new(pid, None)); + + assert!(outcome.stopped(), "{outcome:?}"); + assert_eq!(outcome.as_str(), "forced"); + assert!( + child.wait().is_ok() && !crate::process_is_running(pid), + "the process recorded without a token must be gone" + ); + } } diff --git a/docs/testing.md b/docs/testing.md index ea563e7be1..856f819a65 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1814,7 +1814,12 @@ Windows `process_start_ticks` is the process creation time (`GetProcessTimes`, a `FILETIME`), so a record written by this version refutes a PID that an unrelated process has taken, and the stop reports `identity_mismatch` without running `taskkill` on it; only a record written before Windows recorded a token has -nothing to refute. `a_filetime_is_its_high_word_above_its_low_word` runs the +nothing to refute: a stop refuses a PID only when a token was recorded and the +live one differs, and `what_a_stop_does_for_each_recorded_and_live_token` pins +that for every case on every host (no token, which is also the shape +`service-stop-07` records; a token that matches; one that differs; one that +cannot be read back), asserting whether the tree kill runs and the verdict the +stop reports together. `a_filetime_is_its_high_word_above_its_low_word` runs the conversion on every host. `terminate_recorded_pid_refuses_a_pid_whose_recorded_identity_no_longer_matches` runs the real stop against this test's own PID with a wrong token on Linux and Windows (a stop that went ahead would end the test process), and diff --git a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs index 0bca2e0fdf..d0b2536c83 100644 --- a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs @@ -500,14 +500,24 @@ fn assert_recorded_process_exited(world: &mut E2eWorld) { .as_mut() .expect("scenario recorded no process"); // The report's claim, checked against the process itself: it has exited - // (and is reaped here), rather than merely having been signalled. - let exited = process - .try_wait() - .expect("failed to check the recorded process"); - assert!( - exited.is_some(), - "the stop reported the process gone, but it is still running" - ); + // (and is reaped here), rather than merely having been signalled. The stop + // confirms an exit from the OS's exit code, and this handle is signalled by + // the process object's teardown, so allow that teardown a few seconds on a + // loaded host before calling a process a survivor. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + loop { + let exited = process + .try_wait() + .expect("failed to check the recorded process"); + if exited.is_some() { + break; + } + assert!( + std::time::Instant::now() < deadline, + "the stop reported the process gone, but it is still running" + ); + std::thread::sleep(std::time::Duration::from_millis(200)); + } } #[then("the server's record reads stopped and names no process")] From e1ded955fb824c592f4fec85602ff0a11a77cba0 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 20:47:32 +0000 Subject: [PATCH 08/30] test(e2e): end the process a stop scenario's recorded process started, even when the scenario fails service-stop-07 records a PowerShell parent that has started a ping child. If a stop kills the parent but leaves the child, which is the regression the scenario exists to catch, the world's teardown only killed the recorded parent and a later stop cannot recover a child from a dead parent, so ping ran on for its five minutes. The Given step now keeps the child's identity (its PID with its creation time where the host reads one) and teardown, which runs when a step fails too, ends it when it is still that process. Whether to end it is a pure function, stray_process::leftover_action, with a table test that runs on every host: only a process that is still the recorded one is ended, never one that is gone, whose PID has been reused, or whose identity cannot be confirmed; each row was shown red by a targeted mutation. The Windows clippy check compiles the teardown; the scenario itself has not run on Windows here. Signed-off-by: Jussi Elo --- docs/testing.md | 8 +- tests/e2e-cucumber/src/lib.rs | 1 + tests/e2e-cucumber/src/stray_process.rs | 75 +++++++++++++++++++ tests/e2e-cucumber/tests/e2e.rs | 21 ++++++ .../tests/e2e/service_stop_steps.rs | 4 + 5 files changed, 108 insertions(+), 1 deletion(-) create mode 100644 tests/e2e-cucumber/src/stray_process.rs diff --git a/docs/testing.md b/docs/testing.md index 856f819a65..c11ead1482 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1805,7 +1805,13 @@ services stop --yes`, and asserts that both processes are gone together with the report (`Local server stopped`, `stopped processes: 1`) and the record (stopped, no process, no stop request, key file gone). The child is the tree kill's own work: with the `taskkill /T` step removed the stop ends only the recorded -process and the scenario fails on the surviving child. `service-stop-01` plants +process and the scenario fails on the surviving child. The scenario keeps the +child's own identity (`E2eWorld::recorded_descendant`) and teardown ends it when +it is still that process, so a stop that fails to take it down does not leave +`ping` running for its five minutes behind a dead parent; the decision is +`e2e_cucumber::stray_process::leftover_action`, which a table test covers on +every host (ended only while it is still the recorded process, never once gone, +reused or unconfirmed). `service-stop-01` plants PID 0 and `service-stop-02` is Linux-only, so neither reaches that step; the injected unit tests above cover its ordering and verdicts on every host. diff --git a/tests/e2e-cucumber/src/lib.rs b/tests/e2e-cucumber/src/lib.rs index acd458f969..98e2dc99ee 100644 --- a/tests/e2e-cucumber/src/lib.rs +++ b/tests/e2e-cucumber/src/lib.rs @@ -25,6 +25,7 @@ pub mod send_until; pub mod serve_log; pub mod serve_plan; pub mod shared_runtime; +pub mod stray_process; pub mod terminal_size; use std::path::{Path, PathBuf}; diff --git a/tests/e2e-cucumber/src/stray_process.rs b/tests/e2e-cucumber/src/stray_process.rs new file mode 100644 index 0000000000..815b5a9271 --- /dev/null +++ b/tests/e2e-cucumber/src/stray_process.rs @@ -0,0 +1,75 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! What scenario teardown does about a process a scenario started and recorded. +//! +//! A scenario that records a process a stop is supposed to take down (a parent +//! and the child it started, say) can fail with one of them left running, and a +//! later stop cannot find a child whose parent is gone. Teardown therefore +//! keeps the child's own identity and ends it itself. Whether to do so is +//! decided here, apart from the system reads, so a table test covers each case +//! on every host: only a process that is still the one recorded is ended, never +//! one that has since exited or whose PID another process has taken. + +/// What the host says about a recorded process now. +/// +/// It is read against the identity the scenario captured for the process. The +/// harness maps the production identity check onto these (`rocm-core` is a +/// dev-dependency of this crate, so the library does not name it). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RecordedState { + /// Running, and the same process (or recorded without a token, which cannot + /// be refuted). + Same, + /// No longer running. + Gone, + /// Running, but provably a different process that took the PID. + Reused, + /// Running, but its identity cannot be confirmed. + Unconfirmed, +} + +/// What teardown does about one recorded process. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Leftover { + /// End it: it is still the process the scenario recorded. + Terminate, + /// Leave it: it is gone, or is not (provably) the recorded process. + Leave, +} + +/// The teardown action for a recorded process whose identity reads as `state`. +/// +/// A process that cannot be confirmed as the recorded one is left alone: ending +/// a stranger that happens to hold a recycled PID is worse than leaving a +/// stray five-minute process to time out. +#[must_use] +pub const fn leftover_action(state: RecordedState) -> Leftover { + match state { + RecordedState::Same => Leftover::Terminate, + RecordedState::Gone | RecordedState::Reused | RecordedState::Unconfirmed => Leftover::Leave, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_a_process_that_is_still_the_recorded_one_is_ended() { + for (state, expected) in [ + // Alive and the same process (or recorded without a token, which + // cannot be refuted): the stray child a failed stop left behind. + (RecordedState::Same, Leftover::Terminate), + // The stop took it down already. + (RecordedState::Gone, Leftover::Leave), + // Its PID has been reused: a stranger. + (RecordedState::Reused, Leftover::Leave), + // Its token cannot be read back, so it cannot be confirmed. + (RecordedState::Unconfirmed, Leftover::Leave), + ] { + assert_eq!(leftover_action(state), expected, "{state:?}"); + } + } +} diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index dee0ee502f..28b5dc1cb3 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -195,6 +195,11 @@ pub struct E2eWorld { /// reads the record, but a real `rocm services stop e2e-mock --yes` would /// then signal the harness process itself. pub recorded_process: Option, + /// The identity of a process the recorded one started (`service-stop-07`'s + /// `ping`). Teardown ends it on its own account, because a stop that fails + /// to take it down leaves it running with a dead parent, where nothing else + /// can find it. + pub recorded_descendant: Option, /// Whether the Doctor report preview in the dashboard console was the CLI's /// refusal envelope (it carries a `refused` key). Recorded when the preview /// is read, so a later step can check the card against what the CLI said. @@ -361,6 +366,7 @@ impl Default for E2eWorld { skill_symptom: None, helper_identity: None, recorded_process: None, + recorded_descendant: None, dash_report_refused: None, automation_daemon: None, } @@ -709,6 +715,21 @@ impl Drop for E2eWorld { let _ = process.kill(); let _ = process.wait(); } + // The process the recorded one started: ended too when it is still the + // process that was recorded, so a failed scenario does not leave it + // running until its own timeout. + if let Some(descendant) = self.recorded_descendant.take() { + use e2e_cucumber::stray_process::{Leftover, RecordedState, leftover_action}; + let state = match rocm_core::identity_state(&descendant) { + rocm_core::IdentityState::Matches => RecordedState::Same, + rocm_core::IdentityState::Gone => RecordedState::Gone, + rocm_core::IdentityState::Recycled => RecordedState::Reused, + rocm_core::IdentityState::Indeterminate => RecordedState::Unconfirmed, + }; + if leftover_action(state) == Leftover::Terminate { + let _ = rocm_core::terminate_recorded_pid(&descendant); + } + } // A scenario that enabled a watcher started a real detached `rocm daemon`, // which only exits once the watcher is disabled. If a step failed first, // nothing else would stop it, and it would keep polling a config dir diff --git a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs index d0b2536c83..eea184ed2b 100644 --- a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs @@ -242,6 +242,10 @@ async fn managed_server_whose_process_started_another(world: &mut E2eWorld) { // The premise, so the Then step cannot pass for the wrong reason: the // descendant exists and is running, and so is the parent. let descendant = wait_for_descendant_pid(&pid_file); + // Kept first, with its creation time where the host reads one, so teardown + // can end the child on its own even when the stop (or a step) fails and the + // parent is gone: a later stop cannot recover a child from a dead parent. + world.recorded_descendant = Some(rocm_core::ProcessIdentity::capture(descendant)); assert_ne!(descendant, pid, "the child must be a different process"); assert!( windows_process_is_running(descendant), From d82052d70946f032756876d70ca7adec6b5014f5 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 07:18:17 +0000 Subject: [PATCH 09/30] test(e2e): number the recycled-PID stop scenario service-stop-08 so the feature's ids stay sequential The scenario was numbered 09 to leave room for a restart scenario that belongs to a separate change, but the suite requires each feature's scenario names to be indexed sequentially in declaration order. Renumber it to 08 and update the references to it in the feature comment, docs/testing.md (including the name filter that selects the stop scenarios) and the rocmd stop test's comment. Signed-off-by: Jussi Elo --- apps/rocmd/src/service.rs | 2 +- docs/testing.md | 6 +++--- tests/e2e-cucumber/features/service_stop.feature | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index 441c0b696d..b74e66b94e 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -331,7 +331,7 @@ mod tests { /// read from `/proc`. The same refusal on Windows, where the token is the /// process creation time, is asserted in `rocm-core`'s `proc_lifecycle` /// tests (`terminate_recorded_pid_refuses_a_pid_whose_recorded_identity_no_longer_matches` - /// and its Windows-only neighbour) and by scenario `service-stop-09`. + /// and its Windows-only neighbour) and by scenario `service-stop-08`. #[cfg(target_os = "linux")] #[test] fn stop_managed_service_refuses_a_pid_whose_recorded_identity_no_longer_matches() -> Result<()> diff --git a/docs/testing.md b/docs/testing.md index c11ead1482..07148619f9 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1831,10 +1831,10 @@ runs the real stop against this test's own PID with a wrong token on Linux and Windows (a stop that went ahead would end the test process), and `terminate_recorded_pid_leaves_a_recycled_pid_alone_and_stops_the_verified_one` repeats it against a bystander on Windows only and then stops that bystander -with its true token. `service-stop-09` is the live version on every OS: it +with its true token. `service-stop-08` is the live version on every OS: it records a running process under a token no live process has, stops the server, and asserts that the process is still running and the record names no process. -Neither Windows-only test nor the Windows half of `service-stop-09` ran where +Neither Windows-only test nor the Windows half of `service-stop-08` ran where this was written; the Windows CI jobs are what exercise them. A stop that cannot verify a process names `ps -p -o pid,args` on Linux and `tasklist /FI "PID eq "` on Windows (`process_check_command`). @@ -1882,7 +1882,7 @@ checks that the line omits it and no key file exists. Because the seam replaces `terminate_verified`, they say nothing about what is signalled. They need a hooks build, as `cargo xtask e2e` makes: `cargo build -p rocm -p rocmd --features rocm/e2e-test-hooks`, then the command above with `-n -"service-stop-0[1-69]"`. Not `-n "service-stop"`: a name filter bypasses the +"service-stop-0[1-68]"`. Not `-n "service-stop"`: a name filter bypasses the suite's own host filter, so it would also run the Windows-only `service-stop-07` on a Linux host, where it fails for want of PowerShell. diff --git a/tests/e2e-cucumber/features/service_stop.feature b/tests/e2e-cucumber/features/service_stop.feature index 9a0a9b8882..674fb108f7 100644 --- a/tests/e2e-cucumber/features/service_stop.feature +++ b/tests/e2e-cucumber/features/service_stop.feature @@ -157,7 +157,7 @@ Feature: Stopping a managed service # running, and the scenario fails on it. No GPU, no engine, no network. The # record carries no start-time token, like a Windows record written before the # token was recorded: the case where the PID cannot be refuted, which the tree - # kill and the cleared record exist for. service-stop-09 below is the same stop + # kill and the cleared record exist for. service-stop-08 below is the same stop # with a token that does not match. @id:service-stop-takes-down-the-whole-process-tree-on-windows @@ -182,7 +182,7 @@ Feature: Stopping a managed service # checked on the process itself. Runs on every OS: the planted token is one # no live process has. @id:service-stop-leaves-a-process-that-reused-the-recorded-pid-alone - Scenario: service-stop-09 - Stopping a managed server whose recorded PID now belongs to another process leaves that process running + Scenario: service-stop-08 - Stopping a managed server whose recorded PID now belongs to another process leaves that process running Given a managed server whose recorded process id now belongs to a different process When the user stops it with --yes Then the CLI reports the service as stopped From d965fc8207217c6720ab25c2f652dffea9716965 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 07:33:09 +0000 Subject: [PATCH 10/30] fix(core): keep a stop unconfirmed when a Windows process cannot be queried, or when its tree kill did not complete, and hold the PID across the stop On Windows three things let a stop report success while a process of the service might still run. A live process the stop may not open made OpenProcess fail, and process_is_running read every failure as "not running", so the identity gate classified it Gone and the stop reported it already gone. The liveness gate now keeps a failed query apart from an exit: ProcessProbe::Unqueryable is Indeterminate, so the stop reports it unverified and keeps the record and the endpoint key. Only the OpenProcess code for "no such process" counts as gone. process_is_running itself is unchanged. The creation-time comparison closed its handle before taskkill ran by PID, leaving a window for the process to exit and an unrelated one to take the PID. The PID is now held open from before the identity read until the stop returns; Windows does not hand a PID to another process while a handle to it is open. The taskkill result was discarded, and terminate_verified can only watch the root on Windows, so a descendant taskkill did not take down went unnoticed. The tree kill now returns whether taskkill started and exited cleanly, and a failed one turns any verdict that would confirm the stop into timed_out, which keeps the record, the stop request and the endpoint key. The decisions sit behind pure functions and injected steps so the tests run on every host: identity_state_with, classify_open_failure, classify_taskkill, after_tree_kill and terminate_recorded_pid_with with an injected hold. The Windows API calls themselves (the handle, the error codes, a real taskkill failing) run only on the Windows lane. Signed-off-by: Jussi Elo --- crates/rocm-core/src/proc_lifecycle.rs | 399 ++++++++++++++++++++++--- crates/rocm-core/src/process.rs | 112 +++++++ docs/architecture.md | 2 +- docs/testing.md | 19 ++ 4 files changed, 495 insertions(+), 37 deletions(-) diff --git a/crates/rocm-core/src/proc_lifecycle.rs b/crates/rocm-core/src/proc_lifecycle.rs index 831c9ef5a0..0d57586320 100644 --- a/crates/rocm-core/src/proc_lifecycle.rs +++ b/crates/rocm-core/src/proc_lifecycle.rs @@ -15,6 +15,8 @@ use std::time::{Duration, Instant}; +use crate::process::ProcessProbe; + /// How often the bounded waits poll for the target's exit. const POLL_INTERVAL: Duration = Duration::from_millis(25); @@ -71,8 +73,10 @@ pub enum IdentityState { /// was recycled. Recycled, /// The PID is live and an identity was recorded, but the current start-time - /// cannot be read, so it can be neither confirmed nor refuted. The process - /// must not be signalled, and it must not be reported as stopped. + /// cannot be read, so it can be neither confirmed nor refuted; or the PID + /// cannot be queried at all (on Windows, a live process this one may not + /// open), so it cannot be told from a process that is gone. The process must + /// not be signalled, and it must not be reported as stopped. Indeterminate, /// The PID is not live (or is a not-yet-reaped zombie). Gone, @@ -87,10 +91,34 @@ pub enum IdentityState { /// [`IdentityState::Matches`]. #[must_use] pub fn identity_state(id: &ProcessIdentity) -> IdentityState { - if !crate::process_is_running(id.pid) || process_has_exited(id.pid) { - return IdentityState::Gone; + identity_state_with( + id, + crate::process::probe_process, + process_has_exited, + process_start_ticks, + ) +} + +/// [`identity_state`] with its three system reads supplied by the caller: whether +/// the PID can be queried and is running, whether it is a zombie, and its +/// current start-time token. The liveness gate in front of every stop, so a test +/// drives each answer a read can give on every host. +/// +/// A PID that cannot be queried is [`IdentityState::Indeterminate`], not +/// [`IdentityState::Gone`]: a failure to look is not an exit, and reading it as +/// one would let a stop report a live process as stopped. +fn identity_state_with( + id: &ProcessIdentity, + probe: impl Fn(u32) -> ProcessProbe, + has_exited: impl Fn(u32) -> bool, + start_ticks: impl Fn(u32) -> Option, +) -> IdentityState { + match probe(id.pid) { + ProcessProbe::Gone => IdentityState::Gone, + ProcessProbe::Unqueryable => IdentityState::Indeterminate, + ProcessProbe::Running if has_exited(id.pid) => IdentityState::Gone, + ProcessProbe::Running => classify_live_identity(id.start_ticks, start_ticks(id.pid)), } - classify_live_identity(id.start_ticks, process_start_ticks(id.pid)) } /// The identity verdict for a PID that is running: what its recorded token @@ -466,47 +494,131 @@ pub fn terminate_recorded_service_pids( /// PID, and `taskkill /T /F` then takes that process and its subtree. A stop /// that confirms clears the record's PIDs, so that exposure shrinks to the /// records written before the token existed and not yet stopped. +/// +/// On Windows the PID is held from before the identity check until the stop +/// ends: a process's PID is not handed to another process while any handle to it +/// is open, so the creation time that was compared is the creation time of the +/// process `taskkill` and the root termination then reach by PID. Without the +/// hold, the process could exit and an unrelated one take its PID between the +/// check and the kill. +/// +/// A tree kill that did not complete leaves the stop unconfirmed, whatever the +/// root's own termination says: [`terminate_verified`] can only watch the root +/// on Windows, so a descendant that `taskkill` did not take down is one this stop +/// cannot see exit, and reporting the stop confirmed would clear the record and +/// the endpoint key while a worker may still hold the device. pub fn terminate_recorded_pid(identity: &ProcessIdentity) -> TerminationOutcome { - terminate_recorded_pid_with(identity, platform_tree_kill(), identity_state, |identity| { - terminate_verified(identity, KillScope::Tree, MANAGED_STOP_GRACE, true) - }) + terminate_recorded_pid_with( + identity, + platform_tree_kill(), + hold_pid, + identity_state, + |identity| terminate_verified(identity, KillScope::Tree, MANAGED_STOP_GRACE, true), + ) } -/// [`terminate_recorded_pid`] with its three platform and system touchpoints -/// supplied by the caller, so a test drives the tree-kill step on every host: -/// `tree_kill` takes one PID's descendants down (`None` where the platform -/// needs no separate step), `state_of` reads the PID's identity, and `terminate` -/// is the verified termination of the root. -fn terminate_recorded_pid_with( +/// Whether the separate descendant kill took the tree down. +// Only Windows has the separate step, so nothing builds a value elsewhere; the +// tests build both, and the decision on them is shared by every platform. +#[cfg_attr(not(any(windows, test)), allow(dead_code))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TreeKill { + /// The kill ran and reported that every process it reached is terminated. + Completed, + /// The kill could not run, or reported that it did not take every process + /// down. Which descendants survive is not known. + Failed, +} + +/// The verdict for a stop whose descendant kill ended as `tree` and whose root +/// termination answered `outcome`. A failed tree kill turns any verdict that +/// would confirm the stop into [`TerminationOutcome::TimedOut`]: the root was +/// signalled, and it, or a process under it, is not confirmed exited. The +/// verdicts that already deny the stop stay as they are. +const fn after_tree_kill(tree: TreeKill, outcome: TerminationOutcome) -> TerminationOutcome { + match tree { + TreeKill::Completed => outcome, + TreeKill::Failed if outcome.stopped() => TerminationOutcome::TimedOut, + TreeKill::Failed => outcome, + } +} + +/// [`terminate_recorded_pid`] with its platform and system touchpoints supplied +/// by the caller, so a test drives the tree-kill step on every host: +/// `tree_kill` takes one PID's descendants down (`None` where the platform needs +/// no separate step), `hold` keeps the PID reserved and is dropped when the call +/// returns, `state_of` reads the PID's identity, and `terminate` is the verified +/// termination of the root. +fn terminate_recorded_pid_with( identity: &ProcessIdentity, - tree_kill: Option<&dyn Fn(u32)>, + tree_kill: Option<&dyn Fn(u32) -> TreeKill>, + hold: impl FnOnce(&ProcessIdentity) -> Held, state_of: impl Fn(&ProcessIdentity) -> IdentityState, terminate: impl Fn(&ProcessIdentity) -> TerminationOutcome, ) -> TerminationOutcome { + // Before anything reads the identity, and until the end of the call. + let _hold = hold(identity); if let Some(tree_kill) = tree_kill && matches!(state_of(identity), IdentityState::Matches) { // Taken while the root is still alive: `taskkill /T` resolves children // through the live parent, so it cannot reach them once the root is gone. - tree_kill(identity.pid); + let tree = tree_kill(identity.pid); // `terminate` still owns the bounded wait and the verdict, but a root // the tree kill already took now reads back as `AlreadyGone` — which // would deny a stop this command did perform. - return match terminate(identity) { + let outcome = match terminate(identity) { TerminationOutcome::AlreadyGone => TerminationOutcome::Forced, outcome => outcome, }; + return after_tree_kill(tree, outcome); } terminate(identity) } +/// What keeps a PID reserved while a stop works on it: on Windows an open +/// handle to the process, which stops the OS giving the PID to another process; +/// nothing elsewhere, where the stop signals by PID and `terminate_verified` +/// re-verifies it. +#[cfg(windows)] +struct PidHold { + _handle: Option, +} + +#[cfg(not(windows))] +struct PidHold; + +/// Hold `identity`'s PID. A process that cannot be opened is not held, and the +/// identity read that follows then refuses it as gone or unverifiable. +#[cfg(windows)] +#[allow(unsafe_code)] // Win32 FFI +fn hold_pid(identity: &ProcessIdentity) -> PidHold { + use std::os::windows::io::{FromRawHandle, OwnedHandle}; + use windows_sys::Win32::System::Threading::{OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION}; + + // SAFETY: a plain Win32 call; a non-null handle is owned by nothing else and + // is closed when the `OwnedHandle` drops. + let handle = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, identity.pid) }; + PidHold { + _handle: (!handle.is_null()).then(|| unsafe { OwnedHandle::from_raw_handle(handle) }), + } +} + +// The `cfg` sibling calls non-const code, so this arm cannot be `const` without +// giving the two platforms different signatures. +#[cfg(not(windows))] +#[allow(clippy::missing_const_for_fn)] +fn hold_pid(_identity: &ProcessIdentity) -> PidHold { + PidHold +} + /// The separate descendant kill this platform needs before the verified /// termination of a root, if any: `taskkill /T /F` on Windows, none elsewhere, /// where [`terminate_verified`] walks the tree itself. // The `cfg` arms differ in what they return, so the non-Windows arm cannot be a // `const fn` without giving the two platforms different signatures. #[allow(clippy::missing_const_for_fn)] -fn platform_tree_kill() -> Option<&'static dyn Fn(u32)> { +fn platform_tree_kill() -> Option<&'static dyn Fn(u32) -> TreeKill> { #[cfg(windows)] { Some(&taskkill_tree) @@ -517,15 +629,31 @@ fn platform_tree_kill() -> Option<&'static dyn Fn(u32)> { } } +/// The verdict on one `taskkill` run, from whether it started and the exit code +/// it ended with. Only a clean exit (0) is a completed tree kill: a failed +/// launch, a nonzero code (a process it could not terminate, or none found) and a +/// run that ended without a code are all [`TreeKill::Failed`]. Split out of the +/// process spawn so a test runs it on every host. +#[cfg(any(windows, test))] +fn classify_taskkill(run: std::io::Result>) -> TreeKill { + match run { + Ok(Some(0)) => TreeKill::Completed, + Ok(_) | Err(_) => TreeKill::Failed, + } +} + #[cfg(windows)] -fn taskkill_tree(pid: u32) { +fn taskkill_tree(pid: u32) -> TreeKill { use std::process::{Command, Stdio}; - let _ = Command::new("taskkill") - .args(["/PID", &pid.to_string(), "/T", "/F"]) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); + classify_taskkill( + Command::new("taskkill") + .args(["/PID", &pid.to_string(), "/T", "/F"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + .map(|status| status.code()), + ) } /// Poll until every process in `members` has exited, or `grace` elapses. @@ -1122,23 +1250,44 @@ mod tests { /// What one `terminate_recorded_pid_with` call did, in order. #[derive(Debug, PartialEq, Eq)] enum Step { + Hold(u32), TreeKill(u32), Terminate(u32), + Release, + } + + /// A stand-in for the Windows PID hold: records that it was taken and, when + /// it is dropped, that it was released. + struct RecordedHold<'a>(&'a RefCell>); + + impl Drop for RecordedHold<'_> { + fn drop(&mut self) { + self.0.borrow_mut().push(Step::Release); + } } /// Run `terminate_recorded_pid_with` for `pid`, with a tree-kill step - /// present or not, the identity in `state`, and the root's termination - /// answering `outcome`. Returns the verdict and the steps taken. - fn tree_kill_run( + /// present or not and ending as `tree`, the identity in `state`, and the + /// root's termination answering `outcome`. Returns the verdict and the steps + /// taken, the hold's release included. + fn tree_kill_run_with( has_tree_kill: bool, + tree: TreeKill, state: IdentityState, outcome: TerminationOutcome, ) -> (TerminationOutcome, Vec) { let steps = RefCell::new(Vec::new()); - let tree_kill = |pid: u32| steps.borrow_mut().push(Step::TreeKill(pid)); + let tree_kill = |pid: u32| { + steps.borrow_mut().push(Step::TreeKill(pid)); + tree + }; let verdict = terminate_recorded_pid_with( &ProcessIdentity::new(SUPERVISOR_PID, None), - has_tree_kill.then_some(&tree_kill as &dyn Fn(u32)), + has_tree_kill.then_some(&tree_kill as &dyn Fn(u32) -> TreeKill), + |identity| { + steps.borrow_mut().push(Step::Hold(identity.pid)); + RecordedHold(&steps) + }, |_| state, |identity| { steps.borrow_mut().push(Step::Terminate(identity.pid)); @@ -1148,6 +1297,15 @@ mod tests { (verdict, steps.into_inner()) } + /// [`tree_kill_run_with`] for a tree kill that completes. + fn tree_kill_run( + has_tree_kill: bool, + state: IdentityState, + outcome: TerminationOutcome, + ) -> (TerminationOutcome, Vec) { + tree_kill_run_with(has_tree_kill, TreeKill::Completed, state, outcome) + } + #[test] fn the_tree_is_killed_before_the_root_is_terminated() { let (_, steps) = tree_kill_run(true, IdentityState::Matches, TerminationOutcome::Forced); @@ -1155,8 +1313,10 @@ mod tests { assert_eq!( steps, vec![ + Step::Hold(SUPERVISOR_PID), Step::TreeKill(SUPERVISOR_PID), - Step::Terminate(SUPERVISOR_PID) + Step::Terminate(SUPERVISOR_PID), + Step::Release, ], "`taskkill /T` resolves children through the live parent, so the \ tree kill has to come first" @@ -1199,7 +1359,11 @@ mod tests { assert_eq!( steps, - vec![Step::Terminate(SUPERVISOR_PID)], + vec![ + Step::Hold(SUPERVISOR_PID), + Step::Terminate(SUPERVISOR_PID), + Step::Release + ], "{state:?}: a stranger's process tree must not be taken" ); assert_eq!( @@ -1218,7 +1382,14 @@ mod tests { TerminationOutcome::AlreadyGone, ); - assert_eq!(steps, vec![Step::Terminate(SUPERVISOR_PID)]); + assert_eq!( + steps, + vec![ + Step::Hold(SUPERVISOR_PID), + Step::Terminate(SUPERVISOR_PID), + Step::Release + ] + ); assert_eq!(verdict, TerminationOutcome::AlreadyGone); } @@ -1300,10 +1471,17 @@ mod tests { } in rows { let steps = RefCell::new(Vec::new()); - let tree_kill = |pid: u32| steps.borrow_mut().push(Step::TreeKill(pid)); + let tree_kill = |pid: u32| { + steps.borrow_mut().push(Step::TreeKill(pid)); + TreeKill::Completed + }; let verdict = terminate_recorded_pid_with( &ProcessIdentity::new(SUPERVISOR_PID, recorded), - Some(&tree_kill as &dyn Fn(u32)), + Some(&tree_kill as &dyn Fn(u32) -> TreeKill), + |id| { + steps.borrow_mut().push(Step::Hold(id.pid)); + RecordedHold(&steps) + }, |id| classify_live_identity(id.start_ticks, live), |id| { steps.borrow_mut().push(Step::Terminate(id.pid)); @@ -1316,11 +1494,17 @@ mod tests { ); let expected_steps = if tree_kill_runs { vec![ + Step::Hold(SUPERVISOR_PID), Step::TreeKill(SUPERVISOR_PID), Step::Terminate(SUPERVISOR_PID), + Step::Release, ] } else { - vec![Step::Terminate(SUPERVISOR_PID)] + vec![ + Step::Hold(SUPERVISOR_PID), + Step::Terminate(SUPERVISOR_PID), + Step::Release, + ] }; assert_eq!(steps.into_inner(), expected_steps, "{row}: steps"); assert_eq!(verdict.as_str(), label, "{row}: verdict"); @@ -1347,6 +1531,149 @@ mod tests { assert_eq!(platform_tree_kill().is_some(), cfg!(windows)); } + #[test] + fn the_pid_is_held_before_its_identity_is_read_and_released_after_the_stop() { + // The hold is what makes the creation time that was compared the + // creation time of the process the kill then reaches by PID: taken + // first, released last. `Hold` comes before anything else and `Release` + // after the root's termination, with the tree kill between. + let (_, steps) = tree_kill_run(true, IdentityState::Matches, TerminationOutcome::Forced); + + assert_eq!(steps.first(), Some(&Step::Hold(SUPERVISOR_PID))); + assert_eq!(steps.last(), Some(&Step::Release)); + assert_eq!( + steps + .iter() + .filter(|step| matches!(step, Step::Release)) + .count(), + 1 + ); + } + + #[test] + fn a_tree_kill_that_did_not_complete_leaves_the_stop_unconfirmed() { + use TerminationOutcome::{AlreadyGone, Forced, Graceful, TimedOut, Unverified}; + // Every root verdict that would confirm the stop becomes the one that + // says a process under the root may still be running; the verdicts that + // already deny the stop are kept. + for (root, expected) in [ + (Forced, TimedOut), + (Graceful, TimedOut), + (AlreadyGone, TimedOut), + (TimedOut, TimedOut), + (Unverified, Unverified), + ] { + let (verdict, steps) = + tree_kill_run_with(true, TreeKill::Failed, IdentityState::Matches, root); + + assert_eq!(verdict, expected, "root {root:?}"); + assert!( + !verdict.stopped(), + "root {root:?}: the stop is not confirmed" + ); + assert!( + steps.contains(&Step::Terminate(SUPERVISOR_PID)), + "the root is still terminated, best effort: {steps:?}" + ); + } + } + + #[test] + fn a_completed_tree_kill_leaves_the_roots_verdict_alone() { + use TerminationOutcome::{Forced, Graceful, TimedOut, Unverified}; + for outcome in [Forced, Graceful, TimedOut, Unverified] { + let (verdict, _) = + tree_kill_run_with(true, TreeKill::Completed, IdentityState::Matches, outcome); + + assert_eq!(verdict, outcome); + } + } + + #[test] + fn only_a_clean_taskkill_exit_is_a_completed_tree_kill() { + use std::io::{Error, ErrorKind}; + assert_eq!(classify_taskkill(Ok(Some(0))), TreeKill::Completed); + // A process it could not terminate, or none found. + assert_eq!(classify_taskkill(Ok(Some(1))), TreeKill::Failed); + assert_eq!(classify_taskkill(Ok(Some(128))), TreeKill::Failed); + // Ended without an exit code. + assert_eq!(classify_taskkill(Ok(None)), TreeKill::Failed); + // Could not be started. + assert_eq!( + classify_taskkill(Err(Error::from(ErrorKind::NotFound))), + TreeKill::Failed + ); + } + + /// A PID that cannot be queried is neither gone nor a match, through the + /// same liveness gate every stop starts with. + #[test] + fn a_pid_that_cannot_be_queried_is_unverified_not_already_gone() { + use crate::process::ProcessProbe::{Gone, Running, Unqueryable}; + let state = |probe, zombie, recorded, live| { + identity_state_with( + &ProcessIdentity::new(SUPERVISOR_PID, recorded), + |_| probe, + |_| zombie, + |_| live, + ) + }; + // Whatever token was recorded, and whatever can be read of the process. + for recorded in [None, Some(7)] { + for live in [None, Some(7), Some(8)] { + assert_eq!( + state(Unqueryable, false, recorded, live), + IdentityState::Indeterminate, + "recorded {recorded:?}, live {live:?}" + ); + } + } + assert_eq!(state(Gone, false, Some(7), Some(7)), IdentityState::Gone); + assert_eq!(state(Running, true, Some(7), Some(7)), IdentityState::Gone); + assert_eq!( + state(Running, false, Some(7), Some(7)), + IdentityState::Matches + ); + assert_eq!( + state(Running, false, Some(7), Some(8)), + IdentityState::Recycled + ); + assert_eq!( + state(Running, false, Some(7), None), + IdentityState::Indeterminate + ); + } + + /// The whole stop for a live process the caller may not query: not + /// signalled, no tree kill, and reported as unverified rather than as + /// stopped, so the record and the endpoint key are kept. + #[test] + fn a_stop_of_a_live_process_it_cannot_query_is_not_confirmed() { + use crate::process::ProcessProbe::Unqueryable; + let steps = RefCell::new(Vec::new()); + let tree_kill = |pid: u32| { + steps.borrow_mut().push(Step::TreeKill(pid)); + TreeKill::Completed + }; + let state_of = + |id: &ProcessIdentity| identity_state_with(id, |_| Unqueryable, |_| false, |_| None); + let verdict = terminate_recorded_pid_with( + &ProcessIdentity::new(SUPERVISOR_PID, Some(7)), + Some(&tree_kill as &dyn Fn(u32) -> TreeKill), + |_| (), + state_of, + // The verified termination's own gate, with the same read. + |id| refusal_before_signalling(state_of(id)).unwrap_or(TerminationOutcome::Forced), + ); + + assert_eq!(verdict, TerminationOutcome::Unverified); + assert!(!verdict.stopped()); + assert!( + steps.into_inner().is_empty(), + "a process that cannot be told from a stranger is not tree-killed" + ); + } + #[test] fn terminate_recorded_pid_reports_an_absent_pid_as_already_gone() { let outcome = terminate_recorded_pid(&ProcessIdentity::new(SUPERVISOR_PID, None)); diff --git a/crates/rocm-core/src/process.rs b/crates/rocm-core/src/process.rs index 2e285e3169..74119a37a6 100644 --- a/crates/rocm-core/src/process.rs +++ b/crates/rocm-core/src/process.rs @@ -363,6 +363,85 @@ pub fn terminate_process_tree(pid: u32) -> Result<()> { terminate_process(pid) } +/// What one look at a PID could establish. +/// +/// [`process_is_running`] answers a plain yes or no, so a process that is alive +/// but cannot be queried reads the same as one that is gone. A stop that is about +/// to report a process as exited must not make that mistake, so this keeps the +/// third answer apart. +// Only the Windows probe can fail to query a live process, so nothing builds +// `Unqueryable` elsewhere; the decision on it is shared by every platform. +#[cfg_attr(not(any(windows, test)), allow(dead_code))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ProcessProbe { + /// The PID names a process that has not exited. + Running, + /// The PID names no process, or one that has exited. + Gone, + /// The PID could not be queried, so it can be neither confirmed running nor + /// confirmed gone. On Windows that includes a live process the caller may not + /// open (`ERROR_ACCESS_DENIED`). + Unqueryable, +} + +/// Windows `ERROR_INVALID_PARAMETER`: what `OpenProcess` sets for a PID that +/// names no process. +#[cfg(any(windows, test))] +const ERROR_INVALID_PARAMETER: i32 = 87; + +/// What a failed `OpenProcess` says about the PID, from its `GetLastError` code. +/// +/// Only the code for "no such process" counts as gone. `ERROR_ACCESS_DENIED` +/// (5), which a live process the caller may not open sets, and any other or +/// missing code are [`ProcessProbe::Unqueryable`]: a failure to look is not an +/// exit. Split out of the FFI so a test runs it on every host. +#[cfg(any(windows, test))] +pub(crate) const fn classify_open_failure(code: Option) -> ProcessProbe { + match code { + Some(ERROR_INVALID_PARAMETER) => ProcessProbe::Gone, + _ => ProcessProbe::Unqueryable, + } +} + +/// Look at `pid`, keeping a process that cannot be queried apart from one that +/// is gone. See [`ProcessProbe`]. +#[cfg(windows)] +#[allow(unsafe_code)] // Win32 FFI +pub(crate) fn probe_process(pid: u32) -> ProcessProbe { + use windows_sys::Win32::Foundation::CloseHandle; + + if pid == 0 { + return ProcessProbe::Gone; + } + let handle = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid) }; + if handle.is_null() { + return classify_open_failure(std::io::Error::last_os_error().raw_os_error()); + } + let mut exit_code = 0; + let queried = unsafe { GetExitCodeProcess(handle, &raw mut exit_code) != 0 }; + unsafe { + CloseHandle(handle); + } + match (queried, exit_code) { + (false, _) => ProcessProbe::Unqueryable, + // `STILL_ACTIVE`. + (true, 259) => ProcessProbe::Running, + (true, _) => ProcessProbe::Gone, + } +} + +/// Look at `pid`. `kill(pid, 0)` already tells a live process the caller may +/// not signal (`EPERM`, which counts as running) from one that is gone, so there +/// is no third answer to keep apart here. +#[cfg(not(windows))] +pub(crate) fn probe_process(pid: u32) -> ProcessProbe { + if process_is_running(pid) { + ProcessProbe::Running + } else { + ProcessProbe::Gone + } +} + #[cfg(windows)] #[allow(unsafe_code)] // Win32 FFI pub fn process_is_running(pid: u32) -> bool { @@ -1261,6 +1340,39 @@ mod tests { use crate::test_support::workspace_test_artifact_dir; use std::process::Stdio; + /// A failed `OpenProcess` is a gone process only for the code that means + /// "no such process". Access denied, which a live process the caller may not + /// open sets, and any code that is not known are not an exit. + #[test] + fn a_failed_open_is_gone_only_for_no_such_process() { + const ERROR_ACCESS_DENIED: i32 = 5; + assert_eq!(classify_open_failure(Some(87)), ProcessProbe::Gone); + assert_eq!( + classify_open_failure(Some(ERROR_ACCESS_DENIED)), + ProcessProbe::Unqueryable + ); + // The codes around the two that matter, and no code at all. + for code in [Some(0), Some(6), Some(31), Some(299), Some(-1), None] { + assert_eq!( + classify_open_failure(code), + ProcessProbe::Unqueryable, + "{code:?}" + ); + } + } + + /// Off Windows, a PID that is running (including one the caller may not + /// signal) probes as running and an absent one as gone, matching + /// [`process_is_running`]. + #[cfg(not(windows))] + #[test] + fn the_probe_agrees_with_process_is_running_off_windows() { + let own = std::process::id(); + assert_eq!(probe_process(own), ProcessProbe::Running); + assert_eq!(probe_process(0), ProcessProbe::Gone); + assert_eq!(probe_process(999_999_998), ProcessProbe::Gone); + } + const LOCK_CHILD_PATH_ENV: &str = "ROCM_CORE_TEST_LOCK_PATH"; const LOCK_CHILD_READY_ENV: &str = "ROCM_CORE_TEST_LOCK_READY"; const HANDOFF_PORT_ENV: &str = "ROCM_CORE_TEST_HANDOFF_PORT"; diff --git a/docs/architecture.md b/docs/architecture.md index f428fb722b..b1d12d3327 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -231,7 +231,7 @@ Further `rocm-core` modules, each with its own doc comment except where noted: - [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) — OpenMPI, libatomic and libnuma detection, the `libmpi_cxx` compatibility shim, and the per-library install plans (`build_openmpi_install_plan`, `build_libatomic_install_plan`, `build_libnuma_install_plan`) and hints, needed by vLLM's ROCm wheels. The distro-generic half of that planning lives in [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs); [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) re-exports its public items, so `rocm_core::openmpi::running_as_root`, `plan_for_os_release_read` and the rest keep resolving, and `pub(crate)` re-exports `PackageManager` and `resolve_package_manager` for [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs). - [`os_release.rs`](../crates/rocm-core/src/os_release.rs) — the one `/etc/os-release` parser every reader of that file uses. - [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs) — the planning half of a system-package install that does not depend on which library is installed: `PackageManager` and `resolve_package_manager` (from an os-release `ID`/`ID_LIKE`), `SystemPackageInstallPlan` and `InstallCommand`, `plan_for_os_release_at`/`plan_for_os_release_read` (which take the per-library builder as an argument), `InstallHint`, the root and passwordless-`sudo` checks (`running_as_root`, `can_autoinstall`) and the `apt-get -s` helpers (`apt_simulate_argv`, `parse_apt_simulate_removals`, `is_protected_rocm_package`). It never runs anything and depends on no other module of this crate except [`os_release.rs`](../crates/rocm-core/src/os_release.rs) and [`hardware_root.rs`](../crates/rocm-core/src/hardware_root.rs), which it reads os-release through. The module is private; its public items are re-exported from [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs). It is a different module from [`system_packages.rs`](../apps/rocm/src/system_packages.rs) in `apps/rocm`, which runs these plans with consent and `sudo`. -- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches (`terminate_verified`), and the one managed-stop routine built on it that `rocm services stop` and `rocmd`'s `stop_server` both call: `terminate_recorded_service_pids` walks the processes a service record names (supervisor, then engine, each with its own start-time, the stopping process's own PID reported as `self` and never signalled) and returns one `RecordedPidStop` per PID (each caller, on a stop that confirms every one gone, then clears the record's recorded processes with `ManagedServiceRecord::clear_recorded_processes` in [`service_record.rs`](../crates/rocm-core/src/service_record.rs), so a repeated stop reaches no stale PID; an unconfirmed stop keeps them); `terminate_recorded_pid` terminates one of them, adding on Windows a `taskkill /T /F` of its descendants first, because `terminate_verified` cannot walk a process tree there, and only for a PID that passes the same identity check (`process_start_ticks` is the kernel start-time on Linux and the process creation time on Windows, so a recycled PID is refused before any `taskkill`; a record written before Windows recorded one cannot be refuted); `MANAGED_STOP_GRACE` is the one grace period. The termination is injectable, so a caller's test supplies outcomes it cannot construct and a test of the Windows step runs on every host. +- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches (`terminate_verified`), and the one managed-stop routine built on it that `rocm services stop` and `rocmd`'s `stop_server` both call: `terminate_recorded_service_pids` walks the processes a service record names (supervisor, then engine, each with its own start-time, the stopping process's own PID reported as `self` and never signalled) and returns one `RecordedPidStop` per PID (each caller, on a stop that confirms every one gone, then clears the record's recorded processes with `ManagedServiceRecord::clear_recorded_processes` in [`service_record.rs`](../crates/rocm-core/src/service_record.rs), so a repeated stop reaches no stale PID; an unconfirmed stop keeps them); `terminate_recorded_pid` terminates one of them, adding on Windows a `taskkill /T /F` of its descendants first, because `terminate_verified` cannot walk a process tree there, and only for a PID that passes the same identity check (`process_start_ticks` is the kernel start-time on Linux and the process creation time on Windows, so a recycled PID is refused before any `taskkill`; a record written before Windows recorded one cannot be refuted), holding the PID open on Windows from before that check until the stop ends so the PID cannot be reused in between, and treating a tree kill that did not complete as an unconfirmed stop; a PID that cannot be queried at all, as against one that is gone, reads as unverifiable (`ProcessProbe` in [`process.rs`](../crates/rocm-core/src/process.rs)); `MANAGED_STOP_GRACE` is the one grace period. The termination is injectable, so a caller's test supplies outcomes it cannot construct and a test of the Windows step runs on every host. - [`report.rs`](../crates/rocm-core/src/report.rs) — the content of a Doctor report and the rule deciding whether one may exist; it sends nothing. - [`report_delivery.rs`](../crates/rocm-core/src/report_delivery.rs) — how a report leaves the machine: building its link and deciding whether a browser may be started. - [`runtime.rs`](../crates/rocm-core/src/runtime.rs) — the host-platform layer: `RuntimePlatform`/`RuntimeHost`, OS-specific executable and Python-environment names (including `runtime_python_env_root_for_executable`, the one rule for finding an environment's folder from its interpreter's path: only an interpreter directly inside a `bin` or `Scripts` folder, in any case, has one, and everything else gets `None`; the adopt command, the ComfyUI runtime environment and the vLLM launch all call it rather than keeping a copy, since the engine crate cannot reach `apps/rocm`), default config/data/cache directories, path-text and path-list helpers, and `install_root_has_payload`, the one definition of a non-empty install root (dot-entries such as the runtime marker do not count; a path that does not exist holds nothing; a root that cannot be listed is an error) that tarball extraction, update repair detection and activation validation in `apps/rocm` all ask (no module doc comment). diff --git a/docs/testing.md b/docs/testing.md index 07148619f9..e80e4fd025 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1838,6 +1838,25 @@ Neither Windows-only test nor the Windows half of `service-stop-08` ran where this was written; the Windows CI jobs are what exercise them. A stop that cannot verify a process names `ps -p -o pid,args` on Linux and `tasklist /FI "PID eq "` on Windows (`process_check_command`). +Three further Windows paths of that stop are tested through injected functions +that run on every host; no scenario forces their failure cases. A PID that cannot be +queried (`ProcessProbe::Unqueryable`, which on Windows includes a live process +the stop may not open) is `Indeterminate`, so the stop reports it unverified +instead of already gone (`a_pid_that_cannot_be_queried_is_unverified_not_already_gone`, +`a_stop_of_a_live_process_it_cannot_query_is_not_confirmed`, and +`a_failed_open_is_gone_only_for_no_such_process` for the error codes). The PID is +held open from before the identity read until the stop returns, so the creation +time that was compared is that of the process the kill then reaches +(`the_pid_is_held_before_its_identity_is_read_and_released_after_the_stop`). And a +`taskkill` that failed to start or exited nonzero leaves the stop unconfirmed, +reported as `timed_out`, whatever the root's own termination says, because only +the root's exit is watched on Windows +(`a_tree_kill_that_did_not_complete_leaves_the_stop_unconfirmed`, +`only_a_clean_taskkill_exit_is_a_completed_tree_kill`); the report and the record +a `timed_out` stop leaves are covered by +`an_unconfirmed_stop_keeps_the_recorded_processes_for_the_next_stop`. What no +test here reaches is the Windows API itself: the handle hold, the open-failure +codes and a real `taskkill` failing run only on the Windows lane. `service-stop-06` (Linux) is the same stop of a real recorded process through `rocm services stop`, checking the persisted state: the process exited and the record names none. Run the Windows scenario on a Windows host with From 4ebcdf603ff0d6a6807214c21e61e1599c51deaf Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 10:04:25 +0000 Subject: [PATCH 11/30] fix(core): keep an unverified stop's key through a liveness refresh, and bound the Windows tree kill with a deadline The previous change made a stop report a Windows process it cannot query as unverified and keep the record, the stop request and the endpoint key. The liveness refresh that runs when services are listed then undid that: it still called process_is_running, which reads a process it cannot open as not running, so it deleted the key, cleared the stop request and reconciled the record to stopped while the process might be alive. The same call decided whether a record may be removed or pruned. All three now read each recorded PID through probe_process and treat anything but a confirmed exit as possibly running (ProcessProbe::may_be_running, exported from rocm-core): the deferred key cleanup, the reconciliation to stopped, and the in-use check behind services remove and prune. Each takes the probe as an argument, so the tests drive an unqueryable process on every host. process_is_running itself is unchanged; the launch-time callers that watch a child this process just started keep it. The taskkill that takes a Windows tree down ran through Command::status, which waits without limit and runs before the stop's bounded waits, so a stalled taskkill hung the stop. It now runs under a deadline of MANAGED_STOP_GRACE: a child still running at the deadline is killed and reaped, the run reports a timeout, and that is a failed tree kill, which the stop already treats as unconfirmed. The wait is a plain helper over std::process, tested on every host with a long-sleeping child (ping on Windows) under a short deadline; only the taskkill wiring is Windows-only. Signed-off-by: Jussi Elo --- apps/rocm/src/managed_service_status.rs | 123 ++++++++++++++++- apps/rocm/src/service_control.rs | 46 +++++++ apps/rocm/src/service_records.rs | 40 +++++- crates/rocm-core/src/lib.rs | 4 +- crates/rocm-core/src/proc_lifecycle.rs | 173 ++++++++++++++++++++++-- crates/rocm-core/src/process.rs | 28 +++- docs/architecture.md | 4 +- docs/testing.md | 21 ++- 8 files changed, 411 insertions(+), 28 deletions(-) diff --git a/apps/rocm/src/managed_service_status.rs b/apps/rocm/src/managed_service_status.rs index 586adccf94..93bf1e5f3d 100644 --- a/apps/rocm/src/managed_service_status.rs +++ b/apps/rocm/src/managed_service_status.rs @@ -42,8 +42,8 @@ use std::time::Duration; use anyhow::{Context, Result}; use rocm_core::{ - AppPaths, EndpointReadiness, EndpointReadinessOutcome, ManagedServiceRecord, - managed_service_endpoint_readiness, process_is_running, + AppPaths, EndpointReadiness, EndpointReadinessOutcome, ManagedServiceRecord, ProcessProbe, + managed_service_endpoint_readiness, }; use crate::endpoint_keys; @@ -206,13 +206,22 @@ pub(crate) fn recorded_service_pids(record: &ManagedServiceRecord) -> Vec { /// [`refresh_managed_service_runtime_liveness`], so a record that reached /// "stopped" by another route still gets its deferred cleanup rather than /// stranding the key. -fn settle_pending_stop_key_cleanup(paths: &AppPaths, record: &mut ManagedServiceRecord) -> bool { +/// +/// "Observed gone" means every recorded process is *confirmed* gone: a process +/// that could not be queried (on Windows, a live one this process may not open) +/// is not gone, and the stop that reported it unverified must keep its key and +/// its marker until it is. +fn settle_pending_stop_key_cleanup( + paths: &AppPaths, + record: &mut ManagedServiceRecord, + probe: impl Fn(u32) -> ProcessProbe, +) -> bool { if record.stop_requested_unix_ms.is_none() { return false; } if recorded_service_pids(record) .iter() - .any(|pid| process_is_running(*pid)) + .any(|pid| probe(*pid).may_be_running()) { return false; } @@ -225,7 +234,20 @@ pub(crate) fn refresh_managed_service_runtime_liveness( paths: &AppPaths, record: &mut ManagedServiceRecord, ) -> bool { - let settled_pending_stop = settle_pending_stop_key_cleanup(paths, record); + refresh_managed_service_runtime_liveness_with(paths, record, rocm_core::probe_process) +} + +/// [`refresh_managed_service_runtime_liveness`] with the read of each recorded +/// PID supplied by the caller, so a test drives the answer a host can give for a +/// process it cannot query. Every decision here that a process is gone (the +/// deferred key cleanup, the status reconciled to `stopped`) takes a confirmed +/// exit; a process that could not be queried counts as possibly running. +pub(crate) fn refresh_managed_service_runtime_liveness_with( + paths: &AppPaths, + record: &mut ManagedServiceRecord, + probe: impl Fn(u32) -> ProcessProbe, +) -> bool { + let settled_pending_stop = settle_pending_stop_key_cleanup(paths, record, &probe); if !managed_service_is_live(record) { return settled_pending_stop; } @@ -271,7 +293,7 @@ pub(crate) fn refresh_managed_service_runtime_liveness( let tracked_pids = recorded_service_pids(record); let has_tracked_pid = !tracked_pids.is_empty(); - let has_live_pid = tracked_pids.iter().any(|pid| process_is_running(*pid)); + let has_live_pid = tracked_pids.iter().any(|pid| probe(*pid).may_be_running()); if has_tracked_pid && !has_live_pid { // Reconcile the status only. The endpoint key is *not* dropped here: // dead pids alone do not distinguish a stopped service from a crashed @@ -923,4 +945,93 @@ mod tests { let _ = fs::remove_dir_all(root); Ok(()) } + + /// A record whose stop was requested but not confirmed, with its endpoint + /// key on disk: the state a stop leaves when a recorded process cannot be + /// verified. + fn pending_stop_record(paths: &AppPaths, service_id: &str) -> Result { + let port = std::net::TcpListener::bind(("127.0.0.1", 0))? + .local_addr()? + .port(); + let mut record = ManagedServiceRecord::new( + paths, + service_id, + "lemonade", + "model-ref", + "canonical/model", + "127.0.0.1", + port, + "managed", + 999_999_999, + None, + None, + None, + ); + record.status = "ready".to_owned(); + record.stop_requested_unix_ms = Some(1); + record.write()?; + endpoint_keys::store_endpoint_api_key(paths, service_id, "key-123")?; + Ok(record) + } + + /// A process that cannot be queried (on Windows, a live one this process may + /// not open) is not one that is gone. Whatever is reading liveness must keep + /// the key and the stop marker the unverified stop left, and must not + /// reconcile the record to `stopped`; only a confirmed exit may. + #[test] + fn a_pending_stop_keeps_its_key_and_marker_while_a_process_cannot_be_queried() -> Result<()> { + let (root, paths) = test_paths("liveness-unqueryable-pending-stop"); + paths.ensure()?; + let service_id = "svc-unqueryable"; + let mut record = pending_stop_record(&paths, service_id)?; + + let _ = refresh_managed_service_runtime_liveness_with(&paths, &mut record, |_| { + ProcessProbe::Unqueryable + }); + + assert!( + endpoint_keys::endpoint_key_file_if_present(&paths, service_id).is_some(), + "the key of a service that may still be running is kept" + ); + assert_eq!( + record.stop_requested_unix_ms, + Some(1), + "so is the stop marker that hands the cleanup to a later confirmed exit" + ); + assert_ne!( + record.status, "stopped", + "a process that could not be looked at does not make the service stopped" + ); + + // The same record once the process is confirmed gone: now the deferred + // cleanup and the reconciliation both happen. + let _ = refresh_managed_service_runtime_liveness_with(&paths, &mut record, |_| { + ProcessProbe::Gone + }); + assert!( + endpoint_keys::endpoint_key_file_if_present(&paths, service_id).is_none(), + "a confirmed exit drops the key" + ); + assert_eq!(record.stop_requested_unix_ms, None); + assert_eq!(record.status, "stopped"); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn a_pending_stop_keeps_its_key_and_marker_while_a_process_runs() -> Result<()> { + let (root, paths) = test_paths("liveness-running-pending-stop"); + paths.ensure()?; + let service_id = "svc-running"; + let mut record = pending_stop_record(&paths, service_id)?; + + let _ = refresh_managed_service_runtime_liveness_with(&paths, &mut record, |_| { + ProcessProbe::Running + }); + + assert!(endpoint_keys::endpoint_key_file_if_present(&paths, service_id).is_some()); + assert_eq!(record.stop_requested_unix_ms, Some(1)); + let _ = fs::remove_dir_all(root); + Ok(()) + } } diff --git a/apps/rocm/src/service_control.rs b/apps/rocm/src/service_control.rs index 2dd52108da..323da8ea4c 100644 --- a/apps/rocm/src/service_control.rs +++ b/apps/rocm/src/service_control.rs @@ -2772,4 +2772,50 @@ mod tests { ); } } + + /// The stop and the liveness read after it, end to end: a stop that cannot + /// verify a recorded process leaves the key and the stop request, and a later + /// refresh that still cannot query the process must not take them away or + /// reconcile the record to `stopped`. The refresh reads the record the stop + /// wrote, as a service listing does. + #[test] + fn an_unverified_stop_keeps_its_key_and_marker_through_a_later_liveness_refresh() { + let (root, paths) = test_paths("stop-unverified-then-refresh"); + paths.ensure().unwrap(); + let service_id = "svc-unverified-then-refresh"; + stop_test_record(&paths, service_id).write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + vec![stop_of( + 999_999_999, + rocm_core::TerminationOutcome::Unverified, + )] + }) + .unwrap(); + assert_eq!(report["stopped"], false, "{report}"); + let (mut record, _) = read_managed_service_manifest(&paths, service_id).unwrap(); + assert!( + record.stop_requested_unix_ms.is_some(), + "the stop is pending" + ); + + let changed = crate::managed_service_status::refresh_managed_service_runtime_liveness_with( + &paths, + &mut record, + |_| rocm_core::ProcessProbe::Unqueryable, + ); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + + assert_eq!(key.as_deref(), Some("key-123"), "the key is kept"); + assert!( + record.stop_requested_unix_ms.is_some(), + "the stop request is kept" + ); + assert!( + !changed, + "nothing about the record may change while the process is unqueryable" + ); + } } diff --git a/apps/rocm/src/service_records.rs b/apps/rocm/src/service_records.rs index 7be31d9579..2139db69ed 100644 --- a/apps/rocm/src/service_records.rs +++ b/apps/rocm/src/service_records.rs @@ -36,7 +36,7 @@ use std::path::{Path, PathBuf}; use std::time::{Duration, SystemTime}; use anyhow::{Context, Result, bail}; -use rocm_core::{AppPaths, ManagedServiceRecord, process_is_running}; +use rocm_core::{AppPaths, ManagedServiceRecord, ProcessProbe}; use crate::managed_service_status::{ load_managed_services, managed_service_is_live, recorded_service_pids, @@ -165,11 +165,24 @@ fn remove_service_record_artifacts(artifacts: &ServiceRecordArtifacts) -> Result /// server whose engine reported failure while its process is still up reads as /// removable, and deleting it takes the log and the 0600 endpoint key out from /// under a process that may still be serving. +/// +/// A recorded process that could not be queried (on Windows, a live one this +/// process may not open) counts as running: deleting the record and its key on +/// the strength of a failed look is the mistake this guard exists to prevent. fn managed_service_record_is_in_use(record: &ManagedServiceRecord) -> bool { + managed_service_record_is_in_use_with(record, rocm_core::probe_process) +} + +/// [`managed_service_record_is_in_use`] with the read of each recorded PID +/// supplied by the caller. +fn managed_service_record_is_in_use_with( + record: &ManagedServiceRecord, + probe: impl Fn(u32) -> ProcessProbe, +) -> bool { managed_service_is_live(record) || recorded_service_pids(record) .iter() - .any(|pid| process_is_running(*pid)) + .any(|pid| probe(*pid).may_be_running()) } /// Remove one non-running local server record and everything it owns. @@ -1795,4 +1808,27 @@ mod tests { assert!(manifest_exists, "a refused prune must delete nothing"); Ok(()) } + + /// A record whose status is not live is still in use while a process it + /// recorded may be running, and a process that could not be queried (on + /// Windows, a live one this process may not open) may be. Only a confirmed + /// exit makes it removable. + #[test] + fn a_record_is_in_use_unless_every_recorded_process_is_confirmed_gone() -> Result<()> { + let (root, paths) = test_paths("services-in-use-probe"); + paths.ensure()?; + let record = plant_service_record(&paths, "svc-in-use", "failed", 999_999_999)?; + let _ = fs::remove_dir_all(&root); + + assert!(managed_service_record_is_in_use_with(&record, |_| { + ProcessProbe::Unqueryable + })); + assert!(managed_service_record_is_in_use_with(&record, |_| { + ProcessProbe::Running + })); + assert!(!managed_service_record_is_in_use_with(&record, |_| { + ProcessProbe::Gone + })); + Ok(()) + } } diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 0884c70fb2..212aadc517 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -174,9 +174,9 @@ pub use proc_lifecycle::{ terminate_recorded_pid, terminate_recorded_service_pids, terminate_verified, }; pub use process::{ - FileLock, LoopbackPortLease, MANAGED_LAUNCH_LOCK_TIMEOUT, detach_command_session, + FileLock, LoopbackPortLease, MANAGED_LAUNCH_LOCK_TIMEOUT, ProcessProbe, detach_command_session, is_canonical_loopback_host, launch_port_argument, lease_loopback_port, - lease_loopback_port_or_bind_error, process_is_running, terminate_process, + lease_loopback_port_or_bind_error, probe_process, process_is_running, terminate_process, terminate_process_tree, }; #[cfg(windows)] diff --git a/crates/rocm-core/src/proc_lifecycle.rs b/crates/rocm-core/src/proc_lifecycle.rs index 0d57586320..dfe0628e19 100644 --- a/crates/rocm-core/src/proc_lifecycle.rs +++ b/crates/rocm-core/src/proc_lifecycle.rs @@ -502,6 +502,10 @@ pub fn terminate_recorded_service_pids( /// hold, the process could exit and an unrelated one take its PID between the /// check and the kill. /// +/// The descendant kill is bounded: a `taskkill` that has not finished within +/// [`MANAGED_STOP_GRACE`] is killed and reaped and counts as a tree kill that did +/// not complete, so a stalled helper cannot hang the stop. +/// /// A tree kill that did not complete leaves the stop unconfirmed, whatever the /// root's own termination says: [`terminate_verified`] can only watch the root /// on Windows, so a descendant that `taskkill` did not take down is one this stop @@ -631,9 +635,11 @@ fn platform_tree_kill() -> Option<&'static dyn Fn(u32) -> TreeKill> { /// The verdict on one `taskkill` run, from whether it started and the exit code /// it ended with. Only a clean exit (0) is a completed tree kill: a failed -/// launch, a nonzero code (a process it could not terminate, or none found) and a -/// run that ended without a code are all [`TreeKill::Failed`]. Split out of the -/// process spawn so a test runs it on every host. +/// launch, a run that did not finish within its deadline +/// ([`std::io::ErrorKind::TimedOut`], see [`run_to_exit_code`]), a nonzero code +/// (a process it could not terminate, or none found) and a run that ended +/// without a code are all [`TreeKill::Failed`]. Split out of the process spawn +/// so a test runs it on every host. #[cfg(any(windows, test))] fn classify_taskkill(run: std::io::Result>) -> TreeKill { match run { @@ -642,18 +648,69 @@ fn classify_taskkill(run: std::io::Result>) -> TreeKill { } } +/// How long the descendant kill may run. It happens before +/// [`terminate_verified`]'s bounded waits, so without its own deadline a stalled +/// helper would hang the stop; the same bound as every other wait of a managed +/// stop. +#[cfg(windows)] +const TREE_KILL_DEADLINE: Duration = MANAGED_STOP_GRACE; + +/// Start `command` and wait for it to exit, for at most `deadline`. Returns its +/// exit code (`None` when it ended without one). A child still running at the +/// deadline is killed and reaped, so none is left behind, and the call fails +/// with [`std::io::ErrorKind::TimedOut`]. +#[cfg(any(windows, test))] +fn run_to_exit_code( + command: &mut std::process::Command, + deadline: Duration, +) -> std::io::Result> { + let mut child = command.spawn()?; + wait_for_exit_code(&mut child, deadline) +} + +/// The wait of [`run_to_exit_code`], on a child already started. Kills and reaps +/// `child` when the deadline passes, or when it cannot be polled, rather than +/// leaving it running. +#[cfg(any(windows, test))] +fn wait_for_exit_code( + child: &mut std::process::Child, + deadline: Duration, +) -> std::io::Result> { + let end = Instant::now() + deadline; + loop { + match child.try_wait() { + Ok(Some(status)) => return Ok(status.code()), + Ok(None) => {} + Err(error) => { + let _ = child.kill(); + let _ = child.wait(); + return Err(error); + } + } + let now = Instant::now(); + if now >= end { + let _ = child.kill(); + let _ = child.wait(); + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "the process did not exit before its deadline", + )); + } + std::thread::sleep(POLL_INTERVAL.min(end.saturating_duration_since(now))); + } +} + #[cfg(windows)] fn taskkill_tree(pid: u32) -> TreeKill { use std::process::{Command, Stdio}; - classify_taskkill( + classify_taskkill(run_to_exit_code( Command::new("taskkill") .args(["/PID", &pid.to_string(), "/T", "/F"]) .stdin(Stdio::null()) .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - .map(|status| status.code()), - ) + .stderr(Stdio::null()), + TREE_KILL_DEADLINE, + )) } /// Poll until every process in `members` has exited, or `grace` elapses. @@ -1589,6 +1646,100 @@ mod tests { } } + /// A command that exits at once with `code`, and one that runs far longer + /// than any test waits, on whichever platform the tests run. + fn exits_with(code: i32) -> std::process::Command { + #[cfg(unix)] + let mut command = { + let mut command = std::process::Command::new("sh"); + command.args(["-c", &format!("exit {code}")]); + command + }; + #[cfg(windows)] + let mut command = { + let mut command = std::process::Command::new("cmd"); + command.args(["/C", &format!("exit {code}")]); + command + }; + command + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()); + command + } + + fn runs_for_thirty_seconds() -> std::process::Command { + #[cfg(unix)] + let mut command = { + let mut command = std::process::Command::new("sleep"); + command.arg("30"); + command + }; + #[cfg(windows)] + let mut command = { + let mut command = std::process::Command::new("ping"); + command.args(["-n", "30", "127.0.0.1"]); + command + }; + command + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()); + command + } + + #[test] + fn a_helper_that_exits_in_time_gives_its_exit_code() { + let deadline = Duration::from_secs(20); + + assert_eq!( + run_to_exit_code(&mut exits_with(0), deadline).unwrap(), + Some(0) + ); + assert_eq!( + run_to_exit_code(&mut exits_with(3), deadline).unwrap(), + Some(3) + ); + } + + #[test] + fn a_helper_that_cannot_be_started_is_an_error() { + let mut command = std::process::Command::new("rocm-core-test-no-such-program"); + + assert!(run_to_exit_code(&mut command, Duration::from_secs(1)).is_err()); + } + + /// A helper that stalls must not hold the stop: it is killed and reaped at + /// its deadline, the call ends close to it, and the verdict is a failed tree + /// kill, which leaves the stop unconfirmed. + #[test] + fn a_helper_that_stalls_is_killed_and_reaped_at_its_deadline() { + let mut command = runs_for_thirty_seconds(); + let mut child = command.spawn().expect("spawn the stalled helper"); + let pid = child.id(); + let started = Instant::now(); + + let run = wait_for_exit_code(&mut child, Duration::from_millis(200)); + let elapsed = started.elapsed(); + + let error = run.expect_err("a stalled helper does not finish"); + assert_eq!(error.kind(), std::io::ErrorKind::TimedOut); + assert!( + elapsed >= Duration::from_millis(200) && elapsed < Duration::from_secs(10), + "returned after {elapsed:?}, far from the deadline" + ); + // Reaped: its status is already collected, and the process is gone. + assert!( + matches!(child.try_wait(), Ok(Some(_))), + "the killed helper was waited for" + ); + assert!( + !crate::process_is_running(pid), + "the helper was killed, not left running" + ); + assert_eq!(classify_taskkill(Err(error)), TreeKill::Failed); + } + #[test] fn only_a_clean_taskkill_exit_is_a_completed_tree_kill() { use std::io::{Error, ErrorKind}; @@ -1598,11 +1749,15 @@ mod tests { assert_eq!(classify_taskkill(Ok(Some(128))), TreeKill::Failed); // Ended without an exit code. assert_eq!(classify_taskkill(Ok(None)), TreeKill::Failed); - // Could not be started. + // Could not be started, or did not finish before its deadline. assert_eq!( classify_taskkill(Err(Error::from(ErrorKind::NotFound))), TreeKill::Failed ); + assert_eq!( + classify_taskkill(Err(Error::from(ErrorKind::TimedOut))), + TreeKill::Failed + ); } /// A PID that cannot be queried is neither gone nor a match, through the diff --git a/crates/rocm-core/src/process.rs b/crates/rocm-core/src/process.rs index 74119a37a6..611e39d2e2 100644 --- a/crates/rocm-core/src/process.rs +++ b/crates/rocm-core/src/process.rs @@ -369,11 +369,8 @@ pub fn terminate_process_tree(pid: u32) -> Result<()> { /// but cannot be queried reads the same as one that is gone. A stop that is about /// to report a process as exited must not make that mistake, so this keeps the /// third answer apart. -// Only the Windows probe can fail to query a live process, so nothing builds -// `Unqueryable` elsewhere; the decision on it is shared by every platform. -#[cfg_attr(not(any(windows, test)), allow(dead_code))] #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub(crate) enum ProcessProbe { +pub enum ProcessProbe { /// The PID names a process that has not exited. Running, /// The PID names no process, or one that has exited. @@ -384,6 +381,17 @@ pub(crate) enum ProcessProbe { Unqueryable, } +impl ProcessProbe { + /// Whether the process may still be running: everything but a confirmed + /// exit. The answer for a decision that must not treat a process it could + /// not look at as gone, such as dropping a service's endpoint key, marking + /// its record stopped or deleting it. + #[must_use] + pub const fn may_be_running(self) -> bool { + !matches!(self, Self::Gone) + } +} + /// Windows `ERROR_INVALID_PARAMETER`: what `OpenProcess` sets for a PID that /// names no process. #[cfg(any(windows, test))] @@ -406,8 +414,9 @@ pub(crate) const fn classify_open_failure(code: Option) -> ProcessProbe { /// Look at `pid`, keeping a process that cannot be queried apart from one that /// is gone. See [`ProcessProbe`]. #[cfg(windows)] +#[must_use] #[allow(unsafe_code)] // Win32 FFI -pub(crate) fn probe_process(pid: u32) -> ProcessProbe { +pub fn probe_process(pid: u32) -> ProcessProbe { use windows_sys::Win32::Foundation::CloseHandle; if pid == 0 { @@ -434,7 +443,8 @@ pub(crate) fn probe_process(pid: u32) -> ProcessProbe { /// not signal (`EPERM`, which counts as running) from one that is gone, so there /// is no third answer to keep apart here. #[cfg(not(windows))] -pub(crate) fn probe_process(pid: u32) -> ProcessProbe { +#[must_use] +pub fn probe_process(pid: u32) -> ProcessProbe { if process_is_running(pid) { ProcessProbe::Running } else { @@ -442,6 +452,12 @@ pub(crate) fn probe_process(pid: u32) -> ProcessProbe { } } +/// Whether `pid` names a running process. +/// +/// On Windows a live process this one may not open reads as not running here; a +/// decision that must not take it for gone (dropping a key, marking a record +/// stopped, deleting it) asks [`probe_process`] instead and keeps +/// [`ProcessProbe::Unqueryable`] apart. #[cfg(windows)] #[allow(unsafe_code)] // Win32 FFI pub fn process_is_running(pid: u32) -> bool { diff --git a/docs/architecture.md b/docs/architecture.md index b1d12d3327..93f2111478 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -70,7 +70,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-214): `--device`/`--gpu` parsing (`parse_device_policy`, `parse_gpu_selection`, `parse_gpu_indices_arg`), the managed-launch-lock GPU selection (`select_gpu_indices_under_launch_lock`, `validate_pinned_gpu_index`, `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `busy_gpu_indices`), per-GPU VRAM telemetry (the `GpuVramUsage` type, `gpu_vram_usage`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs`, `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`), and the serve-plan low-VRAM warning (`gpu_low_memory_warning`, `vram_capacity_is_meaningful`, `serve_gpu_low_memory_warning`), plus the best-effort `amd-smi`-based GPU count (`detect_gpu_count`).Both `amd-smi` probes go through `rocm_core::amd_smi_json` (the `/dev/kfd` pre-flight) with this module's own `AMD_SMI_SERVE_PROBE_TIMEOUT`, whose doc records the resulting worst-case `serve` latency. `const fn device_policy_name` sits immediately after this cluster's last function in the original file but stayed in `main.rs` rather than moving — it already had callers outside the cluster (`engine_ipc.rs`, `service_launch.rs`, `serve_cmd.rs`) plus its own test. `GpuVramUsage` (and its fields), `gpu_vram_usage`, `parse_gpu_indices_arg`, `serve_gpu_low_memory_warning`, and `vram_capacity_is_meaningful` are `pub(crate)` and read from [`model_fit.rs`](../apps/rocm/src/model_fit.rs) (`classify_accelerator_memory`, `host_accelerator_memory`) [`main.rs`](../apps/rocm/src/main.rs) (the `dispatch()` `Command::EngineServeHttp` arm) and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) (`collect_serve_notes`); `detect_gpu_count`, `gpu_vram_usage`, `parse_device_policy`, `parse_gpu_selection`, `select_gpu_indices_under_launch_lock`, `serve_gpu_low_memory_warning`, and `validate_pinned_gpu_index` are also `pub(crate)` and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs); `parse_device_policy` is also read from [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs)'s `run_builtin_engine_serve_http` and, as of Phase 6f (ROCMAI-209), from [`service_control.rs`](../apps/rocm/src/service_control.rs)'s `restart_child_argv` (moved out of `main.rs` in that same phase, and since relocated from `services_text.rs`). `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs` (both the Linux and non-Linux `cfg` variants), `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`, `gpu_low_memory_warning`, and `busy_gpu_indices` have no callers outside this module and stay private. The cluster's regression tests live in `gpu_policy.rs`'s own `mod tests`; two small test-only fixture builders (`vram`, `host_gpu`) are duplicated rather than shared, since `serve_cmd.rs`'s own `serve_notes_pair_low_vram_with_the_vllm_utilization_hint` (testing `collect_serve_notes`) needs them too. - [`install_cmd.rs`](../apps/rocm/src/install_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm install`'s command handler (`install()`, for the `sdk` and `driver` targets), the `--version`/`--build-date` selector parsing and display it uses (`therock_install_version_selector`, `therock_install_version_selector_display`) and `sdk_install_request`, which maps the parsed `install sdk` arguments onto a `therock::SdkInstallRequest` and moved here from [`chat.rs`](../apps/rocm/src/chat.rs) with its two tests. `InstallTarget` and `InstallFormat` live in [`cli.rs`](../apps/rocm/src/cli.rs). `install()` is `pub(crate)` and re-imported in `main.rs` for `dispatch()`; `therock_install_version_selector` is `pub(crate)` too, and `chat/mcp_bridge.rs`'s `render_install_sdk_dry_run_for_args` reaches it as `crate::therock_install_version_selector` through the same kind of re-import, which is why `sdk_install_request` lives here and not in `chat.rs`: this module does not depend on `chat.rs`. `sdk_install_request` keeps the `pub(crate)` it had, although only `install()` and the tests call it now. - [`logging.rs`](../apps/rocm/src/logging.rs) — mechanical relocation, no owned types, reached by qualified path. -- [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-211): managed-service liveness classification (`managed_service_is_live`, `refresh_managed_service_runtime_liveness`, `existing_live_managed_service_in`'s idempotency guard) and the dashboard-sidebar status/counts helpers derived from it (`ManagedServiceSidebarCounts`/`managed_service_sidebar_counts`/`local_server_sidebar_status`), plus the engine display-name helper `friendly_engine_label`, read only by [`chat/summaries.rs`](../apps/rocm/src/chat/summaries.rs). `ManagedServiceSidebarCounts` is `pub(crate)` to match the visibility of the `pub(crate)` functions that return/take it; `friendly_engine_label`, `existing_live_managed_service_in`, `recorded_service_pids`, `refresh_managed_service_runtime_liveness`, `local_server_sidebar_status`, `managed_service_sidebar_counts`, and `load_managed_services` are `pub(crate)` for the same reason — each has a confirmed caller outside this module. [`service_records.rs`](../apps/rocm/src/service_records.rs) uses its liveness helpers (`managed_service_is_live`, `recorded_service_pids`) for the in-use check behind service-record removal and pruning, and `load_managed_services` for prune planning, while [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs) uses `load_managed_services` for the codex-bridge snapshot; [`service_launch.rs`](../apps/rocm/src/service_launch.rs) is the only caller of the `existing_live_managed_service_in` duplicate-launch guard; [`services_text.rs`](../apps/rocm/src/services_text.rs) reads the sidebar helpers; other modules, including [`chat/turn.rs`](../apps/rocm/src/chat/turn.rs), [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs), read the registry and liveness helpers. +- [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-211): managed-service liveness classification (`managed_service_is_live`, `refresh_managed_service_runtime_liveness`, `existing_live_managed_service_in`'s idempotency guard) and the dashboard-sidebar status/counts helpers derived from it (`ManagedServiceSidebarCounts`/`managed_service_sidebar_counts`/`local_server_sidebar_status`), plus the engine display-name helper `friendly_engine_label`, read only by [`chat/summaries.rs`](../apps/rocm/src/chat/summaries.rs). `ManagedServiceSidebarCounts` is `pub(crate)` to match the visibility of the `pub(crate)` functions that return/take it; `friendly_engine_label`, `existing_live_managed_service_in`, `recorded_service_pids`, `refresh_managed_service_runtime_liveness`, `local_server_sidebar_status`, `managed_service_sidebar_counts`, and `load_managed_services` are `pub(crate)` for the same reason — each has a confirmed caller outside this module. [`service_records.rs`](../apps/rocm/src/service_records.rs) uses its liveness helpers (`managed_service_is_live`, `recorded_service_pids`) for the in-use check behind service-record removal and pruning, both of which, like the deferred key cleanup and the reconciliation to `stopped` here, take a recorded process for gone only when `probe_process` confirms it (a process that cannot be queried counts as possibly running), and `load_managed_services` for prune planning, while [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs) uses `load_managed_services` for the codex-bridge snapshot; [`service_launch.rs`](../apps/rocm/src/service_launch.rs) is the only caller of the `existing_live_managed_service_in` duplicate-launch guard; [`services_text.rs`](../apps/rocm/src/services_text.rs) reads the sidebar helpers; other modules, including [`chat/turn.rs`](../apps/rocm/src/chat/turn.rs), [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs), read the registry and liveness helpers. - [`model_catalog_text.rs`](../apps/rocm/src/model_catalog_text.rs) — mechanical relocation out of `examine_report_text.rs`: the `rocm model` (alias `models`) catalog renderer. `render_model_registry_text_with_context_and_host` prints the curated list grouped by hardware target (`model_catalog_platform_heading`, `model_catalog_header`, `model_recipe_memory_label`, `model_recipe_gpu_fit_label`), and `render_model_registry_verbose_text_with_context_and_host` prints the per-recipe diagnostics behind `rocm model --verbose` (`append_model_recipe_registry_source`, `append_model_recipe_metadata_lines`, `append_model_engine_recipe_settings_lines`, `append_model_recipe_staleness_lines`, `append_model_artifact_lines`, `append_artifact_source_policy_lines`, `artifact_source_policy_label`, `append_model_host_ram_fit_lines`, `append_model_fit_lines`, `append_manual_alternative_lines`, `manual_alternative_recommendations`, `recipe_is_manual_fit`, `append_model_engine_support_lines`, `model_registry_adapter_availability_note`, `recipe_display_ref`, `preferred_engine_action_target`, `model_registry_reason`, plus the small formatters `format_list_or_none`, `format_string_map_or_none`, `format_gib` and `once_per_call`). `format_bytes`, the human-readable size formatter, lives here too. The two renderers and `format_bytes` are `pub(crate)`: [`main.rs`](../apps/rocm/src/main.rs) imports them for the `Command::Model` arm, and [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs) (the verbose renderer), [`storage.rs`](../apps/rocm/src/storage.rs) and [`automations.rs`](../apps/rocm/src/automations.rs) (`format_bytes`) reach them as `crate::`. Every other item, including `model_registry_adapter_availability_note`, is private. The module has no edge to `examine_report_text.rs`; it reaches the engine-availability helpers in [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs), `PlatformRuleOut` and `engine_ruled_out_by_platform_with` at the crate root, and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs)'s `select_serve_engine`. The catalog's regression tests live in this module's own `mod tests`; they keep the names they had in `examine_report_text.rs`. The `format_bytes` property test's saved failure case lives in [`model_catalog_text.txt`](../apps/rocm/proptest-regressions/model_catalog_text.txt), which proptest finds by the test's source file, so the saved case has to follow the test when it moves (see [Property tests](testing.md#property-tests) for how, which depends on whether the whole file moves). - [`model_fit.rs`](../apps/rocm/src/model_fit.rs) — mechanical relocation, owns a `pub(crate)` type: answering whether a host can run a model for `rocm diagnose --model` (`assess_model_on_this_host`, `assess_model_with_host_paths`, `assess_model_for_host`), the accelerator-memory fact it rests on (`host_accelerator_memory`, `classify_accelerator_memory`, `examination_reports_amd_gpu`) and the platform gate that rules an engine out (`PlatformRuleOut`, `engine_ruled_out_by_platform`, `unsupported_here_for`). It is a module of its own, rather than part of the `diagnose` handler, so [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs) and [`model_catalog_text.rs`](../apps/rocm/src/model_catalog_text.rs), which read `PlatformRuleOut` and the platform gate (`engine_ruled_out_by_platform` and `engine_ruled_out_by_platform_with` respectively), do not depend on a command module. It depends on [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) (`select_serve_engine`) and [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs), neither of which depends on it. `assess_model_on_this_host` (called from [`diagnose_cmd.rs`](../apps/rocm/src/diagnose_cmd.rs)), `PlatformRuleOut` and `engine_ruled_out_by_platform` are `pub(crate)`, and [`main.rs`](../apps/rocm/src/main.rs) re-imports the last two so `examine_report_text.rs` reaches them as `crate::` (`model_catalog_text.rs` reaches `PlatformRuleOut` and `engine_ruled_out_by_platform_with` the same way); every other item is private. The cluster's regression tests live in this module's own `mod tests`; the one that re-runs the test binary on a helper test by its full path (`engine_ruled_out_by_platform_gates_aim_on_docker_reachability`) names the helper's new path, `model_fit::tests::docker_unreachable_subprocess_helper`. - [`prompt_text.rs`](../apps/rocm/src/prompt_text.rs) — text extraction from a natural-language request, shared by `chat/intent.rs`'s prompt-intent heuristics (`fallback_rocm_tool_call_for_prompt`, `fallback_config_tool_call_for_prompt`, `install_sdk_chat_approval_for_prompt` and the `prompt_*` predicates) and the freeform planner, in a leaf module that depends only on `therock::RuntimeVersionSelector` (`tool_policy.rs` does not use it): `any_substring`, the install-folder, TheRock build-date and TheRock version readers (`requested_install_prefix_from_prompt`, `requested_therock_build_date_from_prompt`, `requested_therock_version_from_prompt`, with `requested_bare_install_prefix_from_prompt`, `clean_requested_install_prefix`, `looks_like_user_path` and `looks_like_therock_runtime_version` private beside them) from `chat.rs`, and `infer_engine_from_request` from `freeform_planner.rs`. They moved verbatim, together with `infer_engine_from_request`'s regression test, so that `freeform_planner.rs` no longer imports `chat` for them; `chat.rs` re-exports `any_substring`, which [`freeform_answer.rs`](../apps/rocm/src/freeform_answer.rs) still reaches as `chat::any_substring`. @@ -231,7 +231,7 @@ Further `rocm-core` modules, each with its own doc comment except where noted: - [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) — OpenMPI, libatomic and libnuma detection, the `libmpi_cxx` compatibility shim, and the per-library install plans (`build_openmpi_install_plan`, `build_libatomic_install_plan`, `build_libnuma_install_plan`) and hints, needed by vLLM's ROCm wheels. The distro-generic half of that planning lives in [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs); [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) re-exports its public items, so `rocm_core::openmpi::running_as_root`, `plan_for_os_release_read` and the rest keep resolving, and `pub(crate)` re-exports `PackageManager` and `resolve_package_manager` for [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs). - [`os_release.rs`](../crates/rocm-core/src/os_release.rs) — the one `/etc/os-release` parser every reader of that file uses. - [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs) — the planning half of a system-package install that does not depend on which library is installed: `PackageManager` and `resolve_package_manager` (from an os-release `ID`/`ID_LIKE`), `SystemPackageInstallPlan` and `InstallCommand`, `plan_for_os_release_at`/`plan_for_os_release_read` (which take the per-library builder as an argument), `InstallHint`, the root and passwordless-`sudo` checks (`running_as_root`, `can_autoinstall`) and the `apt-get -s` helpers (`apt_simulate_argv`, `parse_apt_simulate_removals`, `is_protected_rocm_package`). It never runs anything and depends on no other module of this crate except [`os_release.rs`](../crates/rocm-core/src/os_release.rs) and [`hardware_root.rs`](../crates/rocm-core/src/hardware_root.rs), which it reads os-release through. The module is private; its public items are re-exported from [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs). It is a different module from [`system_packages.rs`](../apps/rocm/src/system_packages.rs) in `apps/rocm`, which runs these plans with consent and `sudo`. -- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches (`terminate_verified`), and the one managed-stop routine built on it that `rocm services stop` and `rocmd`'s `stop_server` both call: `terminate_recorded_service_pids` walks the processes a service record names (supervisor, then engine, each with its own start-time, the stopping process's own PID reported as `self` and never signalled) and returns one `RecordedPidStop` per PID (each caller, on a stop that confirms every one gone, then clears the record's recorded processes with `ManagedServiceRecord::clear_recorded_processes` in [`service_record.rs`](../crates/rocm-core/src/service_record.rs), so a repeated stop reaches no stale PID; an unconfirmed stop keeps them); `terminate_recorded_pid` terminates one of them, adding on Windows a `taskkill /T /F` of its descendants first, because `terminate_verified` cannot walk a process tree there, and only for a PID that passes the same identity check (`process_start_ticks` is the kernel start-time on Linux and the process creation time on Windows, so a recycled PID is refused before any `taskkill`; a record written before Windows recorded one cannot be refuted), holding the PID open on Windows from before that check until the stop ends so the PID cannot be reused in between, and treating a tree kill that did not complete as an unconfirmed stop; a PID that cannot be queried at all, as against one that is gone, reads as unverifiable (`ProcessProbe` in [`process.rs`](../crates/rocm-core/src/process.rs)); `MANAGED_STOP_GRACE` is the one grace period. The termination is injectable, so a caller's test supplies outcomes it cannot construct and a test of the Windows step runs on every host. +- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches (`terminate_verified`), and the one managed-stop routine built on it that `rocm services stop` and `rocmd`'s `stop_server` both call: `terminate_recorded_service_pids` walks the processes a service record names (supervisor, then engine, each with its own start-time, the stopping process's own PID reported as `self` and never signalled) and returns one `RecordedPidStop` per PID (each caller, on a stop that confirms every one gone, then clears the record's recorded processes with `ManagedServiceRecord::clear_recorded_processes` in [`service_record.rs`](../crates/rocm-core/src/service_record.rs), so a repeated stop reaches no stale PID; an unconfirmed stop keeps them); `terminate_recorded_pid` terminates one of them, adding on Windows a `taskkill /T /F` of its descendants first, because `terminate_verified` cannot walk a process tree there, and only for a PID that passes the same identity check (`process_start_ticks` is the kernel start-time on Linux and the process creation time on Windows, so a recycled PID is refused before any `taskkill`; a record written before Windows recorded one cannot be refuted), holding the PID open on Windows from before that check until the stop ends so the PID cannot be reused in between, and treating a tree kill that did not complete, or did not finish within `MANAGED_STOP_GRACE` (the `taskkill` is killed and reaped at that deadline), as an unconfirmed stop; a PID that cannot be queried at all, as against one that is gone, reads as unverifiable (`ProcessProbe` and `probe_process` in [`process.rs`](../crates/rocm-core/src/process.rs), whose `may_be_running` is the answer for any decision that must not take an unqueryable process for gone); `MANAGED_STOP_GRACE` is the one grace period. The termination is injectable, so a caller's test supplies outcomes it cannot construct and a test of the Windows step runs on every host. - [`report.rs`](../crates/rocm-core/src/report.rs) — the content of a Doctor report and the rule deciding whether one may exist; it sends nothing. - [`report_delivery.rs`](../crates/rocm-core/src/report_delivery.rs) — how a report leaves the machine: building its link and deciding whether a browser may be started. - [`runtime.rs`](../crates/rocm-core/src/runtime.rs) — the host-platform layer: `RuntimePlatform`/`RuntimeHost`, OS-specific executable and Python-environment names (including `runtime_python_env_root_for_executable`, the one rule for finding an environment's folder from its interpreter's path: only an interpreter directly inside a `bin` or `Scripts` folder, in any case, has one, and everything else gets `None`; the adopt command, the ComfyUI runtime environment and the vLLM launch all call it rather than keeping a copy, since the engine crate cannot reach `apps/rocm`), default config/data/cache directories, path-text and path-list helpers, and `install_root_has_payload`, the one definition of a non-empty install root (dot-entries such as the runtime marker do not count; a path that does not exist holds nothing; a root that cannot be listed is an error) that tarball extraction, update repair detection and activation validation in `apps/rocm` all ask (no module doc comment). diff --git a/docs/testing.md b/docs/testing.md index e80e4fd025..54ca7ac027 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1852,11 +1852,30 @@ time that was compared is that of the process the kill then reaches reported as `timed_out`, whatever the root's own termination says, because only the root's exit is watched on Windows (`a_tree_kill_that_did_not_complete_leaves_the_stop_unconfirmed`, -`only_a_clean_taskkill_exit_is_a_completed_tree_kill`); the report and the record +`only_a_clean_taskkill_exit_is_a_completed_tree_kill`). That `taskkill` runs under +a deadline of `MANAGED_STOP_GRACE`: one that stalls is killed and reaped and +counts as a failed tree kill, so it cannot hang `rocm services stop`, `restart` +or the attached Ctrl-C stop +(`a_helper_that_stalls_is_killed_and_reaped_at_its_deadline` runs a long-sleeping +child under a 200 ms deadline on every host and asserts the error, the bounded +elapsed time, the reaped child and the failed verdict; `a_helper_that_exits_in_time_gives_its_exit_code` +and `a_helper_that_cannot_be_started_is_an_error` are its neighbours). The report and the record a `timed_out` stop leaves are covered by `an_unconfirmed_stop_keeps_the_recorded_processes_for_the_next_stop`. What no test here reaches is the Windows API itself: the handle hold, the open-failure codes and a real `taskkill` failing run only on the Windows lane. + +The same unqueryable-process rule holds wherever a service's liveness decides +something destructive. The deferred key cleanup and the reconciliation to +`stopped` in the liveness refresh (`refresh_managed_service_runtime_liveness`) and +the in-use check behind `rocm services remove` and `prune` read each recorded PID +through `probe_process` and treat anything but a confirmed exit as possibly +running. `a_pending_stop_keeps_its_key_and_marker_while_a_process_cannot_be_queried` +drives the refresh with an unqueryable PID and then a confirmed-gone one, +`an_unverified_stop_keeps_its_key_and_marker_through_a_later_liveness_refresh` runs +an unverified stop and then the refresh over the record it wrote, and +`a_record_is_in_use_unless_every_recorded_process_is_confirmed_gone` covers the +removal guard; each takes the probe as an argument, so they run on every host. `service-stop-06` (Linux) is the same stop of a real recorded process through `rocm services stop`, checking the persisted state: the process exited and the record names none. Run the Windows scenario on a Windows host with From ec3888e097887766f50c0e5c5905f05c428e0599 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 14:52:57 +0000 Subject: [PATCH 12/30] fix(core): treat a taskkill that found no process as a stop when the root is gone, let a readable token mismatch outrank a failed query, and document the Windows unverified routes and the way out A taskkill that exits 128 (process not found) was a failed tree kill, so a root that exited by itself between the identity check and the kill left the stop unconfirmed and the record kept although everything had exited. It is now its own result, judged by the root's re-read: with the root gone the stop reports it already gone, with the root still running the kill counts as failed, and any other nonzero exit stays a failed kill. A recorded start-time that can be read and differs proves another process holds the PID, so it now outranks a failed liveness query. On Windows the token and the query are read with the same access right, so this only reaches a process that opens but fails the exit-code query; a process that cannot be opened has no readable token and stays unverifiable. That leaves one way for a record to stay unverified for good: a stopped service's PID taken by a process the account cannot open. The code comment, the README, the remove help and docs/manual-testing.md now say so and give the way out that exists: end the process if it is the server's, or delete the record's details file by hand and run services prune --any-age --yes. A test ties the two halves together: such a record is refused as in use, and once the details file is deleted, prune sweeps the log, the key and the engine state. The unverified outcome has two routes on Windows (a recorded token that cannot be read back, and a PID that cannot be queried at all), where the process_check_command and render_unconfirmed_service_stop doc comments, the stop help, the README and the e2e hook comment named one. IdentityMismatch's doc now says why it counts as stopped. docs/testing.md names the two scenario gaps (no tokened Windows scenario; no scenario for the self row, which only the hidden sandbox answer carries) and describes the no-private-copy source checks as a tripwire. Signed-off-by: Jussi Elo --- README.md | 11 +- apps/rocm/src/cli.rs | 16 +- apps/rocm/src/service_control.rs | 3 +- apps/rocm/src/service_records.rs | 38 +++++ apps/rocm/src/services_text.rs | 17 ++- crates/rocm-core/src/proc_lifecycle.rs | 196 +++++++++++++++++++++---- crates/rocm-core/src/process.rs | 11 ++ docs/architecture.md | 2 +- docs/manual-testing.md | 33 +++++ docs/testing.md | 32 +++- 10 files changed, 309 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 22cb10f496..e5fdcfba46 100644 --- a/README.md +++ b/README.md @@ -1236,8 +1236,9 @@ gone. If it cannot confirm that, it prints `Local server stop not confirmed` on stderr and exits with an error. It keeps the record, which still names the processes, and any endpoint key, which the server may still be enforcing. It names each process that, or a process under it, did not exit after it was -signaled; for a process it could not verify as the server's, it signals nothing -and names an operating-system command to check on it. It does not suggest +signaled; for a process it could not verify as the server's (its recorded +start-time can no longer be read back, or, on Windows, `rocm` cannot query it at +all), it signals nothing and names an operating-system command to check on it. It does not suggest running the stop again: when only a process under the recorded one survived, a second stop can report the server stopped while that process still runs. @@ -1256,7 +1257,11 @@ bounded (5 seconds), unlike `rocm serve`'s: if a launch still holds the lock, `remove` deletes one record that is no longer running, together with its log, its engine state file, and its endpoint key file; a running server is refused, so stop it first. `prune` does the same in bulk, always leaves running servers -alone, and additionally clears leftover files whose record is already gone. +alone, and additionally clears leftover files whose record is already gone. A +server counts as running while any process it recorded may be running, which on +Windows includes one `rocm` cannot query (a live process the account may not +open): `remove` and `prune` refuse such a record until that process is gone. +`docs/manual-testing.md` describes clearing one by hand. Removal destroys both the log and the `restart` option for the records it takes, so `prune` only considers records untouched for 24 hours. Age is diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index eeed2127f8..3f59708942 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -1124,8 +1124,10 @@ pub(crate) enum ServicesCommand { /// confirmed and exits with an error, keeping the record, which still names /// the processes, and any endpoint key, which the server may still be /// enforcing. It names each process that did not exit, and for one it could - /// not verify it signals nothing and names an operating-system command to - /// check on it. It does not suggest running the stop again. + /// not verify (its recorded start-time can no longer be read back, or, on + /// Windows, it cannot be queried at all) it signals nothing and names an + /// operating-system command to check on it. It does not suggest running the + /// stop again. Stop { /// Service id from `rocm services list`. service_id: String, @@ -1147,10 +1149,12 @@ pub(crate) enum ServicesCommand { }, /// Delete one local server record and its files. /// - /// Only works on a record that is not running: stop the server first. The - /// record's details file, log, engine state file, and endpoint key file are - /// all deleted, so its log can no longer be read and it can no longer be - /// restarted. + /// Only works on a record that is not running: stop the server first. A + /// server counts as running while any process it recorded may be running, + /// which on Windows includes one `rocm` cannot query (a live process the + /// account may not open). The record's details file, log, engine state file, + /// and endpoint key file are all deleted, so its log can no longer be read + /// and it can no longer be restarted. Remove { /// Service id from `rocm services list --all`. service_id: String, diff --git a/apps/rocm/src/service_control.rs b/apps/rocm/src/service_control.rs index 323da8ea4c..3a8e72be7a 100644 --- a/apps/rocm/src/service_control.rs +++ b/apps/rocm/src/service_control.rs @@ -142,7 +142,8 @@ fn stop_recorded_processes(record: &ManagedServiceRecord) -> Vec.json` by hand and running + /// `prune --any-age` leaves nothing behind (log, key and engine state all + /// swept), with the endpoint key among the files removed. + #[test] + fn a_record_held_by_an_unqueryable_process_can_be_cleared_by_hand_and_prune() -> Result<()> { + let (root, paths) = test_paths("services-unqueryable-way-out"); + paths.ensure()?; + let record = plant_service_record(&paths, "svc-stuck", "stopped", 999_999_999)?; + let refused = managed_service_record_is_in_use_with(&record, |_| ProcessProbe::Unqueryable); + let key = endpoint_keys::endpoint_key_file_path(&paths, "svc-stuck"); + assert!(key.exists(), "the planted record has its key"); + + fs::remove_file(&record.manifest_path)?; + let outcome = prune_managed_service_records(&paths, 0, false, true); + let outcome = match outcome { + Ok(outcome) => outcome, + Err(error) => { + let _ = fs::remove_dir_all(&root); + return Err(error); + } + }; + let leftovers = [&record.log_path, &record.engine_state_path, &key] + .iter() + .filter(|path| path.exists()) + .count(); + let _ = fs::remove_dir_all(&root); + + assert!( + refused, + "a record whose process cannot be queried is in use" + ); + assert_eq!(leftovers, 0, "prune swept every leftover file"); + assert_eq!(outcome.removed_files, 3); + Ok(()) + } } diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index 9fa4f1063e..43c9658719 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -1019,8 +1019,11 @@ pub(crate) fn render_service_action_result(tool: &str, value: &serde_json::Value /// - a process that was signalled and that, or a process under it, outlived /// the deadline (`timed_out`); /// - a process that was not signalled because its identity could not be -/// verified (`unverified`), with how to look at it. It may have exited -/// meanwhile: the identity read can fail because the process is going away; +/// verified (`unverified`), with how to look at it. There are two routes to +/// it: a recorded start-time that can no longer be read back from a running +/// process, and, on Windows, a PID that cannot be queried at all. It may have +/// exited meanwhile: the identity read can fail because the process is going +/// away; /// - the record changed while the stop ran, typically because a restart /// recorded new processes, so the stop's verdict says nothing about what the /// record names now. @@ -1140,10 +1143,12 @@ fn render_unconfirmed_service_stop(output: &serde_json::Value) -> String { /// stop left alone because it could not verify it. Not a `rocm` command: the /// CLI deliberately does not act on that process, so the user decides. /// -/// Linux and Windows can both produce that outcome: it needs a recorded -/// start-time that can no longer be read back, and `rocm_core::process_start_ticks` -/// records one on both. So the command is the one the host has: `ps` on Linux, -/// `tasklist` on Windows. +/// Linux and Windows can both produce that outcome, by two routes. On both, a +/// recorded start-time that can no longer be read back from a running process +/// (`rocm_core::process_start_ticks` records one on both). On Windows also, a PID +/// that cannot be queried at all, such as a live process this one may not open, +/// which reads as unverifiable with or without a recorded token. So the command +/// is the one the host has: `ps` on Linux, `tasklist` on Windows. pub(crate) fn process_check_command(pid: u64) -> String { process_check_command_for(cfg!(windows), pid) } diff --git a/crates/rocm-core/src/proc_lifecycle.rs b/crates/rocm-core/src/proc_lifecycle.rs index dfe0628e19..e4a2fc75d8 100644 --- a/crates/rocm-core/src/proc_lifecycle.rs +++ b/crates/rocm-core/src/proc_lifecycle.rs @@ -106,7 +106,17 @@ pub fn identity_state(id: &ProcessIdentity) -> IdentityState { /// /// A PID that cannot be queried is [`IdentityState::Indeterminate`], not /// [`IdentityState::Gone`]: a failure to look is not an exit, and reading it as -/// one would let a stop report a live process as stopped. +/// one would let a stop report a live process as stopped. That is one of the two +/// routes to [`IdentityState::Indeterminate`] and so to an `unverified` stop; the +/// other is a recorded token that cannot be read back from a running process +/// ([`classify_live_identity`]). +/// +/// A token that was recorded and can still be read, and that differs, outranks +/// the failed query: it proves a different process holds the PID, so the +/// recorded one is gone. On Windows this narrows to the case where the process +/// opens but the exit-code query fails: the token and the probe both open the +/// PID with `PROCESS_QUERY_LIMITED_INFORMATION`, so a process that cannot be +/// opened has no readable token either, and stays [`IdentityState::Indeterminate`]. fn identity_state_with( id: &ProcessIdentity, probe: impl Fn(u32) -> ProcessProbe, @@ -115,7 +125,10 @@ fn identity_state_with( ) -> IdentityState { match probe(id.pid) { ProcessProbe::Gone => IdentityState::Gone, - ProcessProbe::Unqueryable => IdentityState::Indeterminate, + ProcessProbe::Unqueryable => match (id.start_ticks, start_ticks(id.pid)) { + (Some(recorded), Some(live)) if recorded != live => IdentityState::Recycled, + _ => IdentityState::Indeterminate, + }, ProcessProbe::Running if has_exited(id.pid) => IdentityState::Gone, ProcessProbe::Running => classify_live_identity(id.start_ticks, start_ticks(id.pid)), } @@ -176,6 +189,15 @@ pub enum TerminationOutcome { AlreadyGone, /// The PID now belongs to a different process (recycled), so the recorded /// process has already exited; nothing was signalled. + /// + /// Counted as stopped, on every platform, because a token that is recorded + /// and readable and differs is positive evidence of a different process. The + /// engine's own token comes from the same [`process_start_ticks`] that reads + /// it back here, taken for that PID's process when it was spawned or reported + /// (a creation time does not change over a process's life), so a mismatch for + /// the real engine is not an expected outcome. It stays visible as + /// `identity_mismatch` in the report rather than as a signal sent to a + /// process nobody could prove was the server's. IdentityMismatch, /// The recorded PID is live but its identity could not be confirmed, so it /// was deliberately left untouched and its state is unknown. @@ -529,6 +551,10 @@ pub fn terminate_recorded_pid(identity: &ProcessIdentity) -> TerminationOutcome enum TreeKill { /// The kill ran and reported that every process it reached is terminated. Completed, + /// The kill ran and reported that it found no such process (`taskkill` exits + /// 128): the root was gone by the time it looked. Not by itself a failure, + /// and not a success either; the root's own verdict decides which. + NotFound, /// The kill could not run, or reported that it did not take every process /// down. Which descendants survive is not known. Failed, @@ -539,11 +565,22 @@ enum TreeKill { /// would confirm the stop into [`TerminationOutcome::TimedOut`]: the root was /// signalled, and it, or a process under it, is not confirmed exited. The /// verdicts that already deny the stop stay as they are. +/// +/// A kill that found no such process ([`TreeKill::NotFound`]) is judged by the +/// root's own re-read, which `outcome` carries: when the root is gone +/// ([`TerminationOutcome::AlreadyGone`]) it exited on its own between the +/// identity check and the kill, and that is what the stop reports, as for any +/// recorded process that is already gone. When the root is still there, "no such +/// process" contradicts what the termination found, so the kill is treated as +/// failed. const fn after_tree_kill(tree: TreeKill, outcome: TerminationOutcome) -> TerminationOutcome { - match tree { - TreeKill::Completed => outcome, - TreeKill::Failed if outcome.stopped() => TerminationOutcome::TimedOut, - TreeKill::Failed => outcome, + match (tree, outcome) { + (TreeKill::Completed, outcome) => outcome, + (TreeKill::NotFound, TerminationOutcome::AlreadyGone) => TerminationOutcome::AlreadyGone, + (TreeKill::NotFound | TreeKill::Failed, outcome) if outcome.stopped() => { + TerminationOutcome::TimedOut + } + (TreeKill::NotFound | TreeKill::Failed, outcome) => outcome, } } @@ -568,12 +605,15 @@ fn terminate_recorded_pid_with( // Taken while the root is still alive: `taskkill /T` resolves children // through the live parent, so it cannot reach them once the root is gone. let tree = tree_kill(identity.pid); - // `terminate` still owns the bounded wait and the verdict, but a root - // the tree kill already took now reads back as `AlreadyGone` — which - // would deny a stop this command did perform. - let outcome = match terminate(identity) { - TerminationOutcome::AlreadyGone => TerminationOutcome::Forced, - outcome => outcome, + // `terminate` still owns the bounded wait and the verdict. A root the + // tree kill already took now reads back as `AlreadyGone`, which would + // deny a stop this command did perform, so it counts as forced; but when + // the kill found no such process the root went by itself, and + // `AlreadyGone` is the truth. + let outcome = match (tree, terminate(identity)) { + (TreeKill::NotFound, outcome) => outcome, + (_, TerminationOutcome::AlreadyGone) => TerminationOutcome::Forced, + (_, outcome) => outcome, }; return after_tree_kill(tree, outcome); } @@ -633,17 +673,23 @@ fn platform_tree_kill() -> Option<&'static dyn Fn(u32) -> TreeKill> { } } +/// `taskkill`'s exit code for "the process was not found". +#[cfg(any(windows, test))] +const TASKKILL_PROCESS_NOT_FOUND: i32 = 128; + /// The verdict on one `taskkill` run, from whether it started and the exit code -/// it ended with. Only a clean exit (0) is a completed tree kill: a failed -/// launch, a run that did not finish within its deadline -/// ([`std::io::ErrorKind::TimedOut`], see [`run_to_exit_code`]), a nonzero code -/// (a process it could not terminate, or none found) and a run that ended -/// without a code are all [`TreeKill::Failed`]. Split out of the process spawn -/// so a test runs it on every host. +/// it ended with. A clean exit (0) is a completed tree kill and 128 is +/// [`TreeKill::NotFound`], left to the root's re-read to judge. A failed launch, +/// a run that did not finish within its deadline +/// ([`std::io::ErrorKind::TimedOut`], see [`run_to_exit_code`]), any other +/// nonzero code (a process it could not terminate) and a run that ended without +/// a code are all [`TreeKill::Failed`]. Split out of the process spawn so a test +/// runs it on every host. #[cfg(any(windows, test))] fn classify_taskkill(run: std::io::Result>) -> TreeKill { match run { Ok(Some(0)) => TreeKill::Completed, + Ok(Some(TASKKILL_PROCESS_NOT_FOUND)) => TreeKill::NotFound, Ok(_) | Err(_) => TreeKill::Failed, } } @@ -1635,6 +1681,89 @@ mod tests { } } + /// `taskkill` answering "no such process" is judged by the root's own + /// re-read. The root that went away by itself between the identity check and + /// the kill is a stop that happened: nothing is left to wait for, and + /// reporting it unconfirmed would keep a record, a stop request and a key for + /// a service that is gone. + #[test] + fn a_tree_kill_that_found_no_process_is_a_stop_when_the_root_is_gone() { + let (verdict, steps) = tree_kill_run_with( + true, + TreeKill::NotFound, + IdentityState::Matches, + TerminationOutcome::AlreadyGone, + ); + + assert_eq!( + verdict, + TerminationOutcome::AlreadyGone, + "the root exited on its own: the truth, not a kill this stop performed" + ); + assert!(verdict.stopped()); + assert!( + steps.contains(&Step::Terminate(SUPERVISOR_PID)), + "the root is re-read by the termination: {steps:?}" + ); + } + + /// "No such process" while the root is still there is a contradiction, so it + /// is a failed kill and the stop stays unconfirmed, as for any other nonzero + /// exit. + #[test] + fn a_tree_kill_that_found_no_process_while_the_root_runs_is_a_failed_kill() { + use TerminationOutcome::{Forced, Graceful, TimedOut, Unverified}; + for (root, expected) in [ + (Forced, TimedOut), + (Graceful, TimedOut), + (TimedOut, TimedOut), + (Unverified, Unverified), + ] { + let (verdict, _) = + tree_kill_run_with(true, TreeKill::NotFound, IdentityState::Matches, root); + + assert_eq!(verdict, expected, "root {root:?}"); + assert!(!verdict.stopped(), "root {root:?}"); + } + } + + /// Any other nonzero exit with the root gone still leaves the stop + /// unconfirmed: unlike "not found", it says a process could not be + /// terminated, so a descendant may survive the root. + #[test] + fn a_failed_tree_kill_with_the_root_gone_is_still_unconfirmed() { + let (verdict, _) = tree_kill_run_with( + true, + TreeKill::Failed, + IdentityState::Matches, + TerminationOutcome::AlreadyGone, + ); + + assert_eq!(verdict, TerminationOutcome::TimedOut); + } + + /// A recorded token that can be read and differs proves another process + /// holds the PID, even when the rest of the query failed; without a readable + /// token the PID stays unverifiable. + #[test] + fn a_readable_token_mismatch_outranks_a_failed_query() { + use crate::process::ProcessProbe::Unqueryable; + let state = |recorded, live| { + identity_state_with( + &ProcessIdentity::new(SUPERVISOR_PID, recorded), + |_| Unqueryable, + |_| false, + |_| live, + ) + }; + + assert_eq!(state(Some(7), Some(8)), IdentityState::Recycled); + assert_eq!(state(Some(7), Some(7)), IdentityState::Indeterminate); + assert_eq!(state(Some(7), None), IdentityState::Indeterminate); + assert_eq!(state(None, Some(8)), IdentityState::Indeterminate); + assert_eq!(state(None, None), IdentityState::Indeterminate); + } + #[test] fn a_completed_tree_kill_leaves_the_roots_verdict_alone() { use TerminationOutcome::{Forced, Graceful, TimedOut, Unverified}; @@ -1741,12 +1870,14 @@ mod tests { } #[test] - fn only_a_clean_taskkill_exit_is_a_completed_tree_kill() { + fn only_a_clean_taskkill_exit_is_a_completed_tree_kill_and_128_is_not_found() { use std::io::{Error, ErrorKind}; assert_eq!(classify_taskkill(Ok(Some(0))), TreeKill::Completed); - // A process it could not terminate, or none found. + // A process it could not terminate. assert_eq!(classify_taskkill(Ok(Some(1))), TreeKill::Failed); - assert_eq!(classify_taskkill(Ok(Some(128))), TreeKill::Failed); + assert_eq!(classify_taskkill(Ok(Some(5))), TreeKill::Failed); + // No such process: not a failure by itself, the root's re-read decides. + assert_eq!(classify_taskkill(Ok(Some(128))), TreeKill::NotFound); // Ended without an exit code. assert_eq!(classify_taskkill(Ok(None)), TreeKill::Failed); // Could not be started, or did not finish before its deadline. @@ -1773,15 +1904,20 @@ mod tests { |_| live, ) }; - // Whatever token was recorded, and whatever can be read of the process. - for recorded in [None, Some(7)] { - for live in [None, Some(7), Some(8)] { - assert_eq!( - state(Unqueryable, false, recorded, live), - IdentityState::Indeterminate, - "recorded {recorded:?}, live {live:?}" - ); - } + // Whatever token was recorded, and whatever can be read of the process, + // short of a readable token that differs (`a_readable_token_mismatch_...`). + for (recorded, live) in [ + (None, None), + (None, Some(7)), + (None, Some(8)), + (Some(7), None), + (Some(7), Some(7)), + ] { + assert_eq!( + state(Unqueryable, false, recorded, live), + IdentityState::Indeterminate, + "recorded {recorded:?}, live {live:?}" + ); } assert_eq!(state(Gone, false, Some(7), Some(7)), IdentityState::Gone); assert_eq!(state(Running, true, Some(7), Some(7)), IdentityState::Gone); diff --git a/crates/rocm-core/src/process.rs b/crates/rocm-core/src/process.rs index 611e39d2e2..69cd2d16d9 100644 --- a/crates/rocm-core/src/process.rs +++ b/crates/rocm-core/src/process.rs @@ -378,6 +378,17 @@ pub enum ProcessProbe { /// The PID could not be queried, so it can be neither confirmed running nor /// confirmed gone. On Windows that includes a live process the caller may not /// open (`ERROR_ACCESS_DENIED`). + /// + /// The cost of reading this as "may be running" is that a stopped service's + /// recorded PID, taken since by a process the caller cannot open, keeps its + /// record live: the record never reconciles to `stopped`, `services remove` + /// and `prune` refuse it, and a stop reports it unverified. That is the safe + /// direction (the alternative deletes the key and record under a process that + /// may be serving), and there is no better evidence to be had: the creation + /// time that would tell the two processes apart is read with the same access + /// right, so a process that cannot be opened has no readable token to + /// compare. The record can be cleared by hand; see `docs/manual-testing.md`, + /// "A record that stays unverified". Unqueryable, } diff --git a/docs/architecture.md b/docs/architecture.md index 93f2111478..e2a69270cf 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -231,7 +231,7 @@ Further `rocm-core` modules, each with its own doc comment except where noted: - [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) — OpenMPI, libatomic and libnuma detection, the `libmpi_cxx` compatibility shim, and the per-library install plans (`build_openmpi_install_plan`, `build_libatomic_install_plan`, `build_libnuma_install_plan`) and hints, needed by vLLM's ROCm wheels. The distro-generic half of that planning lives in [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs); [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) re-exports its public items, so `rocm_core::openmpi::running_as_root`, `plan_for_os_release_read` and the rest keep resolving, and `pub(crate)` re-exports `PackageManager` and `resolve_package_manager` for [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs). - [`os_release.rs`](../crates/rocm-core/src/os_release.rs) — the one `/etc/os-release` parser every reader of that file uses. - [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs) — the planning half of a system-package install that does not depend on which library is installed: `PackageManager` and `resolve_package_manager` (from an os-release `ID`/`ID_LIKE`), `SystemPackageInstallPlan` and `InstallCommand`, `plan_for_os_release_at`/`plan_for_os_release_read` (which take the per-library builder as an argument), `InstallHint`, the root and passwordless-`sudo` checks (`running_as_root`, `can_autoinstall`) and the `apt-get -s` helpers (`apt_simulate_argv`, `parse_apt_simulate_removals`, `is_protected_rocm_package`). It never runs anything and depends on no other module of this crate except [`os_release.rs`](../crates/rocm-core/src/os_release.rs) and [`hardware_root.rs`](../crates/rocm-core/src/hardware_root.rs), which it reads os-release through. The module is private; its public items are re-exported from [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs). It is a different module from [`system_packages.rs`](../apps/rocm/src/system_packages.rs) in `apps/rocm`, which runs these plans with consent and `sudo`. -- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches (`terminate_verified`), and the one managed-stop routine built on it that `rocm services stop` and `rocmd`'s `stop_server` both call: `terminate_recorded_service_pids` walks the processes a service record names (supervisor, then engine, each with its own start-time, the stopping process's own PID reported as `self` and never signalled) and returns one `RecordedPidStop` per PID (each caller, on a stop that confirms every one gone, then clears the record's recorded processes with `ManagedServiceRecord::clear_recorded_processes` in [`service_record.rs`](../crates/rocm-core/src/service_record.rs), so a repeated stop reaches no stale PID; an unconfirmed stop keeps them); `terminate_recorded_pid` terminates one of them, adding on Windows a `taskkill /T /F` of its descendants first, because `terminate_verified` cannot walk a process tree there, and only for a PID that passes the same identity check (`process_start_ticks` is the kernel start-time on Linux and the process creation time on Windows, so a recycled PID is refused before any `taskkill`; a record written before Windows recorded one cannot be refuted), holding the PID open on Windows from before that check until the stop ends so the PID cannot be reused in between, and treating a tree kill that did not complete, or did not finish within `MANAGED_STOP_GRACE` (the `taskkill` is killed and reaped at that deadline), as an unconfirmed stop; a PID that cannot be queried at all, as against one that is gone, reads as unverifiable (`ProcessProbe` and `probe_process` in [`process.rs`](../crates/rocm-core/src/process.rs), whose `may_be_running` is the answer for any decision that must not take an unqueryable process for gone); `MANAGED_STOP_GRACE` is the one grace period. The termination is injectable, so a caller's test supplies outcomes it cannot construct and a test of the Windows step runs on every host. +- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches (`terminate_verified`), and the one managed-stop routine built on it that `rocm services stop` and `rocmd`'s `stop_server` both call: `terminate_recorded_service_pids` walks the processes a service record names (supervisor, then engine, each with its own start-time, the stopping process's own PID reported as `self` and never signalled) and returns one `RecordedPidStop` per PID (each caller, on a stop that confirms every one gone, then clears the record's recorded processes with `ManagedServiceRecord::clear_recorded_processes` in [`service_record.rs`](../crates/rocm-core/src/service_record.rs), so a repeated stop reaches no stale PID; an unconfirmed stop keeps them); `terminate_recorded_pid` terminates one of them, adding on Windows a `taskkill /T /F` of its descendants first, because `terminate_verified` cannot walk a process tree there, and only for a PID that passes the same identity check (`process_start_ticks` is the kernel start-time on Linux and the process creation time on Windows, so a recycled PID is refused before any `taskkill`; a record written before Windows recorded one cannot be refuted), holding the PID open on Windows from before that check until the stop ends so the PID cannot be reused in between, and treating a tree kill that did not complete, or did not finish within `MANAGED_STOP_GRACE` (the `taskkill` is killed and reaped at that deadline), as an unconfirmed stop, except that a `taskkill` which found no such process is judged by the root's own re-read, so a root that exited by itself is a stop; a PID that cannot be queried at all, as against one that is gone, reads as unverifiable unless a readable recorded token differs, in which case that mismatch wins (`ProcessProbe` and `probe_process` in [`process.rs`](../crates/rocm-core/src/process.rs), whose `may_be_running` is the answer for any decision that must not take an unqueryable process for gone); `MANAGED_STOP_GRACE` is the one grace period. The termination is injectable, so a caller's test supplies outcomes it cannot construct and a test of the Windows step runs on every host. - [`report.rs`](../crates/rocm-core/src/report.rs) — the content of a Doctor report and the rule deciding whether one may exist; it sends nothing. - [`report_delivery.rs`](../crates/rocm-core/src/report_delivery.rs) — how a report leaves the machine: building its link and deciding whether a browser may be started. - [`runtime.rs`](../crates/rocm-core/src/runtime.rs) — the host-platform layer: `RuntimePlatform`/`RuntimeHost`, OS-specific executable and Python-environment names (including `runtime_python_env_root_for_executable`, the one rule for finding an environment's folder from its interpreter's path: only an interpreter directly inside a `bin` or `Scripts` folder, in any case, has one, and everything else gets `None`; the adopt command, the ComfyUI runtime environment and the vLLM launch all call it rather than keeping a copy, since the engine crate cannot reach `apps/rocm`), default config/data/cache directories, path-text and path-list helpers, and `install_root_has_payload`, the one definition of a non-empty install root (dot-entries such as the runtime marker do not count; a path that does not exist holds nothing; a root that cannot be listed is an error) that tarball extraction, update repair detection and activation validation in `apps/rocm` all ask (no module doc comment). diff --git a/docs/manual-testing.md b/docs/manual-testing.md index c8528d7118..96a68cb3c6 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -507,6 +507,39 @@ line. `restart` starts the server again only once its stop has confirmed every recorded process gone; otherwise it reports that and starts nothing. +On Windows the stop takes the engine's process tree down with the recorded +process (`taskkill /T /F`, which has to finish within the stop's grace period of +10 seconds, or the stop is not confirmed), and it checks the recorded process +against the creation time written when the server started, so a process that has +since taken the same PID is left alone and reported as `identity_mismatch`. A +process it cannot verify, on either platform, is not signalled and is named with +the check the host has: `ps -p -o pid,args` on Linux and WSL, and +`tasklist /FI "PID eq "` on Windows. There are two ways to be unverifiable: +the recorded start-time can no longer be read back from a running process, or +(Windows only) the PID cannot be queried at all, as for a live process the +account may not open. The text of `rocm services stop` does not list a per-process +outcome for the stopping process itself; the JSON answer of the sandbox +`stop_server` tool does, in `pid_outcomes`, as `self` (never signalled, counted +as stopped). + +#### A record that stays unverified + +If the server is gone but its recorded PID has been taken by a process this +account cannot open, `rocm` cannot tell the two apart, and it treats that as +possibly running, which is the safe direction: the record never reconciles to +`stopped`, `rocm services remove` and `rocm services prune` refuse it, and +`rocm services stop` keeps reporting `unverified`. To clear it: + +1. Run the check command the stop printed. If the process is this server's, end it + yourself (from an elevated prompt if the account may not), then run + `rocm services`; the record reconciles and its endpoint key is dropped. +2. If it is an unrelated process, `rocm` has no command that clears the record. + Delete the record's `.json` by hand from the folder that + `rocm storage report` lists under `local server records`, then run + `rocm services prune --any-age --yes`, which sweeps the log, the endpoint key + and the engine state file that are left with no record. The server can no + longer be restarted afterwards. + The rest of this section needs a **vLLM** server running, on Linux or WSL, and two runtimes to move it between. Nothing earlier in this document sets either up: section 3 serves Lemonade and then stops it on both of its branches, the diff --git a/docs/testing.md b/docs/testing.md index 54ca7ac027..666140a3df 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1788,7 +1788,9 @@ and its neighbours) and, against a real tree, by a Windows-only scenario that each reaches the routine and reports what it returns (`cargo test -p rocm --bin rocm service_control::tests::a_stop_` and `cargo test -p rocmd service::tests::a_stop_`), and each carries a source check -that no private copy of the routine or its constant has come back. A confirmed +that no private copy of the routine or its constant has come back. Those checks +pin names only: they are a tripwire for a copy returning under the same name, not +coverage of what either stop does. A confirmed stop also clears the PIDs and start-time tokens it stopped from the record (`ManagedServiceRecord::clear_recorded_processes`, one call from both binaries), and an unconfirmed one keeps them; a PID recorded without a token is @@ -1852,7 +1854,14 @@ time that was compared is that of the process the kill then reaches reported as `timed_out`, whatever the root's own termination says, because only the root's exit is watched on Windows (`a_tree_kill_that_did_not_complete_leaves_the_stop_unconfirmed`, -`only_a_clean_taskkill_exit_is_a_completed_tree_kill`). That `taskkill` runs under +`only_a_clean_taskkill_exit_is_a_completed_tree_kill_and_128_is_not_found`). A +`taskkill` that exits 128 (process not found) is judged by the root's own re-read: +with the root gone it exited by itself and the stop is reported as for any +recorded process already gone, so the record is not kept for nothing; with the +root still there the kill counts as failed +(`a_tree_kill_that_found_no_process_is_a_stop_when_the_root_is_gone`, +`a_tree_kill_that_found_no_process_while_the_root_runs_is_a_failed_kill`, +`a_failed_tree_kill_with_the_root_gone_is_still_unconfirmed`). That `taskkill` runs under a deadline of `MANAGED_STOP_GRACE`: one that stalls is killed and reaped and counts as a failed tree kill, so it cannot hang `rocm services stop`, `restart` or the attached Ctrl-C stop @@ -1861,7 +1870,13 @@ child under a 200 ms deadline on every host and asserts the error, the bounded elapsed time, the reaped child and the failed verdict; `a_helper_that_exits_in_time_gives_its_exit_code` and `a_helper_that_cannot_be_started_is_an_error` are its neighbours). The report and the record a `timed_out` stop leaves are covered by -`an_unconfirmed_stop_keeps_the_recorded_processes_for_the_next_stop`. What no +`an_unconfirmed_stop_keeps_the_recorded_processes_for_the_next_stop`. A readable +recorded token that differs outranks a failed query +(`a_readable_token_mismatch_outranks_a_failed_query`), but the creation time and +the query are read with the same access right, so on Windows this only reaches a +process that opens and then fails the exit-code query; a process that cannot be +opened has no readable token, stays unverifiable, and is cleared by hand (see +"A record that stays unverified" in [manual-testing.md](manual-testing.md)). What no test here reaches is the Windows API itself: the handle hold, the open-failure codes and a real `taskkill` failing run only on the Windows lane. @@ -1876,6 +1891,17 @@ drives the refresh with an unqueryable PID and then a confirmed-gone one, an unverified stop and then the refresh over the record it wrote, and `a_record_is_in_use_unless_every_recorded_process_is_confirmed_gone` covers the removal guard; each takes the probe as an argument, so they run on every host. +Two gaps in the scenario coverage are deliberate. `service-stop-07` records no +token, so the path a record written by this version takes on Windows (a token +recorded, matching, with the tree kill then running) has no scenario; it rests on +`what_a_stop_does_for_each_recorded_and_live_token` and the other injected tests, +and on the Windows unit tests that stop a real process with its true token. And +the `self` row that `rocm` now puts in `pid_outcomes` has no scenario: it appears +only in the JSON answer of the hidden `sandbox-run stop_server` and the sandbox +tool, never in the text of `rocm services stop`, and it names the stopping +process's own PID, which a scenario cannot plant in a record before the process +that will read it exists. `a_stop_reports_its_own_pid_as_self` and +`a_self_row_adds_no_line_to_an_unconfirmed_report` cover the answer and the report. `service-stop-06` (Linux) is the same stop of a real recorded process through `rocm services stop`, checking the persisted state: the process exited and the record names none. Run the Windows scenario on a Windows host with From 6747590d7e818d69849f1d267603eb8f1e9fa665 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 15:50:08 +0000 Subject: [PATCH 13/30] fix(services): verify a Windows launch child's token in the failed-launch cleanup, and correct the comments that said Windows records none abandon_failed_managed_launch said Windows records no start token and guarded its refusal with cfg!(target_os = "linux"). Windows now records the process creation time, so a child whose token was not captured (it was gone before it could be) is just as unverifiable there: its pid may belong to another process, and the forced tree kill must not run. The decision moves into launch_cleanup_refusal, a pure function taking the captured token and whether the host records tokens (Linux and Windows), tested on every host together with a check that host_records_start_tokens agrees with what process_start_ticks reads on the host the test runs on. The same claim, that Windows has no start-time token, is corrected where it was repeated: the doc comments of three Linux-gated tests (rocm-core service_record, rocmd supervise, the vLLM state test) and the stop scenario's planting helper, which now says it plants the tokenless shape on purpose. Signed-off-by: Jussi Elo --- apps/rocm/src/service_launch.rs | 70 +++++++++++++++++-- apps/rocmd/src/supervise.rs | 5 +- crates/rocm-core/src/service_record.rs | 3 +- engines/vllm/src/state.rs | 7 +- .../tests/e2e/service_stop_steps.rs | 6 +- 5 files changed, 77 insertions(+), 14 deletions(-) diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 0ae29477d8..5b069a6034 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -427,6 +427,30 @@ fn resolve_managed_port_in_transaction( /// be published. Short: the child is milliseconds old and has nothing to flush. const MANAGED_LAUNCH_CLEANUP_GRACE: Duration = Duration::from_secs(2); +/// Whether this host records a start token for a process: Linux (the kernel +/// start-time) and Windows (the creation time). Anywhere else +/// `rocm_core::process_start_ticks` has nothing to read. +const fn host_records_start_tokens() -> bool { + cfg!(any(target_os = "linux", windows)) +} + +/// The verdict that ends a failed launch's cleanup before anything is signalled: +/// `Some(Unverified)` for a child whose token was not captured on a host that +/// records tokens, because that child was gone before it could be captured and +/// its pid may now belong to another process. `None` otherwise: a captured token +/// is verified by the termination, and on a host with no tokens there is nothing +/// to verify, so the kill proceeds as it always did there. +const fn launch_cleanup_refusal( + engine_start_ticks: Option, + host_records_tokens: bool, +) -> Option { + if engine_start_ticks.is_none() && host_records_tokens { + Some(rocm_core::TerminationOutcome::Unverified) + } else { + None + } +} + /// Undo a failed managed launch attempt before the launch lock is released, /// so no failed transaction leaves a live reservation or a stray endpoint key. /// @@ -447,12 +471,15 @@ fn abandon_failed_managed_launch(paths: &AppPaths, record: &ManagedServiceRecord // The token captured at spawn, not a fresh `capture`: when the child // already exited, a fresh one would describe a recycled pid's new owner // and the forced tree kill would hit it. Where start tokens exist - // (Linux), a missing one means the child was gone before it could be - // captured, so the pid is unverifiable and nothing is signalled - // (`terminate_verified` alone would "best-effort proceed"). Windows - // records no token at all, so there the kill proceeds as before. - let outcome = if record.engine_start_ticks.is_none() && cfg!(target_os = "linux") { - rocm_core::TerminationOutcome::Unverified + // (Linux, and Windows, which records the process creation time), a + // missing one means the child was gone before it could be captured, so + // the pid is unverifiable and nothing is signalled (`terminate_verified` + // alone would "best-effort proceed"). A token that was captured is + // verified by `terminate_verified` on both. + let outcome = if let Some(refusal) = + launch_cleanup_refusal(record.engine_start_ticks, host_records_start_tokens()) + { + refusal } else { rocm_core::terminate_verified( &rocm_core::ProcessIdentity::new(pid, record.engine_start_ticks), @@ -4937,4 +4964,35 @@ mod tests { result?; Ok(()) } + + /// A failed launch's cleanup refuses to signal a child whose token was not + /// captured on any host that records tokens. Windows is one of them, since + /// it records the creation time: a missing token there means the child was + /// gone before it could be captured, and its pid may now be another + /// process's, so the kill must not proceed as it did when Windows had none. + #[test] + fn launch_cleanup_refuses_a_child_without_a_token_only_where_tokens_exist() { + use rocm_core::TerminationOutcome::Unverified; + + // A host that records tokens: a missing one is unverifiable, a captured + // one goes on to be verified by the termination. + assert_eq!(launch_cleanup_refusal(None, true), Some(Unverified)); + assert_eq!(launch_cleanup_refusal(Some(7), true), None); + // A host with no tokens has nothing to verify and proceeds as before. + assert_eq!(launch_cleanup_refusal(None, false), None); + assert_eq!(launch_cleanup_refusal(Some(7), false), None); + } + + /// Ties the cleanup's idea of "this host records tokens" to what + /// `process_start_ticks` really does, on the host the test runs on: Linux and + /// Windows read a token for a live process (this one), anywhere else there + /// is none. If a platform gains or loses tokens without the cleanup being + /// told, this fails there. + #[test] + fn the_cleanup_guard_agrees_with_whether_this_host_reads_tokens() { + assert_eq!( + host_records_start_tokens(), + rocm_core::process_start_ticks(std::process::id()).is_some() + ); + } } diff --git a/apps/rocmd/src/supervise.rs b/apps/rocmd/src/supervise.rs index eb594e2167..a0ec61ce34 100644 --- a/apps/rocmd/src/supervise.rs +++ b/apps/rocmd/src/supervise.rs @@ -1224,8 +1224,9 @@ mod tests { /// calls `std::process::exit`, which inside a test binary would take the /// whole suite with it. /// - /// Linux-only: `rocm_core::process_start_ticks` reads `/proc` and returns - /// `None` everywhere else, so on other platforms there is no token to write. + /// Linux-only here: `rocm_core::process_start_ticks` gives a token on Linux + /// and on Windows (the creation time) and `None` elsewhere, and this test has + /// not been taken to Windows. #[cfg(target_os = "linux")] #[test] fn supervise_service_persists_the_supervisor_identity_token() -> Result<()> { diff --git a/crates/rocm-core/src/service_record.rs b/crates/rocm-core/src/service_record.rs index bf405c3728..fc95d662ce 100644 --- a/crates/rocm-core/src/service_record.rs +++ b/crates/rocm-core/src/service_record.rs @@ -740,7 +740,8 @@ mod tests { /// been recycled. One live process stands in for both roles, as it does for /// a `rocm serve --background` launch. /// - /// Linux-only: `process_start_ticks` reads `/proc`, and is `None` elsewhere. + /// Linux-only here, because it spawns `sleep`: `process_start_ticks` gives a + /// token on Linux and on Windows (the creation time) and is `None` elsewhere. #[cfg(target_os = "linux")] #[test] fn recording_a_process_identity_writes_the_pid_with_its_own_token() { diff --git a/engines/vllm/src/state.rs b/engines/vllm/src/state.rs index 2691dc62fa..bf2fce5385 100644 --- a/engines/vllm/src/state.rs +++ b/engines/vllm/src/state.rs @@ -484,9 +484,10 @@ mod tests { // Uses the test's own live PID rather than a fixed placeholder so that, // on Linux, the `start_ticks` assertion below exercises the real // `rocm_core::process_start_ticks` lookup rather than only checking - // the field is present. On other platforms `process_start_ticks` is a - // stub that always returns `None`, so there the assertion only checks - // `None == None`. + // the field is present. On Windows it is the process creation time and + // the assertion still compares like with like; on any other platform + // `process_start_ticks` is a stub that always returns `None`, so there + // the assertion only checks `None == None`. let own_pid = std::process::id(); let state_path = probe_state_path("therock-env"); let request = ServeHttpRequest { diff --git a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs index eea184ed2b..d1a2530ddb 100644 --- a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs @@ -221,8 +221,10 @@ fn descendant_pid_path(world: &E2eWorld) -> PathBuf { } /// A ready, publicly bound server whose recorded supervisor is a parent process -/// that has started a child of its own, recorded the way Windows records it: -/// the PID alone, with no start-time token (there is none to read there). +/// that has started a child of its own, recorded the way a Windows record was +/// before Windows recorded a creation time: the PID alone, with no start-time +/// token. (A record written now carries one; this scenario plants the tokenless +/// shape on purpose, the one a PID cannot be refuted by.) /// /// The parent is PowerShell, the child `ping` for five minutes: both ship with /// Windows, need nothing installed, and are disposable. PowerShell starts the From 441ecea2361cf043d9af33b1cf0f3153514cb357 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 15:51:16 +0000 Subject: [PATCH 14/30] fix(core): report a root that exited under a failed tree kill as descendants unconfirmed, resolve the Windows tools under System32, and share the may-be-running reading When the separate Windows tree kill did not complete but this stop's own wait saw the root exit, the stop was reported timed_out, whose text says a process outlived a deadline. That is not what happened: the root exited and nothing watched the processes under it. It now has its own outcome, descendants_unconfirmed (not a stop, signalled, forced), printed as "not confirmed: process N (role) exited, but a process under it may still be running"; timed_out stays for a root that did outlive its deadline. A test asserts the printed line, that no "still running" line appears, and the record, stop request and key that are kept. taskkill, now the one shared copy, was started by bare name. It and ComfyUI's taskkill and tasklist calls are started from %SystemRoot%\System32 through windows_system_tool, which falls back to the bare name when SystemRoot is unset; the path building is tested on every host. The deferred key cleanup, the reconciliation to stopped and the in-use check behind services remove and prune each repeated "any recorded pid may be running". They share recorded_pids_may_be_running, with a test that both roles are asked about, that only a confirmed exit is gone and that a 0 placeholder is never probed. docs/testing.md states a third deliberate scenario gap: the check line of an unverified stop is a diagnostic, and what clears the condition is the process ending. Signed-off-by: Jussi Elo --- README.md | 4 +- apps/rocm/src/cli.rs | 4 +- apps/rocm/src/comfyui/process.rs | 4 +- apps/rocm/src/managed_service_status.rs | 71 ++++++++++++++++-- apps/rocm/src/service_control.rs | 46 ++++++++++++ apps/rocm/src/service_records.rs | 7 +- apps/rocm/src/services_text.rs | 12 ++++ crates/rocm-core/src/lib.rs | 2 +- crates/rocm-core/src/proc_lifecycle.rs | 96 +++++++++++++++++++------ crates/rocm-core/src/process.rs | 45 ++++++++++++ docs/architecture.md | 2 +- docs/manual-testing.md | 5 +- docs/testing.md | 30 ++++++-- 13 files changed, 284 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index e5fdcfba46..d0769b4190 100644 --- a/README.md +++ b/README.md @@ -1236,7 +1236,9 @@ gone. If it cannot confirm that, it prints `Local server stop not confirmed` on stderr and exits with an error. It keeps the record, which still names the processes, and any endpoint key, which the server may still be enforcing. It names each process that, or a process under it, did not exit after it was -signaled; for a process it could not verify as the server's (its recorded +signaled (on Windows also a process that did exit while taking the processes +under it down did not complete, so one of them may still be running); for a +process it could not verify as the server's (its recorded start-time can no longer be read back, or, on Windows, `rocm` cannot query it at all), it signals nothing and names an operating-system command to check on it. It does not suggest running the stop again: when only a process under the recorded one survived, a diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index 3f59708942..3cd64b8352 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -1123,7 +1123,9 @@ pub(crate) enum ServicesCommand { /// confirmed gone. Otherwise it reports on stderr that the stop was not /// confirmed and exits with an error, keeping the record, which still names /// the processes, and any endpoint key, which the server may still be - /// enforcing. It names each process that did not exit, and for one it could + /// enforcing. It names each process that did not exit (on Windows also one + /// that exited while taking the processes under it down did not complete), + /// and for one it could /// not verify (its recorded start-time can no longer be read back, or, on /// Windows, it cannot be queried at all) it signals nothing and names an /// operating-system command to check on it. It does not suggest running the diff --git a/apps/rocm/src/comfyui/process.rs b/apps/rocm/src/comfyui/process.rs index ec324098d5..5b17ef58c8 100644 --- a/apps/rocm/src/comfyui/process.rs +++ b/apps/rocm/src/comfyui/process.rs @@ -419,7 +419,7 @@ fn process_is_running(pid: u32) -> bool { fn windows_process_is_running(pid: u32) -> bool { let filter = format!("PID eq {pid}"); - let mut command = Command::new("tasklist"); + let mut command = Command::new(rocm_core::windows_system_tool("tasklist.exe")); command.args(["/FI", &filter, "/FO", "CSV", "/NH"]); let Ok(output) = capture_configured_command(command, "check ComfyUI process") else { return false; @@ -468,7 +468,7 @@ fn terminate_process_tree(pid: u32) -> Result<()> { } fn terminate_windows_process_tree(pid: u32) -> Result<()> { - let status = Command::new("taskkill") + let status = Command::new(rocm_core::windows_system_tool("taskkill.exe")) .args(["/PID", &pid.to_string(), "/T", "/F"]) .stdin(Stdio::null()) .stdout(Stdio::null()) diff --git a/apps/rocm/src/managed_service_status.rs b/apps/rocm/src/managed_service_status.rs index 93bf1e5f3d..195f8bc79c 100644 --- a/apps/rocm/src/managed_service_status.rs +++ b/apps/rocm/src/managed_service_status.rs @@ -190,6 +190,23 @@ pub(crate) fn recorded_service_pids(record: &ManagedServiceRecord) -> Vec { .collect() } +/// Whether any process the record names may still be running: everything but a +/// confirmed exit counts (see [`ProcessProbe::may_be_running`]), and a `0` +/// placeholder PID is never asked about. +/// +/// The one reading behind every destructive decision made from a recorded +/// process's liveness: the deferred key cleanup, the reconciliation to +/// `stopped`, and the in-use check behind `rocm services remove` and `prune`. +/// Kept in one place so those cannot drift apart on what counts as gone. +pub(crate) fn recorded_pids_may_be_running( + record: &ManagedServiceRecord, + probe: impl Fn(u32) -> ProcessProbe, +) -> bool { + recorded_service_pids(record) + .iter() + .any(|pid| probe(*pid).may_be_running()) +} + /// Complete a stop that was requested but could not confirm termination, once /// the processes are actually observed gone: drop the endpoint key and clear the /// marker. Returns whether the record changed. @@ -219,10 +236,7 @@ fn settle_pending_stop_key_cleanup( if record.stop_requested_unix_ms.is_none() { return false; } - if recorded_service_pids(record) - .iter() - .any(|pid| probe(*pid).may_be_running()) - { + if recorded_pids_may_be_running(record, probe) { return false; } endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); @@ -293,7 +307,7 @@ pub(crate) fn refresh_managed_service_runtime_liveness_with( let tracked_pids = recorded_service_pids(record); let has_tracked_pid = !tracked_pids.is_empty(); - let has_live_pid = tracked_pids.iter().any(|pid| probe(*pid).may_be_running()); + let has_live_pid = recorded_pids_may_be_running(record, &probe); if has_tracked_pid && !has_live_pid { // Reconcile the status only. The endpoint key is *not* dropped here: // dead pids alone do not distinguish a stopped service from a crashed @@ -1034,4 +1048,51 @@ mod tests { let _ = fs::remove_dir_all(root); Ok(()) } + + /// The one reading behind the three destructive decisions: any recorded + /// process that is not confirmed gone counts, both roles are asked about, and + /// a placeholder `0` PID never is. + #[test] + fn recorded_pids_may_be_running_unless_every_recorded_process_is_confirmed_gone() { + use std::cell::RefCell; + + let (root, paths) = test_paths("recorded-pids-may-be-running"); + let mut record = pending_stop_record(&paths, "svc-pids").expect("record"); + let _ = fs::remove_dir_all(root); + record.supervisor_pid = 111; + record.engine_pid = Some(222); + + let asked_cell = RefCell::new(Vec::new()); + let asked = &asked_cell; + let by_pid = |running: &'static [(u32, ProcessProbe)]| { + move |pid: u32| { + asked.borrow_mut().push(pid); + running + .iter() + .find(|(candidate, _)| *candidate == pid) + .map_or(ProcessProbe::Gone, |(_, probe)| *probe) + } + }; + + assert!(recorded_pids_may_be_running( + &record, + by_pid(&[(222, ProcessProbe::Unqueryable)]) + )); + assert!(recorded_pids_may_be_running( + &record, + by_pid(&[(111, ProcessProbe::Running)]) + )); + assert!(!recorded_pids_may_be_running(&record, by_pid(&[]))); + let engine_only = asked.borrow().iter().filter(|pid| **pid == 222).count(); + assert!(engine_only >= 1, "the engine PID is asked about too"); + + record.supervisor_pid = 0; + record.engine_pid = None; + asked.borrow_mut().clear(); + assert!(!recorded_pids_may_be_running( + &record, + by_pid(&[(0, ProcessProbe::Running)]) + )); + assert!(asked.borrow().is_empty(), "a 0 placeholder is never probed"); + } } diff --git a/apps/rocm/src/service_control.rs b/apps/rocm/src/service_control.rs index 3a8e72be7a..67dac96c95 100644 --- a/apps/rocm/src/service_control.rs +++ b/apps/rocm/src/service_control.rs @@ -2819,4 +2819,50 @@ mod tests { "nothing about the record may change while the process is unqueryable" ); } + + /// A stop whose recorded process was seen to exit but whose tree kill did not + /// complete says so in those words, not as a timeout, and keeps what an + /// unconfirmed stop keeps: the claim and the state are checked together. + #[test] + fn a_stop_with_an_unconfirmed_tree_kill_says_what_happened_and_keeps_the_service() { + let (root, paths) = test_paths("services-stop-descendants-unconfirmed"); + paths.ensure().unwrap(); + let service_id = "svc-descendants-unconfirmed"; + stop_test_record(&paths, service_id).write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + vec![stop_of( + 999_999_999, + rocm_core::TerminationOutcome::DescendantsUnconfirmed, + )] + }) + .unwrap(); + let answer = internal_stop_server_value(report); + let outcome = finish_service_action(&paths, "stop_server", service_id, &answer); + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + + assert_eq!(answer["status"], "stop_unconfirmed", "{answer}"); + let rendered = format!("{:#}", outcome.expect_err("not a confirmed stop")); + assert!( + rendered.contains( + "not confirmed: process 999999999 (supervisor_and_engine) exited, but a process under it may still be running" + ), + "{rendered}" + ); + assert!( + !rendered.contains("still running: process") + && !rendered.contains("did not exit after it was signaled"), + "the root was seen to exit, so nothing outlived a deadline: {rendered}" + ); + assert!(!rendered.contains("Local server stopped"), "{rendered}"); + assert_no_retry_line(&rendered); + // The claim of what was kept, against what was kept. + assert!(rendered.contains("kept: the server record"), "{rendered}"); + assert_eq!(on_disk.supervisor_pid, 999_999_999); + assert!(on_disk.stop_requested_unix_ms.is_some()); + assert_eq!(key.as_deref(), Some("key-123")); + } } diff --git a/apps/rocm/src/service_records.rs b/apps/rocm/src/service_records.rs index 5832fab3d7..7bad1ff825 100644 --- a/apps/rocm/src/service_records.rs +++ b/apps/rocm/src/service_records.rs @@ -39,7 +39,7 @@ use anyhow::{Context, Result, bail}; use rocm_core::{AppPaths, ManagedServiceRecord, ProcessProbe}; use crate::managed_service_status::{ - load_managed_services, managed_service_is_live, recorded_service_pids, + load_managed_services, managed_service_is_live, recorded_pids_may_be_running, }; use crate::services_text::{load_managed_service, validate_service_id}; use crate::{cli_progress, cli_report, endpoint_keys, service_hints}; @@ -179,10 +179,7 @@ fn managed_service_record_is_in_use_with( record: &ManagedServiceRecord, probe: impl Fn(u32) -> ProcessProbe, ) -> bool { - managed_service_is_live(record) - || recorded_service_pids(record) - .iter() - .any(|pid| probe(*pid).may_be_running()) + managed_service_is_live(record) || recorded_pids_may_be_running(record, probe) } /// Remove one non-running local server record and everything it owns. diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index 43c9658719..21d8daf610 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -1018,6 +1018,10 @@ pub(crate) fn render_service_action_result(tool: &str, value: &serde_json::Value /// /// - a process that was signalled and that, or a process under it, outlived /// the deadline (`timed_out`); +/// - a process that is gone, with the separate kill of the processes under it +/// not completed (`descendants_unconfirmed`, Windows): this stop saw the +/// recorded process exit, so it does not say anything outlived a deadline, but +/// one of the processes under it may still be running; /// - a process that was not signalled because its identity could not be /// verified (`unverified`), with how to look at it. There are two routes to /// it: a recorded start-time that can no longer be read back from a running @@ -1058,6 +1062,8 @@ fn render_unconfirmed_service_stop(output: &serde_json::Value) -> String { .collect() }; let timed_out = with_outcome(rocm_core::TerminationOutcome::TimedOut.as_str()); + let descendants_unconfirmed = + with_outcome(rocm_core::TerminationOutcome::DescendantsUnconfirmed.as_str()); let unverified = with_outcome(rocm_core::TerminationOutcome::Unverified.as_str()); let mut text = String::new(); @@ -1081,6 +1087,12 @@ fn render_unconfirmed_service_stop(output: &serde_json::Value) -> String { " still running: process {pid} ({role}) or a process under it did not exit after it was signaled" ); } + for (pid, role) in &descendants_unconfirmed { + let _ = writeln!( + text, + " not confirmed: process {pid} ({role}) exited, but a process under it may still be running, because taking the processes under it down did not complete" + ); + } for (pid, role) in &unverified { let _ = writeln!( text, diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 212aadc517..856f5e3d1c 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -177,7 +177,7 @@ pub use process::{ FileLock, LoopbackPortLease, MANAGED_LAUNCH_LOCK_TIMEOUT, ProcessProbe, detach_command_session, is_canonical_loopback_host, launch_port_argument, lease_loopback_port, lease_loopback_port_or_bind_error, probe_process, process_is_running, terminate_process, - terminate_process_tree, + terminate_process_tree, windows_system_tool, }; #[cfg(windows)] pub use process::{ diff --git a/crates/rocm-core/src/proc_lifecycle.rs b/crates/rocm-core/src/proc_lifecycle.rs index e4a2fc75d8..cc2b7ddc66 100644 --- a/crates/rocm-core/src/proc_lifecycle.rs +++ b/crates/rocm-core/src/proc_lifecycle.rs @@ -208,16 +208,28 @@ pub enum TerminationOutcome { Forced, /// At least one signalled process was still alive after the bounded deadline. TimedOut, + /// The recorded process itself is gone (this stop's own wait saw it exit, or + /// it was already gone), but the separate kill of the processes under it did + /// not complete (on Windows, a `taskkill` that failed, stalled, or found no + /// such process while the root was still running), so one of them may still + /// be running and holding the device. Deliberately not + /// [`TimedOut`](Self::TimedOut): the root did not outlive any deadline. + DescendantsUnconfirmed, } impl TerminationOutcome { /// Is the recorded service confirmed to be no longer running as a result? /// - /// `false` for [`TimedOut`](Self::TimedOut) (still alive) and - /// [`Unverified`](Self::Unverified) (could not be confirmed either way). + /// `false` for [`TimedOut`](Self::TimedOut) (still alive), + /// [`Unverified`](Self::Unverified) (could not be confirmed either way) and + /// [`DescendantsUnconfirmed`](Self::DescendantsUnconfirmed) (the root is gone + /// but a process under it may not be). #[must_use] pub const fn stopped(self) -> bool { - !matches!(self, Self::TimedOut | Self::Unverified) + !matches!( + self, + Self::TimedOut | Self::Unverified | Self::DescendantsUnconfirmed + ) } /// Did the process stop without needing a forced kill (or was it already @@ -240,6 +252,7 @@ impl TerminationOutcome { Self::Graceful => "graceful", Self::Forced => "forced", Self::TimedOut => "timed_out", + Self::DescendantsUnconfirmed => "descendants_unconfirmed", } } } @@ -395,6 +408,7 @@ impl RecordedPidStop { TerminationOutcome::Graceful | TerminationOutcome::Forced | TerminationOutcome::TimedOut + | TerminationOutcome::DescendantsUnconfirmed ) ) } @@ -408,7 +422,9 @@ impl RecordedPidStop { matches!( self.outcome, RecordedPidOutcome::Terminated( - TerminationOutcome::Forced | TerminationOutcome::TimedOut + TerminationOutcome::Forced + | TerminationOutcome::TimedOut + | TerminationOutcome::DescendantsUnconfirmed ) ) } @@ -561,10 +577,18 @@ enum TreeKill { } /// The verdict for a stop whose descendant kill ended as `tree` and whose root -/// termination answered `outcome`. A failed tree kill turns any verdict that -/// would confirm the stop into [`TerminationOutcome::TimedOut`]: the root was -/// signalled, and it, or a process under it, is not confirmed exited. The -/// verdicts that already deny the stop stay as they are. +/// termination answered `outcome`. +/// +/// When the tree kill did not complete ([`TreeKill::Failed`], or +/// [`TreeKill::NotFound`] while the root was still there), a root that this +/// stop saw exit ([`Graceful`](TerminationOutcome::Graceful), +/// [`Forced`](TerminationOutcome::Forced)) or that was already gone is reported +/// [`TerminationOutcome::DescendantsUnconfirmed`]: the root is gone, which is +/// what the wait observed, but nothing watched the processes under it, so the +/// stop cannot be called confirmed. It is not `timed_out`, which would claim +/// that something outlived a deadline. The verdicts that already deny the stop +/// ([`TimedOut`](TerminationOutcome::TimedOut), the root itself outlived the +/// deadline; [`Unverified`](TerminationOutcome::Unverified)) stay as they are. /// /// A kill that found no such process ([`TreeKill::NotFound`]) is judged by the /// root's own re-read, which `outcome` carries: when the root is gone @@ -577,9 +601,12 @@ const fn after_tree_kill(tree: TreeKill, outcome: TerminationOutcome) -> Termina match (tree, outcome) { (TreeKill::Completed, outcome) => outcome, (TreeKill::NotFound, TerminationOutcome::AlreadyGone) => TerminationOutcome::AlreadyGone, - (TreeKill::NotFound | TreeKill::Failed, outcome) if outcome.stopped() => { - TerminationOutcome::TimedOut - } + ( + TreeKill::NotFound | TreeKill::Failed, + TerminationOutcome::Graceful + | TerminationOutcome::Forced + | TerminationOutcome::AlreadyGone, + ) => TerminationOutcome::DescendantsUnconfirmed, (TreeKill::NotFound | TreeKill::Failed, outcome) => outcome, } } @@ -750,7 +777,7 @@ fn wait_for_exit_code( fn taskkill_tree(pid: u32) -> TreeKill { use std::process::{Command, Stdio}; classify_taskkill(run_to_exit_code( - Command::new("taskkill") + Command::new(crate::windows_system_tool("taskkill.exe")) .args(["/PID", &pid.to_string(), "/T", "/F"]) .stdin(Stdio::null()) .stdout(Stdio::null()) @@ -966,6 +993,18 @@ mod tests { assert!(TerminationOutcome::AlreadyGone.stopped()); assert!(TerminationOutcome::IdentityMismatch.stopped()); assert!(!TerminationOutcome::TimedOut.stopped()); + // The root is gone but a process under it may not be: not a stop, not + // graceful, and not the label of a timeout. + assert!(!TerminationOutcome::DescendantsUnconfirmed.stopped()); + assert!(!TerminationOutcome::DescendantsUnconfirmed.graceful()); + assert_eq!( + TerminationOutcome::DescendantsUnconfirmed.as_str(), + "descendants_unconfirmed" + ); + assert_ne!( + TerminationOutcome::DescendantsUnconfirmed.as_str(), + TerminationOutcome::TimedOut.as_str() + ); // Unverified: neither stopped nor graceful — we could not act. assert!(!TerminationOutcome::Unverified.stopped()); @@ -1323,7 +1362,8 @@ mod tests { #[test] fn managed_stop_classifies_each_outcome_as_signaled_forced_and_stopped() { use TerminationOutcome::{ - AlreadyGone, Forced, Graceful, IdentityMismatch, TimedOut, Unverified, + AlreadyGone, DescendantsUnconfirmed, Forced, Graceful, IdentityMismatch, TimedOut, + Unverified, }; let record = record_naming(SUPERVISOR_PID, None, None, None); @@ -1335,6 +1375,8 @@ mod tests { (Graceful, true, false, true), (Forced, true, true, true), (TimedOut, true, true, false), + // Signalled and forced by the tree kill, but not confirmed gone. + (DescendantsUnconfirmed, true, true, false), ]; for (outcome, signaled, forced, stopped) in expected { let (_, stops) = stop_with(&record, outcome); @@ -1655,14 +1697,17 @@ mod tests { #[test] fn a_tree_kill_that_did_not_complete_leaves_the_stop_unconfirmed() { - use TerminationOutcome::{AlreadyGone, Forced, Graceful, TimedOut, Unverified}; + use TerminationOutcome::{ + AlreadyGone, DescendantsUnconfirmed, Forced, Graceful, TimedOut, Unverified, + }; // Every root verdict that would confirm the stop becomes the one that - // says a process under the root may still be running; the verdicts that - // already deny the stop are kept. + // says the root is gone but a process under it may still be running, not + // `timed_out`, which would say something outlived a deadline; the + // verdicts that already deny the stop are kept. for (root, expected) in [ - (Forced, TimedOut), - (Graceful, TimedOut), - (AlreadyGone, TimedOut), + (Forced, DescendantsUnconfirmed), + (Graceful, DescendantsUnconfirmed), + (AlreadyGone, DescendantsUnconfirmed), (TimedOut, TimedOut), (Unverified, Unverified), ] { @@ -1712,10 +1757,14 @@ mod tests { /// exit. #[test] fn a_tree_kill_that_found_no_process_while_the_root_runs_is_a_failed_kill() { - use TerminationOutcome::{Forced, Graceful, TimedOut, Unverified}; + use TerminationOutcome::{DescendantsUnconfirmed, Forced, Graceful, TimedOut, Unverified}; + // The termination saw the root exit (Graceful, Forced) although the kill + // said there was no such process: the exit was observed, so the report + // must not say it outlived a deadline, but nothing watched what is under + // it. for (root, expected) in [ - (Forced, TimedOut), - (Graceful, TimedOut), + (Forced, DescendantsUnconfirmed), + (Graceful, DescendantsUnconfirmed), (TimedOut, TimedOut), (Unverified, Unverified), ] { @@ -1739,7 +1788,8 @@ mod tests { TerminationOutcome::AlreadyGone, ); - assert_eq!(verdict, TerminationOutcome::TimedOut); + assert_eq!(verdict, TerminationOutcome::DescendantsUnconfirmed); + assert!(!verdict.stopped()); } /// A recorded token that can be read and differs proves another process diff --git a/crates/rocm-core/src/process.rs b/crates/rocm-core/src/process.rs index 69cd2d16d9..54ec90cb1c 100644 --- a/crates/rocm-core/src/process.rs +++ b/crates/rocm-core/src/process.rs @@ -422,6 +422,24 @@ pub(crate) const fn classify_open_failure(code: Option) -> ProcessProbe { } } +/// The path of a Windows system tool, `%SystemRoot%\System32\`. +/// +/// So a directory earlier on `PATH` cannot stand in for it; the bare name when +/// `SystemRoot` is not set. `name` includes the extension (`taskkill.exe`). +#[must_use] +pub fn windows_system_tool(name: &str) -> PathBuf { + system_tool_path(std::env::var_os("SystemRoot"), name) +} + +/// [`windows_system_tool`] for a given `SystemRoot`, so a test builds the path +/// on every host. +fn system_tool_path(system_root: Option, name: &str) -> PathBuf { + match system_root.filter(|root| !root.is_empty()) { + Some(root) => PathBuf::from(root).join("System32").join(name), + None => PathBuf::from(name), + } +} + /// Look at `pid`, keeping a process that cannot be queried apart from one that /// is gone. See [`ProcessProbe`]. #[cfg(windows)] @@ -1367,6 +1385,33 @@ mod tests { use crate::test_support::workspace_test_artifact_dir; use std::process::Stdio; + /// A system tool is started from `System32` under `SystemRoot`, never by a + /// bare name that `PATH` could resolve to something else; with no + /// `SystemRoot` it falls back to the bare name. + #[test] + fn a_system_tool_is_resolved_under_system32_of_the_system_root() { + let path = system_tool_path( + Some(std::ffi::OsString::from("C:\\Windows")), + "taskkill.exe", + ); + let mut expected = PathBuf::from("C:\\Windows"); + expected.push("System32"); + expected.push("taskkill.exe"); + assert_eq!(path, expected); + assert!(path.ends_with(Path::new("System32").join("taskkill.exe"))); + assert!(!path.parent().unwrap().as_os_str().is_empty()); + + assert_eq!( + system_tool_path(None, "tasklist.exe"), + PathBuf::from("tasklist.exe") + ); + assert_eq!( + system_tool_path(Some(std::ffi::OsString::new()), "tasklist.exe"), + PathBuf::from("tasklist.exe"), + "an empty SystemRoot is no SystemRoot" + ); + } + /// A failed `OpenProcess` is a gone process only for the code that means /// "no such process". Access denied, which a live process the caller may not /// open sets, and any code that is not known are not an exit. diff --git a/docs/architecture.md b/docs/architecture.md index e2a69270cf..da90ca7e64 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -231,7 +231,7 @@ Further `rocm-core` modules, each with its own doc comment except where noted: - [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) — OpenMPI, libatomic and libnuma detection, the `libmpi_cxx` compatibility shim, and the per-library install plans (`build_openmpi_install_plan`, `build_libatomic_install_plan`, `build_libnuma_install_plan`) and hints, needed by vLLM's ROCm wheels. The distro-generic half of that planning lives in [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs); [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) re-exports its public items, so `rocm_core::openmpi::running_as_root`, `plan_for_os_release_read` and the rest keep resolving, and `pub(crate)` re-exports `PackageManager` and `resolve_package_manager` for [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs). - [`os_release.rs`](../crates/rocm-core/src/os_release.rs) — the one `/etc/os-release` parser every reader of that file uses. - [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs) — the planning half of a system-package install that does not depend on which library is installed: `PackageManager` and `resolve_package_manager` (from an os-release `ID`/`ID_LIKE`), `SystemPackageInstallPlan` and `InstallCommand`, `plan_for_os_release_at`/`plan_for_os_release_read` (which take the per-library builder as an argument), `InstallHint`, the root and passwordless-`sudo` checks (`running_as_root`, `can_autoinstall`) and the `apt-get -s` helpers (`apt_simulate_argv`, `parse_apt_simulate_removals`, `is_protected_rocm_package`). It never runs anything and depends on no other module of this crate except [`os_release.rs`](../crates/rocm-core/src/os_release.rs) and [`hardware_root.rs`](../crates/rocm-core/src/hardware_root.rs), which it reads os-release through. The module is private; its public items are re-exported from [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs). It is a different module from [`system_packages.rs`](../apps/rocm/src/system_packages.rs) in `apps/rocm`, which runs these plans with consent and `sudo`. -- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches (`terminate_verified`), and the one managed-stop routine built on it that `rocm services stop` and `rocmd`'s `stop_server` both call: `terminate_recorded_service_pids` walks the processes a service record names (supervisor, then engine, each with its own start-time, the stopping process's own PID reported as `self` and never signalled) and returns one `RecordedPidStop` per PID (each caller, on a stop that confirms every one gone, then clears the record's recorded processes with `ManagedServiceRecord::clear_recorded_processes` in [`service_record.rs`](../crates/rocm-core/src/service_record.rs), so a repeated stop reaches no stale PID; an unconfirmed stop keeps them); `terminate_recorded_pid` terminates one of them, adding on Windows a `taskkill /T /F` of its descendants first, because `terminate_verified` cannot walk a process tree there, and only for a PID that passes the same identity check (`process_start_ticks` is the kernel start-time on Linux and the process creation time on Windows, so a recycled PID is refused before any `taskkill`; a record written before Windows recorded one cannot be refuted), holding the PID open on Windows from before that check until the stop ends so the PID cannot be reused in between, and treating a tree kill that did not complete, or did not finish within `MANAGED_STOP_GRACE` (the `taskkill` is killed and reaped at that deadline), as an unconfirmed stop, except that a `taskkill` which found no such process is judged by the root's own re-read, so a root that exited by itself is a stop; a PID that cannot be queried at all, as against one that is gone, reads as unverifiable unless a readable recorded token differs, in which case that mismatch wins (`ProcessProbe` and `probe_process` in [`process.rs`](../crates/rocm-core/src/process.rs), whose `may_be_running` is the answer for any decision that must not take an unqueryable process for gone); `MANAGED_STOP_GRACE` is the one grace period. The termination is injectable, so a caller's test supplies outcomes it cannot construct and a test of the Windows step runs on every host. +- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches (`terminate_verified`), and the one managed-stop routine built on it that `rocm services stop` and `rocmd`'s `stop_server` both call: `terminate_recorded_service_pids` walks the processes a service record names (supervisor, then engine, each with its own start-time, the stopping process's own PID reported as `self` and never signalled) and returns one `RecordedPidStop` per PID (each caller, on a stop that confirms every one gone, then clears the record's recorded processes with `ManagedServiceRecord::clear_recorded_processes` in [`service_record.rs`](../crates/rocm-core/src/service_record.rs), so a repeated stop reaches no stale PID; an unconfirmed stop keeps them); `terminate_recorded_pid` terminates one of them, adding on Windows a `taskkill /T /F` of its descendants first, because `terminate_verified` cannot walk a process tree there, and only for a PID that passes the same identity check (`process_start_ticks` is the kernel start-time on Linux and the process creation time on Windows, so a recycled PID is refused before any `taskkill`; a record written before Windows recorded one cannot be refuted), holding the PID open on Windows from before that check until the stop ends so the PID cannot be reused in between, and treating a tree kill that did not complete, or did not finish within `MANAGED_STOP_GRACE` (the `taskkill` is killed and reaped at that deadline), as an unconfirmed stop (`descendants_unconfirmed` when the root itself was seen to exit, `timed_out` only when it outlived its deadline), except that a `taskkill` which found no such process is judged by the root's own re-read, so a root that exited by itself is a stop, and starting that `taskkill` from `%SystemRoot%\System32` through `windows_system_tool`; a PID that cannot be queried at all, as against one that is gone, reads as unverifiable unless a readable recorded token differs, in which case that mismatch wins (`ProcessProbe` and `probe_process` in [`process.rs`](../crates/rocm-core/src/process.rs), whose `may_be_running` is the answer for any decision that must not take an unqueryable process for gone); `MANAGED_STOP_GRACE` is the one grace period. The termination is injectable, so a caller's test supplies outcomes it cannot construct and a test of the Windows step runs on every host. - [`report.rs`](../crates/rocm-core/src/report.rs) — the content of a Doctor report and the rule deciding whether one may exist; it sends nothing. - [`report_delivery.rs`](../crates/rocm-core/src/report_delivery.rs) — how a report leaves the machine: building its link and deciding whether a browser may be started. - [`runtime.rs`](../crates/rocm-core/src/runtime.rs) — the host-platform layer: `RuntimePlatform`/`RuntimeHost`, OS-specific executable and Python-environment names (including `runtime_python_env_root_for_executable`, the one rule for finding an environment's folder from its interpreter's path: only an interpreter directly inside a `bin` or `Scripts` folder, in any case, has one, and everything else gets `None`; the adopt command, the ComfyUI runtime environment and the vLLM launch all call it rather than keeping a copy, since the engine crate cannot reach `apps/rocm`), default config/data/cache directories, path-text and path-list helpers, and `install_root_has_payload`, the one definition of a non-empty install root (dot-entries such as the runtime marker do not count; a path that does not exist holds nothing; a root that cannot be listed is an error) that tarball extraction, update repair detection and activation validation in `apps/rocm` all ask (no module doc comment). diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 96a68cb3c6..ae4eb349c1 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -500,7 +500,10 @@ rocm services restart --yes `stop` prints `Local server stopped` only once every recorded process is confirmed gone; otherwise it prints `Local server stop not confirmed` on stderr, keeps the record and any endpoint key, and exits non-zero. It names each -process that, or a process under it, did not exit, and for one it could not +process that, or a process under it, did not exit (on Windows also one that did +exit while taking the processes under it down did not complete, so one of them +may still be running: `not confirmed: process () exited, but a +process under it may still be running`), and for one it could not verify, an operating-system command to check on it. It never prints a `retry:` line. diff --git a/docs/testing.md b/docs/testing.md index 666140a3df..ef52974e87 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1851,8 +1851,14 @@ held open from before the identity read until the stop returns, so the creation time that was compared is that of the process the kill then reaches (`the_pid_is_held_before_its_identity_is_read_and_released_after_the_stop`). And a `taskkill` that failed to start or exited nonzero leaves the stop unconfirmed, -reported as `timed_out`, whatever the root's own termination says, because only -the root's exit is watched on Windows +whatever the root's own termination says, because only the root's exit is +watched on Windows. When the root itself was seen to exit (or was already gone) +the outcome is `descendants_unconfirmed`, which says the root is gone and a +process under it may still be running; it is not `timed_out`, which is kept for +a root that outlived its deadline. `rocm services stop` prints it as a `not +confirmed:` line, and `a_stop_with_an_unconfirmed_tree_kill_says_what_happened_and_keeps_the_service` +asserts that line, that no `still running` line is printed, and the record, stop +request and key that are kept (`a_tree_kill_that_did_not_complete_leaves_the_stop_unconfirmed`, `only_a_clean_taskkill_exit_is_a_completed_tree_kill_and_128_is_not_found`). A `taskkill` that exits 128 (process not found) is judged by the root's own re-read: @@ -1870,7 +1876,11 @@ child under a 200 ms deadline on every host and asserts the error, the bounded elapsed time, the reaped child and the failed verdict; `a_helper_that_exits_in_time_gives_its_exit_code` and `a_helper_that_cannot_be_started_is_an_error` are its neighbours). The report and the record a `timed_out` stop leaves are covered by -`an_unconfirmed_stop_keeps_the_recorded_processes_for_the_next_stop`. A readable +`an_unconfirmed_stop_keeps_the_recorded_processes_for_the_next_stop`. The +`taskkill` is started from `%SystemRoot%\System32`, not by a bare name `PATH` +could resolve (`a_system_tool_is_resolved_under_system32_of_the_system_root` +builds the path on every host; ComfyUI's own `taskkill` and `tasklist` calls use +the same helper). A readable recorded token that differs outranks a failed query (`a_readable_token_mismatch_outranks_a_failed_query`), but the creation time and the query are read with the same access right, so on Windows this only reaches a @@ -1891,7 +1901,7 @@ drives the refresh with an unqueryable PID and then a confirmed-gone one, an unverified stop and then the refresh over the record it wrote, and `a_record_is_in_use_unless_every_recorded_process_is_confirmed_gone` covers the removal guard; each takes the probe as an argument, so they run on every host. -Two gaps in the scenario coverage are deliberate. `service-stop-07` records no +Three gaps in the scenario coverage are deliberate. `service-stop-07` records no token, so the path a record written by this version takes on Windows (a token recorded, matching, with the tree kill then running) has no scenario; it rests on `what_a_stop_does_for_each_recorded_and_live_token` and the other injected tests, @@ -1902,6 +1912,18 @@ tool, never in the text of `rocm services stop`, and it names the stopping process's own PID, which a scenario cannot plant in a record before the process that will read it exists. `a_stop_reports_its_own_pid_as_self` and `a_self_row_adds_no_line_to_an_unconfirmed_report` cover the answer and the report. +Third, the `check:` line an unverified stop prints is a diagnostic, not a +remediation: it shows whether the process is still running, and what clears the +condition is that process ending (or, for a record held by a process `rocm` +cannot open, the hand-clearing in `manual-testing.md`), after which a stop or a +service listing confirms and drops the key. The command is a stock one the host +has, pinned by `the_process_check_command_is_the_one_the_host_has`, and +`service-stop-07` uses the same `tasklist /FI "PID eq "` filter (in CSV +form, which does not depend on the display language) to see whether a process is +running; no test follows the printed command on a Windows host and reads its +default table output, because that output is localised. What follows once the +process is gone is covered by +`a_pending_stop_keeps_its_key_and_marker_while_a_process_cannot_be_queried`. `service-stop-06` (Linux) is the same stop of a real recorded process through `rocm services stop`, checking the persisted state: the process exited and the record names none. Run the Windows scenario on a Windows host with From c5cc6075c2eea5acf670dc16835e39c40c8b4b3e Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 17:29:07 +0000 Subject: [PATCH 15/30] fix(core): confirm a root that exited alone only when another recorded process is gone, and add a narrow rocm services remove --force for a record held by an unqueryable process A taskkill that found no process while the root was already gone was reported as already_gone, which confirmed the stop on the root's row alone. taskkill /T reaches children only through a live parent, so a child the root left running was out of its reach, and the stop kept no list of what was under the root. That case is now its own outcome, exited_before_tree_kill (not a stop, nothing signalled, printed as "not confirmed: process N (role) was already gone ..."), and terminate_recorded_service_pids settles it across the record: it becomes already_gone only when another recorded process (the engine, which is visited as its own role) is confirmed gone. A root that is the only recorded process, or whose sibling is running or unconfirmed, stays unconfirmed. Both callers fold every row, so the record, PIDs and key are kept unless every recorded role is confirmed; tests drive that fold in rocm and in rocmd, and the settling through the injected functions. The doc comment says the engine's own row is the safety net. A dead server's PID taken by a process the account cannot open reads as unqueryable and is treated as possibly running for good, so remove and prune refused it with no way out that rocm offered, and the refusal advised a stop that only reports unverified again. rocm services remove --force --yes now removes such a record, and only such a record: no recorded process may be confirmed running, at least one must be unqueryable, and a stop must have been requested. It says which process could not be queried and that a process still running keeps running, and the plain refusal names this command. prune has no --force. The e2e-test-hooks build reads ROCM_E2E_UNQUERYABLE_PIDS so scenarios service-unqueryable-01 and -02 drive the stop, the refusal naming --force and the forced removal through the real binary. Also: process_is_running on Windows delegates to probe_process rather than repeating its OpenProcess sequence; every unsafe block this change adds has its own SAFETY comment; the README and stop help are rewrapped; the docs describe the new outcome, the settling and --force. Signed-off-by: Jussi Elo --- README.md | 27 +- apps/rocm/src/cli.rs | 23 +- apps/rocm/src/service_control.rs | 83 +++++ apps/rocm/src/service_hints.rs | 10 + apps/rocm/src/service_records.rs | 291 +++++++++++++++++- apps/rocm/src/services_cmd.rs | 29 +- apps/rocm/src/services_text.rs | 12 + apps/rocmd/src/service.rs | 90 ++++++ crates/rocm-core/src/proc_lifecycle.rs | 262 ++++++++++++++-- crates/rocm-core/src/process.rs | 53 ++-- docs/architecture.md | 4 +- docs/manual-testing.md | 29 +- docs/testing.md | 40 ++- .../service_record_unqueryable.feature | 42 +++ .../tests/e2e/service_cleanup_steps.rs | 176 +++++++++++ tests/e2e-cucumber/tests/feature_naming.rs | 1 + 16 files changed, 1080 insertions(+), 92 deletions(-) create mode 100644 tests/e2e-cucumber/features/service_record_unqueryable.feature diff --git a/README.md b/README.md index a43aead7ac..a7cf41ed3b 100644 --- a/README.md +++ b/README.md @@ -1254,7 +1254,7 @@ rocm services list [--all] [--json] rocm services logs [--follow] rocm services stop --yes rocm services restart --yes -rocm services remove --yes +rocm services remove --yes [--force] rocm services prune [--older-than-hours | --any-age] [--dry-run] [--yes] ``` @@ -1263,13 +1263,15 @@ gone. If it cannot confirm that, it prints `Local server stop not confirmed` on stderr and exits with an error. It keeps the record, which still names the processes, and any endpoint key, which the server may still be enforcing. It names each process that, or a process under it, did not exit after it was -signaled (on Windows also a process that did exit while taking the processes -under it down did not complete, so one of them may still be running); for a -process it could not verify as the server's (its recorded +signaled. On Windows it also names a process that did exit (or was already gone) +when taking the processes under it down did not complete, or found nothing and +no other recorded process is confirmed gone, because one of them may still be +running. For a process it could not verify as the server's (its recorded start-time can no longer be read back, or, on Windows, `rocm` cannot query it at -all), it signals nothing and names an operating-system command to check on it. It does not suggest -running the stop again: when only a process under the recorded one survived, a -second stop can report the server stopped while that process still runs. +all), it signals nothing and names an operating-system command to check on it. +It does not suggest running the stop again: when only a process under the +recorded one survived, a second stop can report the server stopped while that +process still runs. `restart` starts a server again only once its stop has confirmed that every recorded process is gone; if the stop cannot confirm that, it says so, starts @@ -1278,8 +1280,9 @@ nothing, and leaves the stop request recorded. After the stop confirms, `restart` takes the same launch lock `rocm serve` uses and re-checks the port before it respawns: if another live managed service is now recorded on that port, or (on the default `127.0.0.1` host only) the port is -bound by another process, it refuses and the service stays stopped (`rocm services list` shows it). The lock wait is -bounded (5 seconds), unlike `rocm serve`'s: if a launch still holds the lock, +bound by another process, it refuses and the service stays stopped +(`rocm services list` shows it). The lock wait is bounded (5 seconds), unlike +`rocm serve`'s: if a launch still holds the lock, `restart` fails saying the server was stopped but not restarted and names `rocm services restart --yes` to run once that launch finishes. @@ -1290,7 +1293,11 @@ alone, and additionally clears leftover files whose record is already gone. A server counts as running while any process it recorded may be running, which on Windows includes one `rocm` cannot query (a live process the account may not open): `remove` and `prune` refuse such a record until that process is gone. -`docs/manual-testing.md` describes clearing one by hand. +`remove --force --yes` is the way out for exactly that case: it +removes the record when a stop was requested, no recorded process is confirmed +running, and at least one could not be queried, says which one, and never +removes a record whose process `rocm` can see running. `prune` has no +`--force`. `docs/manual-testing.md` walks through it. Removal destroys both the log and the `restart` option for the records it takes, so `prune` only considers records untouched for 24 hours. Age is diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index 258d1c7491..c56111a97c 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -1131,12 +1131,12 @@ pub(crate) enum ServicesCommand { /// confirmed and exits with an error, keeping the record, which still names /// the processes, and any endpoint key, which the server may still be /// enforcing. It names each process that did not exit (on Windows also one - /// that exited while taking the processes under it down did not complete), - /// and for one it could - /// not verify (its recorded start-time can no longer be read back, or, on - /// Windows, it cannot be queried at all) it signals nothing and names an - /// operating-system command to check on it. It does not suggest running the - /// stop again. + /// that exited, or was already gone, when taking the processes under it + /// down did not complete or found nothing and no other recorded process is + /// confirmed gone), and for one it could not verify (its recorded + /// start-time can no longer be read back, or, on Windows, it cannot be + /// queried at all) it signals nothing and names an operating-system + /// command to check on it. It does not suggest running the stop again. Stop { /// Service id from `rocm services list`. service_id: String, @@ -1170,6 +1170,17 @@ pub(crate) enum ServicesCommand { /// Required; remove refuses without it. #[arg(long)] yes: bool, + /// Remove a record whose recorded process `rocm` cannot query. + /// + /// For the one case with no other way out: a stop was requested, no + /// recorded process is confirmed running, and at least one could not be + /// queried (on Windows, a live process the account may not open, which + /// can be a dead server's PID taken by another program). The record, its + /// log, engine state and endpoint key are deleted, the command says + /// which process could not be queried, and a process that is still + /// running keeps running. Refused for any process confirmed running. + #[arg(long)] + force: bool, }, /// Delete local server records that are no longer running. /// diff --git a/apps/rocm/src/service_control.rs b/apps/rocm/src/service_control.rs index 67dac96c95..433e186b8d 100644 --- a/apps/rocm/src/service_control.rs +++ b/apps/rocm/src/service_control.rs @@ -2865,4 +2865,87 @@ mod tests { assert!(on_disk.stop_requested_unix_ms.is_some()); assert_eq!(key.as_deref(), Some("key-123")); } + + /// A root that exited by itself before its tree could be taken down confirms + /// the stop only through the record's other roles, through the real routine + /// and this caller's own folding: the root's row alone is not enough, and the + /// record, its PIDs, its stop request and its key are kept unless the engine + /// role is confirmed gone too. + #[test] + fn a_root_that_exited_alone_confirms_the_stop_only_when_the_engine_role_does() { + use rocm_core::TerminationOutcome::{ExitedBeforeTreeKill, Forced, TimedOut}; + + for (engine, confirmed) in [(Forced, true), (TimedOut, false)] { + let (root, paths) = test_paths("services-stop-root-exited-alone"); + paths.ensure().unwrap(); + let service_id = "svc-root-exited-alone"; + let mut record = stop_test_record(&paths, service_id); + record.supervisor_pid = 999_999_998; + record.engine_pid = Some(999_999_997); + record.write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |record| { + rocm_core::terminate_recorded_service_pids(record, |identity| { + if identity.pid == 999_999_998 { + ExitedBeforeTreeKill + } else { + engine + } + }) + }) + .unwrap(); + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + + assert_eq!(report["stopped"], confirmed, "{engine:?}: {report}"); + if confirmed { + assert_eq!(on_disk.status, "stopped"); + assert_eq!( + on_disk.supervisor_pid, 0, + "the confirmed stop clears the PIDs" + ); + assert_eq!(key, None, "and the key"); + } else { + assert_eq!(on_disk.supervisor_pid, 999_999_998, "the PIDs are kept"); + assert!(on_disk.stop_requested_unix_ms.is_some()); + assert_eq!(key.as_deref(), Some("key-123"), "the key is kept"); + } + } + } + + /// A lone root that exited by itself is not vouched for by anything else the + /// record names, so this caller does not clear the record on its say-so. + #[test] + fn a_lone_root_that_exited_alone_leaves_the_stop_unconfirmed() { + let (root, paths) = test_paths("services-stop-lone-root-exited-alone"); + paths.ensure().unwrap(); + let service_id = "svc-lone-root-exited-alone"; + stop_test_record(&paths, service_id).write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |record| { + rocm_core::terminate_recorded_service_pids(record, |_| { + rocm_core::TerminationOutcome::ExitedBeforeTreeKill + }) + }) + .unwrap(); + let answer = internal_stop_server_value(report); + let outcome = finish_service_action(&paths, "stop_server", service_id, &answer); + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + + let rendered = format!("{:#}", outcome.expect_err("not a confirmed stop")); + assert!( + rendered.contains("was already gone when the processes under it were to be taken down"), + "{rendered}" + ); + assert!(!rendered.contains("Local server stopped"), "{rendered}"); + assert_no_retry_line(&rendered); + assert_eq!(on_disk.supervisor_pid, 999_999_999); + assert!(on_disk.stop_requested_unix_ms.is_some()); + assert_eq!(key.as_deref(), Some("key-123")); + } } diff --git a/apps/rocm/src/service_hints.rs b/apps/rocm/src/service_hints.rs index 49d659dc10..6556c2e70a 100644 --- a/apps/rocm/src/service_hints.rs +++ b/apps/rocm/src/service_hints.rs @@ -38,6 +38,16 @@ pub(crate) fn service_action_hint(action: &str, service_id: &str) -> String { format!("rocm services {action} {} --yes", shell_quote(service_id)) } +/// The directly-runnable `rocm services remove --force --yes` hint: the +/// removal that goes ahead for a record whose recorded processes cannot be +/// queried (see `remove_managed_service_record`). +pub(crate) fn service_forced_remove_hint(service_id: &str) -> String { + format!( + "rocm services remove {} --force --yes", + shell_quote(service_id) + ) +} + /// The directly-runnable `rocm services stop --yes` hint. pub(crate) fn service_stop_hint(service_id: &str) -> String { service_action_hint("stop", service_id) diff --git a/apps/rocm/src/service_records.rs b/apps/rocm/src/service_records.rs index 7bad1ff825..6c6aea781f 100644 --- a/apps/rocm/src/service_records.rs +++ b/apps/rocm/src/service_records.rs @@ -40,6 +40,7 @@ use rocm_core::{AppPaths, ManagedServiceRecord, ProcessProbe}; use crate::managed_service_status::{ load_managed_services, managed_service_is_live, recorded_pids_may_be_running, + recorded_service_pids, }; use crate::services_text::{load_managed_service, validate_service_id}; use crate::{cli_progress, cli_report, endpoint_keys, service_hints}; @@ -182,6 +183,57 @@ fn managed_service_record_is_in_use_with( managed_service_is_live(record) || recorded_pids_may_be_running(record, probe) } +/// The recorded processes `--force` may remove a record over: `Some` only when +/// none of them is confirmed running, at least one could not be queried (so +/// `rocm` cannot tell whether it is still running), and a stop was requested on +/// the record, so the user has already tried to stop it. A process confirmed +/// running is never forced past. +fn forceable_unqueryable_pids( + record: &ManagedServiceRecord, + probe: impl Fn(u32) -> ProcessProbe, +) -> Option> { + let probed: Vec<(u32, ProcessProbe)> = recorded_service_pids(record) + .into_iter() + .map(|pid| (pid, probe(pid))) + .collect(); + if probed + .iter() + .any(|(_, probed)| *probed == ProcessProbe::Running) + { + return None; + } + let unqueryable: Vec = probed + .into_iter() + .filter(|(_, probed)| *probed == ProcessProbe::Unqueryable) + .map(|(pid, _)| pid) + .collect(); + (!unqueryable.is_empty() && record.stop_requested_unix_ms.is_some()).then_some(unqueryable) +} + +/// The recorded PIDs as the message names them: `process 4242`, or +/// `processes 4242 and 4243`. +fn named_processes(pids: &[u32]) -> String { + match pids { + [one] => format!("process {one}"), + many => format!( + "processes {}", + many.iter() + .map(u32::to_string) + .collect::>() + .join(" and ") + ), + } +} + +/// What [`remove_managed_service_record`] did. +pub(crate) struct RemovedRecord { + /// The text to print. + pub(crate) text: String, + /// The removal went ahead over a recorded process that could not be queried + /// (`--force`), rather than for a record that is not running. + pub(crate) forced: bool, +} + /// Remove one non-running local server record and everything it owns. /// /// Refuses a live record rather than stopping it: a removal that silently killed @@ -190,22 +242,59 @@ fn managed_service_record_is_in_use_with( /// which has already been refreshed against the engine state file and the real /// processes — a manifest still saying `ready` for a dead PID has demoted to /// `stopped` by then, and must stay removable. +/// +/// A recorded process that could not be queried (on Windows, a live one this +/// process may not open) makes the record count as running, and nothing else +/// `rocm` has can tell a dead server whose PID was reused from a live one. For +/// that case only, `--force` removes the record anyway: when no recorded process +/// is confirmed running, at least one could not be queried, and a stop was +/// requested. The removal then says exactly that, since the process may still +/// be serving and its endpoint key is deleted with the record. +#[cfg(test)] pub(crate) fn remove_managed_service_record( paths: &AppPaths, service_id: &str, yes: bool, ) -> Result { + remove_managed_service_record_with(paths, service_id, yes, false, rocm_core::probe_process) + .map(|removed| removed.text) +} + +/// [`remove_managed_service_record`] with `--force` and the read of each +/// recorded PID supplied by the caller. +pub(crate) fn remove_managed_service_record_with( + paths: &AppPaths, + service_id: &str, + yes: bool, + force: bool, + probe: impl Fn(u32) -> ProcessProbe, +) -> Result { validate_service_id(service_id)?; let record = load_managed_service(paths, service_id)?; + let mut forced_over = None; // Checked before `--yes` on purpose: for a running server "stop it first" is // the actionable error, and repeating the command with --yes must not be the // advice a user takes away from it. - if managed_service_record_is_in_use(&record) { - bail!( - "local server `{service_id}` is {} and cannot be removed while it is running.\n\nTry: {}", - record.status, - service_hints::service_stop_hint(service_id) - ); + if managed_service_record_is_in_use_with(&record, &probe) { + let forceable = forceable_unqueryable_pids(&record, &probe); + match (force, forceable) { + (true, Some(pids)) => forced_over = Some(pids), + (false, Some(pids)) => bail!( + "local server `{service_id}` cannot be removed: {} it recorded could not be queried, so `rocm` cannot tell whether it is still running.\n\nTry: {}", + named_processes(&pids), + service_hints::service_forced_remove_hint(service_id) + ), + (true, None) => bail!( + "--force removes a record only when none of its recorded processes is confirmed running, at least one could not be queried, and a stop was requested. Local server `{service_id}` is {} and does not meet that.\n\nTry: {}", + record.status, + service_hints::service_stop_hint(service_id) + ), + (false, None) => bail!( + "local server `{service_id}` is {} and cannot be removed while it is running.\n\nTry: {}", + record.status, + service_hints::service_stop_hint(service_id) + ), + } } if !yes { bail!( @@ -223,6 +312,15 @@ pub(crate) fn remove_managed_service_record( "Removing local server record `{service_id}` (status: {}).", record.status ); + if let Some(pids) = &forced_over { + // Said before the delete and in full: what is being risked is the user's + // to know, and after this there is no record left to ask about. + let _ = writeln!( + output, + "--force: {} could not be queried, so `rocm` cannot tell whether it is still running. If it is, it keeps running; this removes the record and the endpoint key it may still be serving with.", + named_processes(pids) + ); + } let _ = writeln!(output, " log: {}", artifacts.log.display()); let _ = writeln!( output, @@ -241,7 +339,10 @@ pub(crate) fn remove_managed_service_record( report = report.detail("removed", path.display()); } let _ = write!(output, "{}", report.render()); - Ok(output) + Ok(RemovedRecord { + text: output, + forced: forced_over.is_some(), + }) } /// One record `rocm services prune` would delete. @@ -1829,8 +1930,8 @@ mod tests { Ok(()) } - /// The way out `docs/manual-testing.md` gives for a record whose process - /// `rocm` cannot query: the record is refused while that process may be + /// The by-hand way out, which still works alongside `--force`, for a record + /// whose process `rocm` cannot query: the record is refused while that process may be /// running, and deleting its `.json` by hand and running /// `prune --any-age` leaves nothing behind (log, key and engine state all /// swept), with the endpoint key among the files removed. @@ -1866,4 +1967,176 @@ mod tests { assert_eq!(outcome.removed_files, 3); Ok(()) } + + /// A record the user already tried to stop, held by the test's own live PID, + /// so the real liveness read that loads it leaves it alone and only the + /// injected probe says the process cannot be queried. + fn held_record(paths: &AppPaths, service_id: &str, stop_requested: bool) -> Result<()> { + let mut record = plant_service_record(paths, service_id, "ready", std::process::id())?; + if stop_requested { + record.stop_requested_unix_ms = Some(1); + record.write()?; + } + Ok(()) + } + + fn files_left(paths: &AppPaths, service_id: &str) -> usize { + let manifest = paths.service_manifest_path(service_id).exists(); + let log = paths.service_log_path(service_id).exists(); + let key = endpoint_keys::endpoint_key_file_path(paths, service_id).exists(); + usize::from(manifest) + usize::from(log) + usize::from(key) + } + + /// Without `--force` the refusal names the forced removal, and everything the + /// record owns is still there: the advice and the state, together. + #[test] + fn a_record_held_by_an_unqueryable_process_is_refused_naming_the_forced_removal() -> Result<()> + { + let (root, paths) = test_paths("services-remove-unqueryable-refused"); + paths.ensure()?; + held_record(&paths, "svc-held", true)?; + + let error = remove_managed_service_record_with(&paths, "svc-held", true, false, |_| { + ProcessProbe::Unqueryable + }) + .map(|removed| removed.text) + .expect_err("a record held by an unqueryable process is refused"); + let message = format!("{error:#}"); + let left = files_left(&paths, "svc-held"); + let _ = fs::remove_dir_all(root); + + assert!(message.contains("could not be queried"), "{message}"); + assert!( + message.contains("rocm services remove svc-held --force --yes"), + "{message}" + ); + assert_eq!(left, 3, "nothing was removed"); + // The command the refusal names parses as printed. + let words: Vec = service_hints::service_forced_remove_hint("svc-held") + .split_whitespace() + .map(str::to_owned) + .collect(); + assert_eq!( + crate::advised_commands::rocm_verdict(&words[1..]), + crate::advised_commands::Verdict::Parses, + "{words:?}" + ); + Ok(()) + } + + /// `--force` removes it and says what it risked; the audit-relevant flag is set. + #[test] + fn force_removes_a_record_held_by_an_unqueryable_process_and_says_so() -> Result<()> { + let (root, paths) = test_paths("services-remove-unqueryable-forced"); + paths.ensure()?; + held_record(&paths, "svc-held", true)?; + let pid = std::process::id(); + + let removed = remove_managed_service_record_with(&paths, "svc-held", true, true, |_| { + ProcessProbe::Unqueryable + })?; + let left = files_left(&paths, "svc-held"); + let _ = fs::remove_dir_all(root); + + assert!(removed.forced); + assert!( + removed.text.contains(&format!( + "--force: process {pid} could not be queried, so `rocm` cannot tell whether it is still running" + )), + "{}", + removed.text + ); + assert!(removed.text.contains("endpoint key"), "{}", removed.text); + assert_eq!(left, 0, "every file the record owned is gone"); + Ok(()) + } + + /// `--force` never goes past a process that is confirmed running, and it + /// needs a stop to have been requested: the refusal says which, and names the + /// stop that is the way forward. + #[test] + fn force_is_refused_for_a_running_process_and_without_a_requested_stop() -> Result<()> { + let (root, paths) = test_paths("services-remove-force-refused"); + paths.ensure()?; + held_record(&paths, "svc-running", true)?; + held_record(&paths, "svc-no-stop", false)?; + + let running = remove_managed_service_record_with(&paths, "svc-running", true, true, |_| { + ProcessProbe::Running + }) + .map(|removed| removed.text) + .expect_err("a confirmed running process is never forced past"); + let no_stop = remove_managed_service_record_with(&paths, "svc-no-stop", true, true, |_| { + ProcessProbe::Unqueryable + }) + .map(|removed| removed.text) + .expect_err("a record nobody tried to stop is not forced"); + let left = files_left(&paths, "svc-running") + files_left(&paths, "svc-no-stop"); + let _ = fs::remove_dir_all(root); + + for error in [format!("{running:#}"), format!("{no_stop:#}")] { + assert!( + error.contains("--force removes a record only when"), + "{error}" + ); + } + assert!( + format!("{running:#}").contains("rocm services stop svc-running --yes"), + "{running:#}" + ); + assert_eq!(left, 6, "nothing was removed"); + Ok(()) + } + + /// `--force` on a record that is simply not running changes nothing: it is + /// removed as usual and not reported as forced. + #[test] + fn force_on_a_record_that_is_not_running_is_an_ordinary_removal() -> Result<()> { + let (root, paths) = test_paths("services-remove-force-unneeded"); + paths.ensure()?; + plant_service_record(&paths, "svc-stopped", "stopped", 999_999_999)?; + + let removed = + remove_managed_service_record_with(&paths, "svc-stopped", true, true, |_| { + ProcessProbe::Gone + })?; + let _ = fs::remove_dir_all(root); + + assert!(!removed.forced); + assert!(!removed.text.contains("--force"), "{}", removed.text); + Ok(()) + } + + /// One recorded process that cannot be queried does not make a record + /// forceable while another is confirmed running: the running one is never + /// forced past, whichever role it has. + #[test] + fn force_is_refused_when_one_process_is_unqueryable_and_another_is_running() -> Result<()> { + let (root, paths) = test_paths("services-remove-force-mixed"); + paths.ensure()?; + let own = std::process::id(); + let mut record = plant_service_record(&paths, "svc-mixed", "ready", own)?; + record.engine_pid = Some(999_999_997); + record.stop_requested_unix_ms = Some(1); + record.write()?; + + let error = remove_managed_service_record_with(&paths, "svc-mixed", true, true, |pid| { + if pid == own { + ProcessProbe::Unqueryable + } else { + ProcessProbe::Running + } + }) + .map(|removed| removed.text) + .expect_err("a record with a process confirmed running is never forced"); + let left = files_left(&paths, "svc-mixed"); + let _ = fs::remove_dir_all(root); + + assert!( + format!("{error:#}").contains("--force removes a record only when"), + "{error:#}" + ); + assert_eq!(left, 3, "nothing was removed"); + Ok(()) + } } diff --git a/apps/rocm/src/services_cmd.rs b/apps/rocm/src/services_cmd.rs index b559503110..79c21675f4 100644 --- a/apps/rocm/src/services_cmd.rs +++ b/apps/rocm/src/services_cmd.rs @@ -38,7 +38,7 @@ use crate::service_hints::{ service_action_verb, }; use crate::service_records::{ - prune_managed_service_records, remove_managed_service_record, service_prune_min_age_hours, + prune_managed_service_records, remove_managed_service_record_with, service_prune_min_age_hours, }; use crate::services_text::{ follow_service_logs, render_service_action_result, @@ -74,17 +74,32 @@ pub(crate) fn services(command: Option) -> Result<()> { ServicesCommand::Restart { service_id, yes } => { run_approved_service_action(&paths, "restart_server", &service_id, yes) } - ServicesCommand::Remove { service_id, yes } => { - print!( - "{}", - remove_managed_service_record(&paths, &service_id, yes)? - ); + ServicesCommand::Remove { + service_id, + yes, + force, + } => { + let removed = remove_managed_service_record_with( + &paths, + &service_id, + yes, + force, + rocm_core::probe_process, + )?; + print!("{}", removed.text); + let message = if removed.forced { + format!( + "removed local server record {service_id} (--force: a recorded process could not be queried)" + ) + } else { + format!("removed local server record {service_id}") + }; record_cli_audit_event( &paths, "service", "remove_record", "info", - format!("removed local server record {service_id}"), + message, Some(&service_id), ); Ok(()) diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index 21d3927952..fd32ce48e7 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -1054,6 +1054,10 @@ pub(crate) fn render_service_action_result(tool: &str, value: &serde_json::Value /// not completed (`descendants_unconfirmed`, Windows): this stop saw the /// recorded process exit, so it does not say anything outlived a deadline, but /// one of the processes under it may still be running; +/// - a recorded process that exited by itself before the processes under it +/// could be taken down (`exited_before_tree_kill`, Windows), with no other +/// recorded process confirmed gone to vouch for what was under it: nothing +/// was signalled, and one of them may still be running; /// - a process that was not signalled because its identity could not be /// verified (`unverified`), with how to look at it. There are two routes to /// it: a recorded start-time that can no longer be read back from a running @@ -1096,6 +1100,8 @@ fn render_unconfirmed_service_stop(output: &serde_json::Value) -> String { let timed_out = with_outcome(rocm_core::TerminationOutcome::TimedOut.as_str()); let descendants_unconfirmed = with_outcome(rocm_core::TerminationOutcome::DescendantsUnconfirmed.as_str()); + let exited_before_tree_kill = + with_outcome(rocm_core::TerminationOutcome::ExitedBeforeTreeKill.as_str()); let unverified = with_outcome(rocm_core::TerminationOutcome::Unverified.as_str()); let mut text = String::new(); @@ -1125,6 +1131,12 @@ fn render_unconfirmed_service_stop(output: &serde_json::Value) -> String { " not confirmed: process {pid} ({role}) exited, but a process under it may still be running, because taking the processes under it down did not complete" ); } + for (pid, role) in &exited_before_tree_kill { + let _ = writeln!( + text, + " not confirmed: process {pid} ({role}) was already gone when the processes under it were to be taken down, and nothing else the server recorded vouches for them, so one of them may still be running" + ); + } for (pid, role) in &unverified { let _ = writeln!( text, diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index b74e66b94e..b259df59d0 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -1093,4 +1093,94 @@ mod tests { ); } } + + /// A root that exited by itself before its tree could be taken down confirms + /// the stop only through the record's other roles, as this caller folds + /// them: the supervisor's own row is not enough, and the record, its PIDs, + /// its stop marker and its key are kept unless the engine role is + /// confirmed gone too. + #[test] + fn a_root_that_exited_alone_confirms_the_stop_only_when_the_engine_role_does() -> Result<()> { + use rocm_core::TerminationOutcome::{ExitedBeforeTreeKill, Forced, TimedOut}; + + for (engine, confirmed) in [(Forced, true), (TimedOut, false)] { + let (root, paths, service_id) = plant_record_naming( + "stop-root-exited-alone", + SUPERVISOR_PID, + None, + Some(ENGINE_PID), + None, + )?; + let key = rocm_engine_protocol::endpoint_key_file_path(&paths, service_id); + fs::create_dir_all(paths.services_dir())?; + fs::write(&key, "secret-key")?; + + let report = stop_managed_service_with(&paths, service_id, |identity| { + if identity.pid == SUPERVISOR_PID { + ExitedBeforeTreeKill + } else { + engine + } + })?; + let record = find_managed_service(&paths, service_id)?; + let key_kept = key.exists(); + fs::remove_dir_all(root).ok(); + + assert_eq!( + stop_report_confirmed(&report), + confirmed, + "{engine:?}: {report}" + ); + assert_eq!( + report["pid_outcomes"][0]["outcome"], + if confirmed { + "already_gone" + } else { + "exited_before_tree_kill" + }, + "{engine:?}: {report}" + ); + if confirmed { + assert_eq!(record.status, "stopped"); + assert_eq!( + record.supervisor_pid, 0, + "the confirmed stop clears the PIDs" + ); + assert!(!key_kept, "and the key"); + } else { + assert_eq!(record.status, "ready", "{engine:?}"); + assert_eq!(record.supervisor_pid, SUPERVISOR_PID, "the PIDs are kept"); + assert!( + record.stop_requested_unix_ms.is_some(), + "the marker is kept" + ); + assert!(key_kept, "the key is kept"); + } + } + Ok(()) + } + + /// A root that exited by itself with no other role recorded cannot be + /// vouched for, so this caller does not clear the record on its say-so. + #[test] + fn a_lone_root_that_exited_alone_leaves_the_stop_unconfirmed() -> Result<()> { + let (root, paths, service_id) = plant_record_naming( + "stop-lone-root-exited-alone", + SUPERVISOR_PID, + None, + None, + None, + )?; + + let report = stop_managed_service_with(&paths, service_id, |_| { + rocm_core::TerminationOutcome::ExitedBeforeTreeKill + })?; + let record = find_managed_service(&paths, service_id)?; + fs::remove_dir_all(root).ok(); + + assert!(!stop_report_confirmed(&report), "{report}"); + assert_eq!(record.supervisor_pid, SUPERVISOR_PID); + assert!(record.stop_requested_unix_ms.is_some()); + Ok(()) + } } diff --git a/crates/rocm-core/src/proc_lifecycle.rs b/crates/rocm-core/src/proc_lifecycle.rs index cc2b7ddc66..3c09ae336a 100644 --- a/crates/rocm-core/src/proc_lifecycle.rs +++ b/crates/rocm-core/src/proc_lifecycle.rs @@ -215,6 +215,15 @@ pub enum TerminationOutcome { /// be running and holding the device. Deliberately not /// [`TimedOut`](Self::TimedOut): the root did not outlive any deadline. DescendantsUnconfirmed, + /// The recorded process was gone by the time the separate kill of the + /// processes under it ran (on Windows, `taskkill` found no such process and + /// the root's re-read agrees): it exited by itself after the identity check. + /// `taskkill /T` reaches children only through a live parent, and nothing + /// recorded what was under the root, so this stop cannot say whether a + /// process it left running is still there. Not a stop on its own; the + /// aggregate over the record's processes confirms it when another recorded + /// process is confirmed gone ([`terminate_recorded_service_pids`]). + ExitedBeforeTreeKill, } impl TerminationOutcome { @@ -223,12 +232,17 @@ impl TerminationOutcome { /// `false` for [`TimedOut`](Self::TimedOut) (still alive), /// [`Unverified`](Self::Unverified) (could not be confirmed either way) and /// [`DescendantsUnconfirmed`](Self::DescendantsUnconfirmed) (the root is gone - /// but a process under it may not be). + /// but a process under it may not be) and + /// [`ExitedBeforeTreeKill`](Self::ExitedBeforeTreeKill) (the root is gone and + /// nothing says what was under it). #[must_use] pub const fn stopped(self) -> bool { !matches!( self, - Self::TimedOut | Self::Unverified | Self::DescendantsUnconfirmed + Self::TimedOut + | Self::Unverified + | Self::DescendantsUnconfirmed + | Self::ExitedBeforeTreeKill ) } @@ -253,6 +267,7 @@ impl TerminationOutcome { Self::Forced => "forced", Self::TimedOut => "timed_out", Self::DescendantsUnconfirmed => "descendants_unconfirmed", + Self::ExitedBeforeTreeKill => "exited_before_tree_kill", } } } @@ -468,6 +483,28 @@ impl RecordedPidStop { /// `terminate` is injectable so a caller's test can supply the verdicts it /// cannot construct on demand (a process that survives `SIGKILL`, one whose /// identity cannot be read). +/// +/// One verdict is settled across the record rather than per PID. A root that +/// exited by itself before its processes could be taken down +/// ([`TerminationOutcome::ExitedBeforeTreeKill`]) cannot say what it left +/// running. It becomes [`TerminationOutcome::AlreadyGone`] only when another +/// process the record names is confirmed gone (a stop, or itself such a root): +/// the record's other process is the one thing that was under the root and is +/// independently verified. A root that is the only process the record names, or +/// whose sibling is still running or unconfirmed, stays unconfirmed. Processes +/// the record does not name that were under a root can be neither found nor +/// verified, which is the same limit as for a root that was already gone when a +/// stop began. +/// +/// This relies on the engine being visited as its own role, which it is: the +/// engine PID is an entry of its own, with its own token and its own verdict, so +/// a worker the root left running is reached by the engine's own termination +/// when the record names it. Both callers fold every row (`rocm services stop` +/// in `stop_internal_managed_service_confirmed_with`, `rocmd` in +/// `stop_managed_service_with`): the stop is confirmed, and the recorded PIDs and +/// the endpoint key cleared, only when every recorded role is individually +/// stopped. A root that exited alone therefore never confirms a stop on its own +/// row, and the record is kept when the engine role is not confirmed. pub fn terminate_recorded_service_pids( record: &crate::ManagedServiceRecord, terminate: impl Fn(&ProcessIdentity) -> TerminationOutcome, @@ -496,7 +533,7 @@ pub fn terminate_recorded_service_pids( } } - entries + let mut stops: Vec = entries .into_iter() .map(|(pid, role, ticks)| { let outcome = if pid == std::process::id() { @@ -506,7 +543,47 @@ pub fn terminate_recorded_service_pids( }; RecordedPidStop { pid, role, outcome } }) - .collect() + .collect(); + settle_roots_that_exited_alone(&mut stops); + stops +} + +/// Whether `stop` is a recorded process that is confirmed gone: stopped, or a +/// root that exited by itself ([`TerminationOutcome::ExitedBeforeTreeKill`]). +/// The stopping process's own PID is not a service process and never counts. +const fn recorded_process_is_gone(stop: RecordedPidStop) -> bool { + match stop.outcome { + RecordedPidOutcome::OwnProcess => false, + RecordedPidOutcome::Terminated(outcome) => { + outcome.stopped() || matches!(outcome, TerminationOutcome::ExitedBeforeTreeKill) + } + } +} + +/// Turn each [`TerminationOutcome::ExitedBeforeTreeKill`] into +/// [`TerminationOutcome::AlreadyGone`] when some other recorded process is +/// confirmed gone, and leave it unconfirmed otherwise. Judged on the verdicts as +/// they came in, so two roots that both exited by themselves corroborate each +/// other. +fn settle_roots_that_exited_alone(stops: &mut [RecordedPidStop]) { + let corroborated: Vec = (0..stops.len()) + .map(|index| { + stops + .iter() + .enumerate() + .any(|(other, stop)| other != index && recorded_process_is_gone(*stop)) + }) + .collect(); + for (stop, corroborated) in stops.iter_mut().zip(corroborated) { + if corroborated + && matches!( + stop.outcome, + RecordedPidOutcome::Terminated(TerminationOutcome::ExitedBeforeTreeKill) + ) + { + stop.outcome = RecordedPidOutcome::Terminated(TerminationOutcome::AlreadyGone); + } + } } /// Terminate one recorded process, and its descendants, after verifying its @@ -591,16 +668,22 @@ enum TreeKill { /// deadline; [`Unverified`](TerminationOutcome::Unverified)) stay as they are. /// /// A kill that found no such process ([`TreeKill::NotFound`]) is judged by the -/// root's own re-read, which `outcome` carries: when the root is gone +/// root's own re-read, which `outcome` carries. When the root is gone /// ([`TerminationOutcome::AlreadyGone`]) it exited on its own between the -/// identity check and the kill, and that is what the stop reports, as for any -/// recorded process that is already gone. When the root is still there, "no such +/// identity check and the kill: that is [`TerminationOutcome::ExitedBeforeTreeKill`], +/// not a stop. `taskkill /T` reaches children only through a live parent, so a +/// child the root left running is out of its reach, and the stop kept no list of +/// what was under the root to check afterwards. The aggregate over the record's +/// processes decides whether another recorded process corroborates the exit +/// ([`terminate_recorded_service_pids`]). When the root is still there, "no such /// process" contradicts what the termination found, so the kill is treated as /// failed. const fn after_tree_kill(tree: TreeKill, outcome: TerminationOutcome) -> TerminationOutcome { match (tree, outcome) { (TreeKill::Completed, outcome) => outcome, - (TreeKill::NotFound, TerminationOutcome::AlreadyGone) => TerminationOutcome::AlreadyGone, + (TreeKill::NotFound, TerminationOutcome::AlreadyGone) => { + TerminationOutcome::ExitedBeforeTreeKill + } ( TreeKill::NotFound | TreeKill::Failed, TerminationOutcome::Graceful @@ -667,10 +750,13 @@ fn hold_pid(identity: &ProcessIdentity) -> PidHold { use std::os::windows::io::{FromRawHandle, OwnedHandle}; use windows_sys::Win32::System::Threading::{OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION}; - // SAFETY: a plain Win32 call; a non-null handle is owned by nothing else and - // is closed when the `OwnedHandle` drops. + // SAFETY: `OpenProcess` takes only plain values and returns a handle or null; + // there is no pointer to keep valid. let handle = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, identity.pid) }; PidHold { + // SAFETY: this closure runs only for a non-null handle, which + // `OpenProcess` just returned and which nothing else owns or closes, so + // the `OwnedHandle` may take ownership and close it exactly once on drop. _handle: (!handle.is_null()).then(|| unsafe { OwnedHandle::from_raw_handle(handle) }), } } @@ -1005,6 +1091,13 @@ mod tests { TerminationOutcome::DescendantsUnconfirmed.as_str(), TerminationOutcome::TimedOut.as_str() ); + // A root that exited by itself before its tree kill: not a stop alone. + assert!(!TerminationOutcome::ExitedBeforeTreeKill.stopped()); + assert!(!TerminationOutcome::ExitedBeforeTreeKill.graceful()); + assert_eq!( + TerminationOutcome::ExitedBeforeTreeKill.as_str(), + "exited_before_tree_kill" + ); // Unverified: neither stopped nor graceful — we could not act. assert!(!TerminationOutcome::Unverified.stopped()); @@ -1362,8 +1455,8 @@ mod tests { #[test] fn managed_stop_classifies_each_outcome_as_signaled_forced_and_stopped() { use TerminationOutcome::{ - AlreadyGone, DescendantsUnconfirmed, Forced, Graceful, IdentityMismatch, TimedOut, - Unverified, + AlreadyGone, DescendantsUnconfirmed, ExitedBeforeTreeKill, Forced, Graceful, + IdentityMismatch, TimedOut, Unverified, }; let record = record_naming(SUPERVISOR_PID, None, None, None); @@ -1377,6 +1470,8 @@ mod tests { (TimedOut, true, true, false), // Signalled and forced by the tree kill, but not confirmed gone. (DescendantsUnconfirmed, true, true, false), + // Nothing signalled: the root was gone before the tree kill ran. + (ExitedBeforeTreeKill, false, false, false), ]; for (outcome, signaled, forced, stopped) in expected { let (_, stops) = stop_with(&record, outcome); @@ -1728,11 +1823,12 @@ mod tests { /// `taskkill` answering "no such process" is judged by the root's own /// re-read. The root that went away by itself between the identity check and - /// the kill is a stop that happened: nothing is left to wait for, and - /// reporting it unconfirmed would keep a record, a stop request and a key for - /// a service that is gone. + /// the kill is not reported as a kill this stop performed, and not as a stop + /// either on its own: `taskkill /T` reaches children only through a live + /// parent, so a child the root left running was out of its reach, and nothing + /// recorded what was under it. The record-level settling decides. #[test] - fn a_tree_kill_that_found_no_process_is_a_stop_when_the_root_is_gone() { + fn a_tree_kill_that_found_no_process_with_the_root_gone_is_not_a_stop_by_itself() { let (verdict, steps) = tree_kill_run_with( true, TreeKill::NotFound, @@ -1742,16 +1838,144 @@ mod tests { assert_eq!( verdict, - TerminationOutcome::AlreadyGone, - "the root exited on its own: the truth, not a kill this stop performed" + TerminationOutcome::ExitedBeforeTreeKill, + "the root exited on its own and nothing says what it left running" ); - assert!(verdict.stopped()); + assert!(!verdict.stopped()); assert!( steps.contains(&Step::Terminate(SUPERVISOR_PID)), "the root is re-read by the termination: {steps:?}" ); } + /// What the record's processes say together about a root that exited by + /// itself. `exits` maps each recorded PID to the verdict `terminate` gives. + fn settled( + record: &crate::ManagedServiceRecord, + exits: &[(u32, TerminationOutcome)], + ) -> Vec { + terminate_recorded_service_pids(record, |identity| { + exits + .iter() + .find(|(pid, _)| *pid == identity.pid) + .map(|(_, outcome)| *outcome) + .expect("a verdict for every recorded pid") + }) + .into_iter() + .map(|stop| match stop.outcome { + RecordedPidOutcome::Terminated(outcome) => outcome, + RecordedPidOutcome::OwnProcess => panic!("not the stopping process"), + }) + .collect() + } + + /// A parent that exits by itself while its child keeps running is not a + /// stop: with only the parent recorded, nothing independently verifies the + /// child, so the stop stays unconfirmed (and the record, the stop request + /// and the key are kept by the callers). This is the case a root's exit + /// cannot settle. + #[test] + fn a_parent_that_exits_alone_with_nothing_else_recorded_is_not_a_confirmed_stop() { + use TerminationOutcome::ExitedBeforeTreeKill; + let record = record_naming(SUPERVISOR_PID, None, None, None); + + let verdicts = settled(&record, &[(SUPERVISOR_PID, ExitedBeforeTreeKill)]); + + assert_eq!(verdicts, vec![ExitedBeforeTreeKill]); + assert!(!verdicts[0].stopped()); + } + + /// The same parent with its engine recorded: the engine is the process under + /// it that is independently verified, by its own entry. Confirmed gone, the + /// stop is confirmed, which keeps the concern that a root exiting on its own + /// must not leave a stop unconfirmed for nothing. + #[test] + fn a_parent_that_exits_alone_is_confirmed_when_the_recorded_engine_is_confirmed_gone() { + use TerminationOutcome::{AlreadyGone, ExitedBeforeTreeKill, Forced, Graceful}; + let record = record_naming(SUPERVISOR_PID, None, Some(ENGINE_PID), None); + + for engine in [Forced, Graceful, AlreadyGone] { + let verdicts = settled( + &record, + &[(SUPERVISOR_PID, ExitedBeforeTreeKill), (ENGINE_PID, engine)], + ); + + assert_eq!(verdicts, vec![AlreadyGone, engine], "engine {engine:?}"); + assert!(verdicts.iter().all(|verdict| verdict.stopped())); + } + // And the other way round: the engine is the root that exited alone. + let verdicts = settled( + &record, + &[ + (SUPERVISOR_PID, Graceful), + (ENGINE_PID, ExitedBeforeTreeKill), + ], + ); + assert_eq!(verdicts, vec![Graceful, AlreadyGone]); + } + + /// A parent that exits while the recorded engine is still running, or not + /// confirmed, stays unconfirmed: the engine is the child that survived. + #[test] + fn a_parent_that_exits_alone_stays_unconfirmed_while_the_recorded_engine_is_not_gone() { + use TerminationOutcome::{ + DescendantsUnconfirmed, ExitedBeforeTreeKill, TimedOut, Unverified, + }; + let record = record_naming(SUPERVISOR_PID, None, Some(ENGINE_PID), None); + + for engine in [TimedOut, Unverified, DescendantsUnconfirmed] { + let verdicts = settled( + &record, + &[(SUPERVISOR_PID, ExitedBeforeTreeKill), (ENGINE_PID, engine)], + ); + + assert_eq!( + verdicts, + vec![ExitedBeforeTreeKill, engine], + "engine {engine:?}" + ); + assert!( + !verdicts[0].stopped(), + "the parent is not settled by {engine:?}" + ); + } + } + + /// Two roots that both exited by themselves corroborate each other: every + /// process the record names is gone. + #[test] + fn two_roots_that_both_exited_alone_are_a_confirmed_stop() { + use TerminationOutcome::{AlreadyGone, ExitedBeforeTreeKill}; + let record = record_naming(SUPERVISOR_PID, None, Some(ENGINE_PID), None); + + let verdicts = settled( + &record, + &[ + (SUPERVISOR_PID, ExitedBeforeTreeKill), + (ENGINE_PID, ExitedBeforeTreeKill), + ], + ); + + assert_eq!(verdicts, vec![AlreadyGone, AlreadyGone]); + } + + /// The stopping process's own PID is not a service process: it never + /// corroborates a root that exited alone. + #[test] + fn the_stopping_process_does_not_corroborate_a_root_that_exited_alone() { + use TerminationOutcome::ExitedBeforeTreeKill; + let record = record_naming(SUPERVISOR_PID, None, Some(std::process::id()), None); + + let stops = terminate_recorded_service_pids(&record, |_| ExitedBeforeTreeKill); + + assert_eq!( + stops[0].outcome, + RecordedPidOutcome::Terminated(ExitedBeforeTreeKill) + ); + assert_eq!(stops[1].outcome, RecordedPidOutcome::OwnProcess); + assert!(!stops[0].stopped()); + } + /// "No such process" while the root is still there is a contradiction, so it /// is a failed kill and the stop stays unconfirmed, as for any other nonzero /// exit. diff --git a/crates/rocm-core/src/process.rs b/crates/rocm-core/src/process.rs index 54ec90cb1c..80114ef259 100644 --- a/crates/rocm-core/src/process.rs +++ b/crates/rocm-core/src/process.rs @@ -387,8 +387,10 @@ pub enum ProcessProbe { /// may be serving), and there is no better evidence to be had: the creation /// time that would tell the two processes apart is read with the same access /// right, so a process that cannot be opened has no readable token to - /// compare. The record can be cleared by hand; see `docs/manual-testing.md`, - /// "A record that stays unverified". + /// compare. `rocm services remove --force --yes` clears such a record + /// (see `docs/manual-testing.md`, "A record that stays unverified"); it + /// applies only when no recorded process is confirmed running, one could not + /// be queried, and a stop was requested. Unqueryable, } @@ -440,6 +442,23 @@ fn system_tool_path(system_root: Option, name: &str) -> Path } } +/// E2E hook: `ROCM_E2E_UNQUERYABLE_PIDS`, a comma-separated list of PIDs, makes +/// [`probe_process`] answer [`ProcessProbe::Unqueryable`] for each of them. +/// +/// A live process this account may not open cannot be staged from outside the +/// binary, and without it the stop, the liveness refresh and `services remove` +/// have no live coverage for that case off Windows. Compiled only into the +/// `e2e-test-hooks` build; production builds never read the variable. +#[cfg(feature = "e2e-test-hooks")] +fn scripted_probe(pid: u32) -> Option { + let listed = std::env::var("ROCM_E2E_UNQUERYABLE_PIDS").ok()?; + listed + .split(',') + .filter_map(|entry| entry.trim().parse::().ok()) + .any(|listed| listed == pid) + .then_some(ProcessProbe::Unqueryable) +} + /// Look at `pid`, keeping a process that cannot be queried apart from one that /// is gone. See [`ProcessProbe`]. #[cfg(windows)] @@ -448,15 +467,24 @@ fn system_tool_path(system_root: Option, name: &str) -> Path pub fn probe_process(pid: u32) -> ProcessProbe { use windows_sys::Win32::Foundation::CloseHandle; + #[cfg(feature = "e2e-test-hooks")] + if let Some(scripted) = scripted_probe(pid) { + return scripted; + } if pid == 0 { return ProcessProbe::Gone; } + // SAFETY: `OpenProcess` takes only plain values and returns a handle or null. let handle = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid) }; if handle.is_null() { return classify_open_failure(std::io::Error::last_os_error().raw_os_error()); } let mut exit_code = 0; + // SAFETY: `handle` is the non-null handle opened above, still open, and + // `exit_code` is a live local that is valid for this one call to write to. let queried = unsafe { GetExitCodeProcess(handle, &raw mut exit_code) != 0 }; + // SAFETY: `handle` is the handle opened above, closed here exactly once and + // not used afterwards. unsafe { CloseHandle(handle); } @@ -474,6 +502,10 @@ pub fn probe_process(pid: u32) -> ProcessProbe { #[cfg(not(windows))] #[must_use] pub fn probe_process(pid: u32) -> ProcessProbe { + #[cfg(feature = "e2e-test-hooks")] + if let Some(scripted) = scripted_probe(pid) { + return scripted; + } if process_is_running(pid) { ProcessProbe::Running } else { @@ -488,23 +520,8 @@ pub fn probe_process(pid: u32) -> ProcessProbe { /// stopped, deleting it) asks [`probe_process`] instead and keeps /// [`ProcessProbe::Unqueryable`] apart. #[cfg(windows)] -#[allow(unsafe_code)] // Win32 FFI pub fn process_is_running(pid: u32) -> bool { - use windows_sys::Win32::Foundation::CloseHandle; - - if pid == 0 { - return false; - } - let handle = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid) }; - if handle.is_null() { - return false; - } - let mut exit_code = 0; - let ok = unsafe { GetExitCodeProcess(handle, &raw mut exit_code) != 0 }; - unsafe { - CloseHandle(handle); - } - ok && exit_code == 259 + probe_process(pid) == ProcessProbe::Running } #[cfg(not(windows))] diff --git a/docs/architecture.md b/docs/architecture.md index 3debb6d00e..4f0fa13221 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -94,7 +94,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`serve_port.rs`](../apps/rocm/src/serve_port.rs) — full domain extraction: the central `rocm serve` port policy (automatic vs explicit request, reservation by live managed services, OS preflight lease), used by [`service_launch.rs`](../apps/rocm/src/service_launch.rs); the CLI restart path shares the `rocm-core` lease primitives (`lease_loopback_port_or_bind_error`, `is_canonical_loopback_host`) rather than this module. - [`service_hints.rs`](../apps/rocm/src/service_hints.rs) — shared CLI-output component: the `rocm services stop|restart|remove --yes` hint text, shared by [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs), [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs), [`service_launch.rs`](../apps/rocm/src/service_launch.rs), [`service_records.rs`](../apps/rocm/src/service_records.rs), [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs), and [`services_text.rs`](../apps/rocm/src/services_text.rs) so they cannot go out of sync with each other, as `serve_summary.rs`'s stop row did from the day it was added — #417. It also holds the helpers that map a service tool name to the command word, verb and past tense those hints and messages use (`service_action_command`, `service_action_verb`, `service_action_past_tense`) and to its `SandboxToolArg` (`sandbox_tool_arg_from_service_tool`); [`services_text.rs`](../apps/rocm/src/services_text.rs) reads `service_action_past_tense` from here rather than through the crate root. It quotes the service id with `shell_quote`, so a hint names exactly the argv it runs. - [`service_launch.rs`](../apps/rocm/src/service_launch.rs) — mechanical relocation, owns types (Phase 6b, ROCMAI-91): spawning/attaching to a managed service process and streaming its log, the port allocation transaction and failed-launch cleanup (under the shared launch lock), plus the endpoint-key/auth checks that gate a public bind — `AttachOutcome`/`RawModeGuard`/`ManagedSpawn`/`AutostartClaim` are its private types; `ManagedLaunchReport` is `pub(crate)`, since [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) reads its fields directly for the interactive summary. Most of this module's functions are `pub(crate)` for the same reason — other modules, including [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`automations.rs`](../apps/rocm/src/automations.rs) and [`service_control.rs`](../apps/rocm/src/service_control.rs), call into it; `main.rs` does not. `ServicesCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs) rather than moving alongside, and `services()` lives in [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs); the record removal and pruning it dispatches to now lives in [`service_records.rs`](../apps/rocm/src/service_records.rs), and only stop/restart are also reachable from the chat/LLM sandbox tool-calling path, out of scope for Phase 6b. This module does not import [`service_control.rs`](../apps/rocm/src/service_control.rs), which imports it: `run_attached_service` takes the stop for a Ctrl-C as a parameter, and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) passes `run_internal_sandbox_tool`. -- [`service_records.rs`](../apps/rocm/src/service_records.rs) — mechanical relocation, owns types: removing one managed-service record and every file it owns (`remove_managed_service_record`, `service_record_artifacts`) and the `rocm services prune` plan/render/apply steps (`build_service_prune_plan`, `render_service_prune_plan`, `apply_service_prune_plan`, `prune_managed_service_records`) with the age gate behind them (`DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS`, `service_prune_min_age_hours`). It reads record state through [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) and [`services_text.rs`](../apps/rocm/src/services_text.rs) and builds its advice text with [`service_hints.rs`](../apps/rocm/src/service_hints.rs); none of the three depends on it. `remove_managed_service_record`, `prune_managed_service_records`, `service_prune_min_age_hours` and `DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS` are `pub(crate)` (`services()` in [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs) calls the first three; the `ServicesCommand::Prune` clap definition in [`cli.rs`](../apps/rocm/src/cli.rs) uses the constant), and `ServicePruneOutcome`, the return type of `prune_managed_service_records`, is `pub(crate)` with `pub(crate)` `text`, `removed_records`, `removed_files` and `failures` fields that `services()` reads. Every other item is private. The cluster's regression tests live in this module's own `mod tests`, reaching the shared `test_paths`, `plant_service_record` and `plant_service_record_without_its_key` fixtures through `crate::tests::` (the last two widened to `pub(crate)` for that). +- [`service_records.rs`](../apps/rocm/src/service_records.rs) — mechanical relocation, owns types: removing one managed-service record and every file it owns (`remove_managed_service_record`, `service_record_artifacts`) and the `rocm services prune` plan/render/apply steps (`build_service_prune_plan`, `render_service_prune_plan`, `apply_service_prune_plan`, `prune_managed_service_records`) with the age gate behind them (`DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS`, `service_prune_min_age_hours`). It reads record state through [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) and [`services_text.rs`](../apps/rocm/src/services_text.rs) and builds its advice text with [`service_hints.rs`](../apps/rocm/src/service_hints.rs); none of the three depends on it. `remove_managed_service_record_with` takes `--force` and the read of each recorded PID: `--force` removes a record only when none of its recorded processes is confirmed running, one could not be queried and a stop was requested, and reports which. `remove_managed_service_record_with`, `prune_managed_service_records`, `service_prune_min_age_hours` and `DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS` are `pub(crate)` (`services()` in [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs) calls the first three; the `ServicesCommand::Prune` clap definition in [`cli.rs`](../apps/rocm/src/cli.rs) uses the constant), and `ServicePruneOutcome`, the return type of `prune_managed_service_records`, is `pub(crate)` with `pub(crate)` `text`, `removed_records`, `removed_files` and `failures` fields that `services()` reads. Every other item is private. The cluster's regression tests live in this module's own `mod tests`, reaching the shared `test_paths`, `plant_service_record` and `plant_service_record_without_its_key` fixtures through `crate::tests::` (the last two widened to `pub(crate)` for that). - [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm services`' command handler (`services()`, which dispatches `list`, `logs`, `stop`, `restart`, `remove` and `prune`) and `run_approved_service_action`, the approval-gated stop/restart path. `ServicesCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs). Both functions are `pub(crate)`: `dispatch()` calls `services()` through a re-import, and [`advised_commands.rs`](../apps/rocm/src/advised_commands.rs)'s test reaches `run_approved_service_action` through a test-only re-import in `main.rs`. The module reads from [`services_text.rs`](../apps/rocm/src/services_text.rs), [`service_control.rs`](../apps/rocm/src/service_control.rs), [`service_records.rs`](../apps/rocm/src/service_records.rs), [`service_hints.rs`](../apps/rocm/src/service_hints.rs) and [`cli_audit.rs`](../apps/rocm/src/cli_audit.rs); none of them calls back into it. The cluster's one regression test lives in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::`. - [`services_text.rs`](../apps/rocm/src/services_text.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-209): the CLI/TUI logs browser (`render_logs_text`, `render_logs_browser_text`, `render_logs_browser_page_text`, `render_logs_browser_page_text_with_options`, `logs_browser_total_pages`, the `LogBrowserEntry` type, `collect_log_browser_entries`, `filter_log_browser_entries`, `log_browser_source_label`, `write_past_attempts_hint`), the `rocm services`/`rocm services logs` status surfaces (`render_services_text`, `render_services_json`, `render_service_logs_text`, `render_service_logs_text_with_options`, `render_service_action_result`, `read_managed_service_manifest`, `load_managed_service`), the `--follow` tail behind `rocm logs --service --follow` and `rocm services logs --follow` (`follow_service_logs`, `follow_service_logs_with`, the `LogTail`, `LogSegment` and `FollowFooter` types, `write_log_segments`, `newly_ready`; new code written in this module, not relocated), the internal `rocmd`-compatible status surface (`render_internal_status_text`), the `stop_server` answer shape and its renderer (`internal_stop_server_value`, `service_stop_unconfirmed`, `render_unconfirmed_service_stop`, `process_check_command`), plus `validate_service_id`, `list_log_files`, `read_optional_tail_lines`, and the CLI-lifecycle-log tail formatting helpers (`format_cli_lifecycle_tail_line`, `lifecycle_event_label`, `humanize_log_label`, `lifecycle_field`, `read_tail_lines`). The managed-service stop and restart implementation that used to live here is in [`service_control.rs`](../apps/rocm/src/service_control.rs). The `LogBrowserEntry` struct moved here from `main.rs` with the logs browser and stays private. `dispatch()` calls directly into this module at several points — `render_internal_status_text`, `render_service_logs_text`, `render_logs_browser_text`, and `render_logs_text` — all re-imported via `use crate::services_text::{...};`; [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs)'s `services()` calls `render_services_text`, `render_services_json` and `render_service_logs_text` through its own imports. `render_service_action_result`, `service_stop_unconfirmed` and `validate_service_id` are also `pub(crate)` and read from `services_cmd.rs`'s `run_approved_service_action`; `validate_service_id` is also read from [`service_records.rs`](../apps/rocm/src/service_records.rs)'s `remove_managed_service_record`/`service_record_artifacts`, and `load_managed_service` from its `remove_managed_service_record`/`apply_service_prune_plan`. `load_managed_service` is `pub(crate)`, read from [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) (reconciling live servers against a runtime activation), from [`service_control.rs`](../apps/rocm/src/service_control.rs) and from [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs)'s own test module; `read_managed_service_manifest` and `internal_stop_server_value` are `pub(crate)` for `service_control.rs`, and `process_check_command` for its tests; `read_tail_lines`, `render_logs_text`, `render_logs_browser_text`, `render_services_text`, `render_service_logs_text`, and `validate_service_id` are also `pub(crate)`, read from [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs)'s read-only command runner; `read_optional_tail_lines` and `render_service_action_result` are also `pub(crate)`, read from [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s managed-spawn/attach path. Every other item in this list has no caller outside this module and stays private. The cluster's regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` the same way other mechanically relocated modules do; `assert_no_retry_line` stays here as a `pub(crate)` test helper because it checks the unconfirmed-stop rendering and `service_launch.rs`'s tests read it too. Two tests that call this module's `load_managed_service` only incidentally — their primary subject is `runtime_services::pin_service_record_to_runtime`/`restore_service_record_pin` and `runtime_services::restart_service_onto_runtime_with`'s outcome classification respectively — stay in `main.rs`'s shared `mod tests` instead. - [`setup_cmd.rs`](../apps/rocm/src/setup_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm setup`'s command handler (`setup()`, which runs `setup status` by default and `setup reset`), the text `rocm setup status` prints (`render_setup_status_text`) and the reset that clears the saved first-time prompt state (`reset_setup_prompt_state`). `SetupCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs). `setup()` and `render_setup_status_text` are `pub(crate)` and re-imported in `main.rs`: `dispatch()` calls the first, and [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs)'s read-only handling of `rocm setup` and `rocm setup status` reaches the second as `crate::render_setup_status_text`; `reset_setup_prompt_state` is private. The three regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` and `write_test_pip_runtime` through `crate::test_support::`. @@ -238,7 +238,7 @@ Further `rocm-core` modules, each with its own doc comment except where noted: - [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) — OpenMPI, libatomic and libnuma detection, the `libmpi_cxx` compatibility shim, and the per-library install plans (`build_openmpi_install_plan`, `build_libatomic_install_plan`, `build_libnuma_install_plan`) and hints, needed by vLLM's ROCm wheels. The distro-generic half of that planning lives in [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs); [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) re-exports its public items, so `rocm_core::openmpi::running_as_root`, `plan_for_os_release_read` and the rest keep resolving, and `pub(crate)` re-exports `PackageManager` and `resolve_package_manager` for [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs). - [`os_release.rs`](../crates/rocm-core/src/os_release.rs) — the one `/etc/os-release` parser every reader of that file uses. - [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs) — the planning half of a system-package install that does not depend on which library is installed: `PackageManager` and `resolve_package_manager` (from an os-release `ID`/`ID_LIKE`), `SystemPackageInstallPlan` and `InstallCommand`, `plan_for_os_release_at`/`plan_for_os_release_read` (which take the per-library builder as an argument), `InstallHint`, the root and passwordless-`sudo` checks (`running_as_root`, `can_autoinstall`) and the `apt-get -s` helpers (`apt_simulate_argv`, `parse_apt_simulate_removals`, `is_protected_rocm_package`). It never runs anything and depends on no other module of this crate except [`os_release.rs`](../crates/rocm-core/src/os_release.rs) and [`hardware_root.rs`](../crates/rocm-core/src/hardware_root.rs), which it reads os-release through. The module is private; its public items are re-exported from [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs). It is a different module from [`system_packages.rs`](../apps/rocm/src/system_packages.rs) in `apps/rocm`, which runs these plans with consent and `sudo`. -- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches (`terminate_verified`), and the one managed-stop routine built on it that `rocm services stop` and `rocmd`'s `stop_server` both call: `terminate_recorded_service_pids` walks the processes a service record names (supervisor, then engine, each with its own start-time, the stopping process's own PID reported as `self` and never signalled) and returns one `RecordedPidStop` per PID (each caller, on a stop that confirms every one gone, then clears the record's recorded processes with `ManagedServiceRecord::clear_recorded_processes` in [`service_record.rs`](../crates/rocm-core/src/service_record.rs), so a repeated stop reaches no stale PID; an unconfirmed stop keeps them); `terminate_recorded_pid` terminates one of them, adding on Windows a `taskkill /T /F` of its descendants first, because `terminate_verified` cannot walk a process tree there, and only for a PID that passes the same identity check (`process_start_ticks` is the kernel start-time on Linux and the process creation time on Windows, so a recycled PID is refused before any `taskkill`; a record written before Windows recorded one cannot be refuted), holding the PID open on Windows from before that check until the stop ends so the PID cannot be reused in between, and treating a tree kill that did not complete, or did not finish within `MANAGED_STOP_GRACE` (the `taskkill` is killed and reaped at that deadline), as an unconfirmed stop (`descendants_unconfirmed` when the root itself was seen to exit, `timed_out` only when it outlived its deadline), except that a `taskkill` which found no such process is judged by the root's own re-read, so a root that exited by itself is a stop, and starting that `taskkill` from `%SystemRoot%\System32` through `windows_system_tool`; a PID that cannot be queried at all, as against one that is gone, reads as unverifiable unless a readable recorded token differs, in which case that mismatch wins (`ProcessProbe` and `probe_process` in [`process.rs`](../crates/rocm-core/src/process.rs), whose `may_be_running` is the answer for any decision that must not take an unqueryable process for gone); `MANAGED_STOP_GRACE` is the one grace period. The termination is injectable, so a caller's test supplies outcomes it cannot construct and a test of the Windows step runs on every host. +- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches (`terminate_verified`), and the one managed-stop routine built on it that `rocm services stop` and `rocmd`'s `stop_server` both call: `terminate_recorded_service_pids` walks the processes a service record names (supervisor, then engine, each with its own start-time, the stopping process's own PID reported as `self` and never signalled) and returns one `RecordedPidStop` per PID (each caller, on a stop that confirms every one gone, then clears the record's recorded processes with `ManagedServiceRecord::clear_recorded_processes` in [`service_record.rs`](../crates/rocm-core/src/service_record.rs), so a repeated stop reaches no stale PID; an unconfirmed stop keeps them); `terminate_recorded_pid` terminates one of them, adding on Windows a `taskkill /T /F` of its descendants first, because `terminate_verified` cannot walk a process tree there, and only for a PID that passes the same identity check (`process_start_ticks` is the kernel start-time on Linux and the process creation time on Windows, so a recycled PID is refused before any `taskkill`; a record written before Windows recorded one cannot be refuted), holding the PID open on Windows from before that check until the stop ends so the PID cannot be reused in between, and treating a tree kill that did not complete, or did not finish within `MANAGED_STOP_GRACE` (the `taskkill` is killed and reaped at that deadline), as an unconfirmed stop (`descendants_unconfirmed` when the root itself was seen to exit, `timed_out` only when it outlived its deadline), except that a `taskkill` which found no such process is judged by the root's own re-read, so a root that exited by itself is `exited_before_tree_kill`, which the record-wide settling in `terminate_recorded_service_pids` turns into a stop only when another recorded process (the engine, which is visited as its own role and folded by both callers) is confirmed gone, and starting that `taskkill` from `%SystemRoot%\System32` through `windows_system_tool`; a PID that cannot be queried at all, as against one that is gone, reads as unverifiable unless a readable recorded token differs, in which case that mismatch wins (`ProcessProbe` and `probe_process` in [`process.rs`](../crates/rocm-core/src/process.rs), whose `may_be_running` is the answer for any decision that must not take an unqueryable process for gone); `MANAGED_STOP_GRACE` is the one grace period. The termination is injectable, so a caller's test supplies outcomes it cannot construct and a test of the Windows step runs on every host. - [`report.rs`](../crates/rocm-core/src/report.rs) — the content of a Doctor report and the rule deciding whether one may exist; it sends nothing. - [`report_delivery.rs`](../crates/rocm-core/src/report_delivery.rs) — how a report leaves the machine: building its link and deciding whether a browser may be started. - [`runtime.rs`](../crates/rocm-core/src/runtime.rs) — the host-platform layer: `RuntimePlatform`/`RuntimeHost`, OS-specific executable and Python-environment names (including `runtime_python_env_root_for_executable`, the one rule for finding an environment's folder from its interpreter's path: only an interpreter directly inside a `bin` or `Scripts` folder, in any case, has one, and everything else gets `None`; the adopt command, the ComfyUI runtime environment and the vLLM launch all call it rather than keeping a copy, since the engine crate cannot reach `apps/rocm`), default config/data/cache directories, path-text and path-list helpers, and `install_root_has_payload`, the one definition of a non-empty install root (dot-entries such as the runtime marker do not count; a path that does not exist holds nothing; a root that cannot be listed is an error) that tarball extraction, update repair detection and activation validation in `apps/rocm` all ask (no module doc comment). diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 3e0ade16bb..56a9cb34c7 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -505,12 +505,14 @@ rocm services restart --yes `stop` prints `Local server stopped` only once every recorded process is confirmed gone; otherwise it prints `Local server stop not confirmed` on stderr, keeps the record and any endpoint key, and exits non-zero. It names each -process that, or a process under it, did not exit (on Windows also one that did -exit while taking the processes under it down did not complete, so one of them -may still be running: `not confirmed: process () exited, but a -process under it may still be running`), and for one it could not -verify, an operating-system command to check on it. It never prints a `retry:` -line. +process that, or a process under it, did not exit. On Windows it also names one +that did exit while taking the processes under it down did not complete +(`not confirmed: process () exited, but a process under it may still +be running`), and one that was already gone when that step ran with no other +recorded process confirmed gone (`not confirmed: process () was +already gone when the processes under it were to be taken down`). For one it +could not verify, it names an operating-system command to check on it. It never +prints a `retry:` line. `restart` starts the server again only once its stop has confirmed every recorded process gone; otherwise it reports that and starts nothing. @@ -541,12 +543,15 @@ possibly running, which is the safe direction: the record never reconciles to 1. Run the check command the stop printed. If the process is this server's, end it yourself (from an elevated prompt if the account may not), then run `rocm services`; the record reconciles and its endpoint key is dropped. -2. If it is an unrelated process, `rocm` has no command that clears the record. - Delete the record's `.json` by hand from the folder that - `rocm storage report` lists under `local server records`, then run - `rocm services prune --any-age --yes`, which sweeps the log, the endpoint key - and the engine state file that are left with no record. The server can no - longer be restarted afterwards. +2. If it is an unrelated process, run `rocm services remove --force + --yes`. The refusal of a plain `remove` names exactly this command. `--force` + applies only when a stop was requested (the stop that reported `unverified` + did that), no recorded process is confirmed running, and at least one could + not be queried; a process `rocm` can see running is never forced past. The + command says which process could not be queried and that, if it is still + running, it keeps running, and it deletes the record, its log, its engine + state and its endpoint key. The server can no longer be restarted afterwards. + `rocm services prune` has no `--force`: it leaves such a record alone. The rest of this section needs a **vLLM** server running, on Linux or WSL, and two runtimes to move it between. Nothing earlier in this document sets either diff --git a/docs/testing.md b/docs/testing.md index a41b4a0c9f..a7ebf7df54 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2005,12 +2005,25 @@ request and key that are kept (`a_tree_kill_that_did_not_complete_leaves_the_stop_unconfirmed`, `only_a_clean_taskkill_exit_is_a_completed_tree_kill_and_128_is_not_found`). A `taskkill` that exits 128 (process not found) is judged by the root's own re-read: -with the root gone it exited by itself and the stop is reported as for any -recorded process already gone, so the record is not kept for nothing; with the -root still there the kill counts as failed -(`a_tree_kill_that_found_no_process_is_a_stop_when_the_root_is_gone`, -`a_tree_kill_that_found_no_process_while_the_root_runs_is_a_failed_kill`, -`a_failed_tree_kill_with_the_root_gone_is_still_unconfirmed`). That `taskkill` runs under +with the root still there the kill counts as failed +(`a_tree_kill_that_found_no_process_while_the_root_runs_is_a_failed_kill`); with +the root gone it exited by itself, and the outcome is `exited_before_tree_kill`, +not a stop on its own, because `taskkill /T` reaches children only through a live +parent and nothing recorded what was under the root +(`a_tree_kill_that_found_no_process_with_the_root_gone_is_not_a_stop_by_itself`). +Across the record, such a root becomes `already_gone`, and the stop confirmed, +only when another recorded process is confirmed gone (`a_parent_that_exits_alone_is_confirmed_when_the_recorded_engine_is_confirmed_gone`, +`two_roots_that_both_exited_alone_are_a_confirmed_stop`); with nothing else +recorded, or the engine still running or unconfirmed, it stays unconfirmed +(`a_parent_that_exits_alone_with_nothing_else_recorded_is_not_a_confirmed_stop`, +`a_parent_that_exits_alone_stays_unconfirmed_while_the_recorded_engine_is_not_gone`, +`the_stopping_process_does_not_corroborate_a_root_that_exited_alone`). Both +callers fold every row, so the record, PIDs and key are kept unless every +recorded role is confirmed +(`a_root_that_exited_alone_confirms_the_stop_only_when_the_engine_role_does` in +`rocm` and in `rocmd`, `a_lone_root_that_exited_alone_leaves_the_stop_unconfirmed` +in both). A failed tree kill with the root gone stays unconfirmed +(`a_failed_tree_kill_with_the_root_gone_is_still_unconfirmed`). That `taskkill` runs under a deadline of `MANAGED_STOP_GRACE`: one that stalls is killed and reaped and counts as a failed tree kill, so it cannot hang `rocm services stop`, `restart` or the attached Ctrl-C stop @@ -2028,8 +2041,17 @@ recorded token that differs outranks a failed query (`a_readable_token_mismatch_outranks_a_failed_query`), but the creation time and the query are read with the same access right, so on Windows this only reaches a process that opens and then fails the exit-code query; a process that cannot be -opened has no readable token, stays unverifiable, and is cleared by hand (see -"A record that stays unverified" in [manual-testing.md](manual-testing.md)). What no +opened has no readable token, stays unverifiable, and is cleared with +`rocm services remove --force --yes` (see "A record that stays unverified" +in [manual-testing.md](manual-testing.md); the unit tests +`force_removes_a_record_held_by_an_unqueryable_process_and_says_so`, +`a_record_held_by_an_unqueryable_process_is_refused_naming_the_forced_removal`, +`force_is_refused_for_a_running_process_and_without_a_requested_stop` and +`force_on_a_record_that_is_not_running_is_an_ordinary_removal` cover it with the +probe injected, and `service-unqueryable-01` and `-02` drive the stop, the +refusal naming `--force`, and the forced removal through the real binary, with +the `e2e-test-hooks` variable `ROCM_E2E_UNQUERYABLE_PIDS` making the listed PIDs +read as unqueryable; `-02` is Linux-only because it starts a real process). What no test here reaches is the Windows API itself: the handle hold, the open-failure codes and a real `taskkill` failing run only on the Windows lane. @@ -2058,7 +2080,7 @@ that will read it exists. `a_stop_reports_its_own_pid_as_self` and Third, the `check:` line an unverified stop prints is a diagnostic, not a remediation: it shows whether the process is still running, and what clears the condition is that process ending (or, for a record held by a process `rocm` -cannot open, the hand-clearing in `manual-testing.md`), after which a stop or a +cannot open, `rocm services remove --force --yes`), after which a stop or a service listing confirms and drops the key. The command is a stock one the host has, pinned by `the_process_check_command_is_the_one_the_host_has`, and `service-stop-07` uses the same `tasklist /FI "PID eq "` filter (in CSV diff --git a/tests/e2e-cucumber/features/service_record_unqueryable.feature b/tests/e2e-cucumber/features/service_record_unqueryable.feature new file mode 100644 index 0000000000..b006469d16 --- /dev/null +++ b/tests/e2e-cucumber/features/service_record_unqueryable.feature @@ -0,0 +1,42 @@ +Feature: A local server record held by a process rocm cannot query + + # On Windows a live process the account may not open cannot be queried, so + # `rocm` cannot tell it from a dead server whose PID another program took. + # It treats that as possibly running: a stop reports it unverified, and + # `rocm services remove` refuses the record. `remove --force --yes` is the way + # out for exactly that case, and these scenarios drive the whole path through + # the real binary rather than the pieces. + # + # An unqueryable process cannot be staged from outside the binary, so the + # `e2e-test-hooks` build reads ROCM_E2E_UNQUERYABLE_PIDS and answers + # "cannot be queried" for the PIDs it lists; production builds never read it. + # The records are planted as plain JSON, so no scenario needs a GPU. + # + # A new feature file, as `service_record_cleanup.feature` explains for its + # own: indexes must be sequential per file, and several open branches claim the + # next ones in the files that exist. + + @id:service-unqueryable-remove-force-clears-an-unverified-record + Scenario: service-unqueryable-01 - A record whose process cannot be queried is stopped as unverified, refused by remove, and cleared by remove --force + Given a local server record whose recorded process cannot be queried + When the user stops that local server while its process cannot be queried + Then the CLI reports the stop as not confirmed because the process could not be queried + And every file belonging to that record is still there + When the user removes that local server record while its process cannot be queried + Then the CLI refuses the removal and names the forced removal + And every file belonging to that record is still there + When the user force-removes that local server record while its process cannot be queried + Then the CLI removes the record and says its process could not be queried + And every file belonging to that record is gone + + # `--force` is for a process `rocm` cannot query, never for one it can see + # running. The recorded process here is a real one the scenario started, and a + # stop was requested on the record, so the only thing standing in the way is + # that the process is running. + @id:service-unqueryable-force-never-passes-a-running-process + @requires-os:linux + Scenario: service-unqueryable-02 - --force is refused for a record whose process is confirmed running + Given a local server record whose recorded process is running and whose stop was requested + When the user force-removes that local server record + Then the CLI refuses the forced removal because the record does not qualify + And every file belonging to that record is still there diff --git a/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs b/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs index e1bdddbfb3..ec02018e70 100644 --- a/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs @@ -696,3 +696,179 @@ fn assert_succeeded(world: &E2eWorld) { combined_output(world) ); } + +// ── A record held by a process `rocm` cannot query ───────────────── + +/// A PID no host is using, which the `e2e-test-hooks` build is told to read as +/// "cannot be queried" (`ROCM_E2E_UNQUERYABLE_PIDS`). +const UNQUERYABLE_PID: u32 = 2_000_000_001; + +/// Rewrite fields of the planted manifest. +fn set_manifest_fields(world: &E2eWorld, fields: &[(&str, serde_json::Value)]) { + let manifest = services_dir(world).join(format!("{SERVICE_ID}.json")); + let mut record: serde_json::Value = serde_json::from_slice( + &std::fs::read(&manifest).expect("failed to read the planted manifest"), + ) + .expect("failed to parse the planted manifest"); + for (key, value) in fields { + record[*key] = value.clone(); + } + std::fs::write( + &manifest, + serde_json::to_vec_pretty(&record).expect("failed to serialize record"), + ) + .expect("failed to rewrite the planted manifest"); +} + +/// Make the next `rocm` invocation read [`UNQUERYABLE_PID`] as unqueryable. +fn script_unqueryable(world: &mut E2eWorld) { + world.command_env.push(( + "ROCM_E2E_UNQUERYABLE_PIDS", + UNQUERYABLE_PID.to_string().into(), + )); +} + +#[given("a local server record whose recorded process cannot be queried")] +async fn record_with_unqueryable_process(world: &mut E2eWorld) { + plant_record(world, "ready"); + set_manifest_fields( + world, + &[("supervisor_pid", serde_json::json!(UNQUERYABLE_PID))], + ); +} + +#[given("a local server record whose recorded process is running and whose stop was requested")] +async fn record_with_running_process(world: &mut E2eWorld) { + plant_record(world, "ready"); + // A real process the scenario owns, so the teardown (which stops every + // record left behind) ends it rather than the harness. + let child = std::process::Command::new("sleep") + .arg("300") + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .expect("failed to start the recorded process"); + set_manifest_fields( + world, + &[ + ("supervisor_pid", serde_json::json!(child.id())), + ("stop_requested_unix_ms", serde_json::json!(1)), + ], + ); + world.recorded_process = Some(child); +} + +#[when("the user stops that local server while its process cannot be queried")] +async fn stop_unqueryable(world: &mut E2eWorld) { + script_unqueryable(world); + let (stdout, stderr, rc) = + crate::run_rocm_with_scenario_env(world, &["services", "stop", SERVICE_ID, "--yes"]); + record(world, stdout, stderr, rc); +} + +#[when("the user removes that local server record while its process cannot be queried")] +async fn remove_unqueryable(world: &mut E2eWorld) { + script_unqueryable(world); + let (stdout, stderr, rc) = + crate::run_rocm_with_scenario_env(world, &["services", "remove", SERVICE_ID, "--yes"]); + record(world, stdout, stderr, rc); +} + +#[when("the user force-removes that local server record while its process cannot be queried")] +async fn force_remove_unqueryable(world: &mut E2eWorld) { + script_unqueryable(world); + let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env( + world, + &["services", "remove", SERVICE_ID, "--force", "--yes"], + ); + record(world, stdout, stderr, rc); +} + +#[when("the user force-removes that local server record")] +async fn force_remove(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm( + world, + &["services", "remove", SERVICE_ID, "--force", "--yes"], + ); + record(world, stdout, stderr, rc); +} + +#[then("the CLI reports the stop as not confirmed because the process could not be queried")] +async fn stop_not_confirmed_unverified(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no command rc recorded"); + let combined = combined_output(world); + assert_ne!(rc, 0, "an unverified stop must exit non-zero:\n{combined}"); + assert!( + combined.contains("Local server stop not confirmed"), + "{combined}" + ); + assert!( + combined.contains(&format!("not signaled: process {UNQUERYABLE_PID}")), + "{combined}" + ); + // The check the stop names is the host's own command, and names the PID. + assert!( + combined.contains(&format!("ps -p {UNQUERYABLE_PID} -o pid,args")) + || combined.contains(&format!("tasklist /FI \"PID eq {UNQUERYABLE_PID}\"")), + "{combined}" + ); + // What the stop says it kept is what is on disk: the record names the + // process and the stop request is recorded. + let manifest: serde_json::Value = serde_json::from_slice( + &std::fs::read(services_dir(world).join(format!("{SERVICE_ID}.json"))) + .expect("the record must still be there"), + ) + .expect("the record must parse"); + assert_eq!(manifest["supervisor_pid"], UNQUERYABLE_PID, "{manifest}"); + assert!(manifest["stop_requested_unix_ms"].is_u64(), "{manifest}"); +} + +#[then("the CLI refuses the removal and names the forced removal")] +async fn refuses_and_names_force(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no command rc recorded"); + let combined = combined_output(world); + assert_ne!(rc, 0, "{combined}"); + assert!( + combined.contains(&format!( + "process {UNQUERYABLE_PID} it recorded could not be queried" + )), + "{combined}" + ); + // The command the refusal names is the one the next step runs. + assert!( + combined.contains(&format!("rocm services remove {SERVICE_ID} --force --yes")), + "{combined}" + ); +} + +#[then("the CLI removes the record and says its process could not be queried")] +async fn force_removed(world: &mut E2eWorld) { + assert_succeeded(world); + let combined = combined_output(world); + assert!( + combined.contains(&format!( + "--force: process {UNQUERYABLE_PID} could not be queried" + )), + "{combined}" + ); + assert!( + combined.contains("Local server record removed"), + "{combined}" + ); +} + +#[then("the CLI refuses the forced removal because the record does not qualify")] +async fn refuses_force(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no command rc recorded"); + let combined = combined_output(world); + assert_ne!(rc, 0, "{combined}"); + assert!( + combined.contains("--force removes a record only when"), + "{combined}" + ); + assert!( + combined.contains(&format!("rocm services stop {SERVICE_ID} --yes")), + "{combined}" + ); +} diff --git a/tests/e2e-cucumber/tests/feature_naming.rs b/tests/e2e-cucumber/tests/feature_naming.rs index 230f791c8b..519ee335b5 100644 --- a/tests/e2e-cucumber/tests/feature_naming.rs +++ b/tests/e2e-cucumber/tests/feature_naming.rs @@ -50,6 +50,7 @@ const FEATURE_KEYS: &[(&str, &str)] = &[ ("runtime_lifecycle.feature", "runtime-lifecycle"), ("runtime_setup.feature", "runtime"), ("service_record_cleanup.feature", "service-cleanup"), + ("service_record_unqueryable.feature", "service-unqueryable"), ("service_stop.feature", "service-stop"), ("service_unknown_id.feature", "service-unknown"), ("storage.feature", "storage"), From 7118669a4ed19880fab32529b60a029df0993baf Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 19:56:54 +0000 Subject: [PATCH 16/30] fix(core): refuse an unverifiable stop before the tree kill is skipped, stop an identity mismatch from vouching for another root, let prune name remove --force, and say plainly which Windows paths no CI job runs Windows: when the first identity read was unverifiable and the read inside the termination then found the process verifiable, the tree kill was skipped and the root was stopped alone, which both callers then counted as a stop. The first read is now the verdict when it refuses, and the root is not handed to the termination. A sibling that is identity_mismatch (a stranger holds its PID) no longer vouches for what was under a root that exited by itself; only a stop, an already-gone process or another such root does. rocm services remove --force without --yes now tells the user to add --yes, not to drop --force, and the test runs the command it names. prune never forces, but lists a record held only by an unqueryable process, after a stop was requested, with the exact remove --force --yes command; a test runs that command and checks it clears the record, and service-unqueryable-01 drives it through the binary. More of the Windows logic is pure and tested on every host: the exit-code query mapping (classify_exit_query) and the taskkill arguments (taskkill_args), next to the open-failure mapping, the taskkill result and the System32 path building that already were. The failed-launch cleanup comment states the tradeoff of not killing a child whose creation time could not be read. The docs and comments no longer say the Windows-only paths (the OpenProcess codes, the creation-time token, the PID hold, taskkill and its exit 128) run on a CI lane: no CI job executes the stop scenarios on Windows, and none of it was run on a Windows host. docs/manual-testing.md gains a Windows host check. Signed-off-by: Jussi Elo --- README.md | 3 +- apps/rocm/src/cli.rs | 6 +- apps/rocm/src/service_launch.rs | 11 ++ apps/rocm/src/service_records.rs | 167 ++++++++++++++++-- crates/rocm-core/src/proc_lifecycle.rs | 145 ++++++++++++--- crates/rocm-core/src/process.rs | 36 +++- docs/manual-testing.md | 15 +- docs/testing.md | 31 +++- .../service_record_unqueryable.feature | 7 + .../tests/e2e/service_cleanup_steps.rs | 22 +++ 10 files changed, 381 insertions(+), 62 deletions(-) diff --git a/README.md b/README.md index a0da29fcef..94b8a30088 100644 --- a/README.md +++ b/README.md @@ -1331,7 +1331,8 @@ open): `remove` and `prune` refuse such a record until that process is gone. removes the record when a stop was requested, no recorded process is confirmed running, and at least one could not be queried, says which one, and never removes a record whose process `rocm` can see running. `prune` has no -`--force`. `docs/manual-testing.md` walks through it. +`--force`, but when it leaves such a record alone it lists it with the exact +`remove --force --yes` command. `docs/manual-testing.md` walks through it. Removal destroys both the log and the `restart` option for the records it takes, so `prune` only considers records untouched for 24 hours. Age is diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index f4b046a072..362d1ddc9c 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -1193,8 +1193,10 @@ pub(crate) enum ServicesCommand { }, /// Delete local server records that are no longer running. /// - /// Running servers are always left alone. Leftover files whose record is - /// already gone are cleaned up too. + /// Running servers are always left alone; a record held only by a process + /// `rocm` cannot query (on Windows) is listed with the `remove --force` + /// command that clears it. Leftover files whose record is already gone are + /// cleaned up too. /// /// Waits for a managed launch already under way to publish its record /// before reading the directory, and waits as long as that launch takes. diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 5b069a6034..3ae63e2e04 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -476,6 +476,17 @@ fn abandon_failed_managed_launch(paths: &AppPaths, record: &ManagedServiceRecord // the pid is unverifiable and nothing is signalled (`terminate_verified` // alone would "best-effort proceed"). A token that was captured is // verified by `terminate_verified` on both. + // + // The tradeoff on Windows, chosen deliberately: a child whose creation + // time could not be read at capture is not killed, so if `OpenProcess` + // failed for a child that is in fact alive (it is this account's own + // fresh child, so that is not expected), a failed launch leaves its + // engine running and the audit event says `unverified`. The alternative + // is a forced tree kill of a pid that may by now belong to another + // process, which cannot be undone. The spawn primitive returns only the + // pid and closes its handle, so a handle-based identity would mean + // changing it; that is not done here, and this path is not executed on + // Windows by any CI job. let outcome = if let Some(refusal) = launch_cleanup_refusal(record.engine_start_ticks, host_records_start_tokens()) { diff --git a/apps/rocm/src/service_records.rs b/apps/rocm/src/service_records.rs index 6c6aea781f..305140e647 100644 --- a/apps/rocm/src/service_records.rs +++ b/apps/rocm/src/service_records.rs @@ -170,12 +170,10 @@ fn remove_service_record_artifacts(artifacts: &ServiceRecordArtifacts) -> Result /// A recorded process that could not be queried (on Windows, a live one this /// process may not open) counts as running: deleting the record and its key on /// the strength of a failed look is the mistake this guard exists to prevent. -fn managed_service_record_is_in_use(record: &ManagedServiceRecord) -> bool { - managed_service_record_is_in_use_with(record, rocm_core::probe_process) -} - -/// [`managed_service_record_is_in_use`] with the read of each recorded PID -/// supplied by the caller. +/// +/// The read of each recorded PID is supplied by the caller, so a test drives an +/// unqueryable process on every host; production passes +/// [`rocm_core::probe_process`]. fn managed_service_record_is_in_use_with( record: &ManagedServiceRecord, probe: impl Fn(u32) -> ProcessProbe, @@ -297,10 +295,15 @@ pub(crate) fn remove_managed_service_record_with( } } if !yes { - bail!( - "Removing local server record `{service_id}` requires --yes.\n\nTry: {}", + // The command that completes this removal, not a neighbour of it: a + // forced removal that is missing `--yes` is told to add it, not to drop + // the `--force` it asked for. + let hint = if forced_over.is_some() { + service_hints::service_forced_remove_hint(service_id) + } else { service_hints::service_action_hint("remove", service_id) - ); + }; + bail!("Removing local server record `{service_id}` requires --yes.\n\nTry: {hint}"); } let artifacts = service_record_artifacts(paths, &record.service_id, &record.engine)?; @@ -527,6 +530,7 @@ fn build_service_prune_plan( min_age: Duration, hours: u64, now: SystemTime, + probe: &impl Fn(u32) -> ProcessProbe, ) -> Result { let mut plan = ServicePrunePlan::default(); // Taken first, because `load_managed_services` below rewrites the manifest of @@ -538,14 +542,28 @@ fn build_service_prune_plan( // the real processes before returning it, so liveness below is read from the // refreshed view, never from the manifest as it was on disk. for record in load_managed_services(paths)? { - if managed_service_record_is_in_use(&record) { + if managed_service_record_is_in_use_with(&record, probe) { plan.skipped_live += 1; - plan.skipped.push(format!( - "{} is {} - stop it first with `{}`", - record.service_id, - record.status, - service_hints::service_stop_hint(&record.service_id) - )); + // A record held only by a process `rocm` cannot query, after a stop + // was requested, is the one `remove --force` exists for: say so and + // name it, instead of advising a stop that can only report the same + // thing again. `prune` itself never forces. + let skipped = match forceable_unqueryable_pids(&record, probe) { + Some(pids) => format!( + "{} is {} and {} it recorded could not be queried - remove it with `{}`", + record.service_id, + record.status, + named_processes(&pids), + service_hints::service_forced_remove_hint(&record.service_id) + ), + None => format!( + "{} is {} - stop it first with `{}`", + record.service_id, + record.status, + service_hints::service_stop_hint(&record.service_id) + ), + }; + plan.skipped.push(skipped); continue; } let artifacts = match service_record_artifacts(paths, &record.service_id, &record.engine) { @@ -690,10 +708,22 @@ pub(crate) struct ServicePruneOutcome { /// was built. That window is the only reason the liveness re-check below exists, /// and no test driving `prune` end to end can open it: one call builds the plan /// and applies it, so a record is either live for both halves or dead for both. +#[cfg(test)] fn apply_service_prune_plan( paths: &AppPaths, plan: &ServicePrunePlan, outcome: &mut ServicePruneOutcome, +) { + apply_service_prune_plan_with(paths, plan, outcome, &rocm_core::probe_process); +} + +/// [`apply_service_prune_plan`] with the read of each recorded PID supplied by +/// the caller. +fn apply_service_prune_plan_with( + paths: &AppPaths, + plan: &ServicePrunePlan, + outcome: &mut ServicePruneOutcome, + probe: &impl Fn(u32) -> ProcessProbe, ) { for entry in &plan.remove { // Liveness was snapshotted while the plan was built. A @@ -703,7 +733,7 @@ fn apply_service_prune_plan( // `remove` narrows the same window by loading the record it deletes. // Neither closes it: the check and the delete are not atomic either way. if load_managed_service(paths, &entry.service_id) - .is_ok_and(|record| managed_service_record_is_in_use(&record)) + .is_ok_and(|record| managed_service_record_is_in_use_with(&record, probe)) { outcome.skipped_live += 1; let _ = writeln!( @@ -829,6 +859,18 @@ pub(crate) fn prune_managed_service_records( hours: u64, dry_run: bool, yes: bool, +) -> Result { + prune_managed_service_records_with(paths, hours, dry_run, yes, &rocm_core::probe_process) +} + +/// [`prune_managed_service_records`] with the read of each recorded PID supplied +/// by the caller. +fn prune_managed_service_records_with( + paths: &AppPaths, + hours: u64, + dry_run: bool, + yes: bool, + probe: &impl Fn(u32) -> ProcessProbe, ) -> Result { if !dry_run && !yes { bail!( @@ -853,7 +895,7 @@ pub(crate) fn prune_managed_service_records( rocm_core::FileLock::acquire(paths.managed_launch_lock_path())? }; let min_age = Duration::from_secs(hours.saturating_mul(3600)); - let plan = build_service_prune_plan(paths, min_age, hours, SystemTime::now())?; + let plan = build_service_prune_plan(paths, min_age, hours, SystemTime::now(), probe)?; let mut outcome = ServicePruneOutcome { text: render_service_prune_plan(&plan, hours, dry_run), skipped_live: plan.skipped_live, @@ -864,7 +906,7 @@ pub(crate) fn prune_managed_service_records( return Ok(outcome); } - apply_service_prune_plan(paths, &plan, &mut outcome); + apply_service_prune_plan_with(paths, &plan, &mut outcome, probe); let _ = writeln!(outcome.text); let _ = write!( @@ -2139,4 +2181,91 @@ mod tests { assert_eq!(left, 3, "nothing was removed"); Ok(()) } + + /// A forced removal that is missing `--yes` is told to add `--yes`, not to + /// drop the `--force` it asked for: the command it names completes the + /// removal, which is asserted by running it. + #[test] + fn a_forced_removal_missing_yes_names_the_command_that_completes_it() -> Result<()> { + let (root, paths) = test_paths("services-remove-force-needs-yes"); + paths.ensure()?; + held_record(&paths, "svc-held", true)?; + let unqueryable = |_: u32| ProcessProbe::Unqueryable; + + let error = + remove_managed_service_record_with(&paths, "svc-held", false, true, unqueryable) + .map(|removed| removed.text) + .expect_err("a forced removal without --yes is refused"); + let message = format!("{error:#}"); + let left_before = files_left(&paths, "svc-held"); + // The named command, run as named, removes the record. + let named = service_hints::service_forced_remove_hint("svc-held"); + let completed = + remove_managed_service_record_with(&paths, "svc-held", true, true, unqueryable)?; + let left_after = files_left(&paths, "svc-held"); + let _ = fs::remove_dir_all(root); + + assert!(message.contains("requires --yes"), "{message}"); + assert!(message.contains(&format!("Try: {named}")), "{message}"); + assert_eq!(left_before, 3, "the refusal removed nothing"); + assert!(completed.forced); + assert_eq!(left_after, 0, "the command it named cleared the record"); + // An ordinary removal still names the plain command. + let plain = service_hints::service_action_hint("remove", "svc-held"); + assert!( + !named.eq(&plain) && named.contains("--force"), + "{named} / {plain}" + ); + Ok(()) + } + + /// `prune` never forces, and says what does: a record held only by an + /// unqueryable process, after a stop was requested, is reported with the + /// exact `remove --force --yes` command, and that command, run, clears the + /// record. A record with a process confirmed running keeps the stop advice. + #[test] + fn prune_names_the_forced_removal_for_a_record_held_by_an_unqueryable_process() -> Result<()> { + let (root, paths) = test_paths("services-prune-names-force"); + paths.ensure()?; + held_record(&paths, "svc-held", true)?; + let unqueryable = |_: u32| ProcessProbe::Unqueryable; + + let outcome = prune_managed_service_records_with(&paths, 0, false, true, &unqueryable)?; + let left_after_prune = files_left(&paths, "svc-held"); + let named = service_hints::service_forced_remove_hint("svc-held"); + // Run exactly what prune named. + let removed = + remove_managed_service_record_with(&paths, "svc-held", true, true, unqueryable)?; + let left_after_named = files_left(&paths, "svc-held"); + + held_record(&paths, "svc-running", true)?; + let running = + prune_managed_service_records_with(&paths, 0, true, false, &|_| ProcessProbe::Running)?; + let _ = fs::remove_dir_all(root); + + assert!( + outcome.text.contains(&format!( + "svc-held is starting and process {} it recorded could not be queried - remove it with `{named}`", + std::process::id() + )), + "{}", + outcome.text + ); + assert_eq!(outcome.removed_records, 0, "prune itself never forces"); + assert_eq!(left_after_prune, 3, "and left every file alone"); + assert!(removed.forced); + assert_eq!( + left_after_named, 0, + "the command prune named clears the record" + ); + assert!( + running.text.contains( + "svc-running is starting - stop it first with `rocm services stop svc-running --yes`" + ), + "{}", + running.text + ); + assert!(!running.text.contains("--force"), "{}", running.text); + Ok(()) + } } diff --git a/crates/rocm-core/src/proc_lifecycle.rs b/crates/rocm-core/src/proc_lifecycle.rs index 3c09ae336a..1e1f744247 100644 --- a/crates/rocm-core/src/proc_lifecycle.rs +++ b/crates/rocm-core/src/proc_lifecycle.rs @@ -548,16 +548,25 @@ pub fn terminate_recorded_service_pids( stops } -/// Whether `stop` is a recorded process that is confirmed gone: stopped, or a -/// root that exited by itself ([`TerminationOutcome::ExitedBeforeTreeKill`]). -/// The stopping process's own PID is not a service process and never counts. +/// Whether `stop` vouches for what was under another recorded root: a recorded +/// process this stop took down ([`Graceful`](TerminationOutcome::Graceful), +/// [`Forced`](TerminationOutcome::Forced)), found already gone +/// ([`AlreadyGone`](TerminationOutcome::AlreadyGone)), or a root that exited by +/// itself ([`TerminationOutcome::ExitedBeforeTreeKill`]). Not +/// [`IdentityMismatch`](TerminationOutcome::IdentityMismatch): it says a stranger +/// holds that PID now, which is "gone" for that process but is no evidence about +/// what was under a different root. The stopping process's own PID is not a +/// service process and never counts. const fn recorded_process_is_gone(stop: RecordedPidStop) -> bool { - match stop.outcome { - RecordedPidOutcome::OwnProcess => false, - RecordedPidOutcome::Terminated(outcome) => { - outcome.stopped() || matches!(outcome, TerminationOutcome::ExitedBeforeTreeKill) - } - } + matches!( + stop.outcome, + RecordedPidOutcome::Terminated( + TerminationOutcome::Graceful + | TerminationOutcome::Forced + | TerminationOutcome::AlreadyGone + | TerminationOutcome::ExitedBeforeTreeKill + ) + ) } /// Turn each [`TerminationOutcome::ExitedBeforeTreeKill`] into @@ -709,9 +718,15 @@ fn terminate_recorded_pid_with( ) -> TerminationOutcome { // Before anything reads the identity, and until the end of the call. let _hold = hold(identity); - if let Some(tree_kill) = tree_kill - && matches!(state_of(identity), IdentityState::Matches) - { + if let Some(tree_kill) = tree_kill { + // The identity read that decides about the tree kill also decides about + // the stop. A refusal here (gone, recycled, unverifiable) is the verdict: + // the root is not handed to `terminate`, which re-reads the identity and + // could find a process that was unverifiable a moment ago verifiable now, + // and then stop the root alone with the tree kill skipped. + if let Some(refusal) = refusal_before_signalling(state_of(identity)) { + return refusal; + } // Taken while the root is still alive: `taskkill /T` resolves children // through the live parent, so it cannot reach them once the root is gone. let tree = tree_kill(identity.pid); @@ -859,12 +874,24 @@ fn wait_for_exit_code( } } +/// The arguments that take `pid` and everything under it down: by PID, with its +/// tree (`/T`), forced (`/F`). Split out so a test pins them on every host. +#[cfg(any(windows, test))] +fn taskkill_args(pid: u32) -> [String; 4] { + [ + "/PID".to_owned(), + pid.to_string(), + "/T".to_owned(), + "/F".to_owned(), + ] +} + #[cfg(windows)] fn taskkill_tree(pid: u32) -> TreeKill { use std::process::{Command, Stdio}; classify_taskkill(run_to_exit_code( Command::new(crate::windows_system_tool("taskkill.exe")) - .args(["/PID", &pid.to_string(), "/T", "/F"]) + .args(taskkill_args(pid)) .stdin(Stdio::null()) .stdout(Stdio::null()) .stderr(Stdio::null()), @@ -1599,17 +1626,18 @@ mod tests { assert_eq!( steps, - vec![ - Step::Hold(SUPERVISOR_PID), - Step::Terminate(SUPERVISOR_PID), - Step::Release - ], - "{state:?}: a stranger's process tree must not be taken" + vec![Step::Hold(SUPERVISOR_PID), Step::Release], + "{state:?}: a stranger's process tree must not be taken, and the root \ + is not handed to the termination, which would re-read the identity" ); + let expected = match state { + IdentityState::Gone => TerminationOutcome::AlreadyGone, + IdentityState::Recycled => TerminationOutcome::IdentityMismatch, + _ => TerminationOutcome::Unverified, + }; assert_eq!( - verdict, - TerminationOutcome::AlreadyGone, - "{state:?}: and the verdict is the termination's, unmapped" + verdict, expected, + "{state:?}: the initial refusal is the verdict" ); } } @@ -1740,11 +1768,8 @@ mod tests { Step::Release, ] } else { - vec![ - Step::Hold(SUPERVISOR_PID), - Step::Terminate(SUPERVISOR_PID), - Step::Release, - ] + // Refused on the initial read: not handed to the termination. + vec![Step::Hold(SUPERVISOR_PID), Step::Release] }; assert_eq!(steps.into_inner(), expected_steps, "{row}: steps"); assert_eq!(verdict.as_str(), label, "{row}: verdict"); @@ -1848,6 +1873,67 @@ mod tests { ); } + /// The identity read that gates the tree kill is the one that decides the + /// stop. If it is unverifiable and a second read, inside the termination, + /// would find the process verifiable, the root must not be stopped alone with + /// the tree kill skipped: the initial refusal is the verdict and the + /// termination is never called. + #[test] + fn an_unverifiable_first_read_is_not_overtaken_by_a_verifiable_second_read() { + use std::cell::Cell; + let reads = Cell::new(0_u32); + let terminated = Cell::new(false); + let tree_killed = Cell::new(false); + let tree_kill = |_pid: u32| { + tree_killed.set(true); + TreeKill::Completed + }; + + let verdict = terminate_recorded_pid_with( + &ProcessIdentity::new(SUPERVISOR_PID, Some(7)), + Some(&tree_kill as &dyn Fn(u32) -> TreeKill), + |_| (), + |_| { + // Indeterminate the first time, matching from then on. + reads.set(reads.get() + 1); + if reads.get() == 1 { + IdentityState::Indeterminate + } else { + IdentityState::Matches + } + }, + |_| { + terminated.set(true); + TerminationOutcome::Forced + }, + ); + + assert_eq!(verdict, TerminationOutcome::Unverified); + assert!(!verdict.stopped()); + assert!(!terminated.get(), "the root is not stopped alone"); + assert!(!tree_killed.get()); + assert_eq!(reads.get(), 1, "the initial read is the only one"); + } + + /// A sibling whose PID a stranger now holds ([`IdentityMismatch`]) says + /// nothing about what was under this root, so it does not corroborate it. + #[test] + fn an_identity_mismatch_sibling_does_not_corroborate_a_root_that_exited_alone() { + use TerminationOutcome::{ExitedBeforeTreeKill, IdentityMismatch}; + let record = record_naming(SUPERVISOR_PID, None, Some(ENGINE_PID), None); + + let verdicts = settled( + &record, + &[ + (SUPERVISOR_PID, ExitedBeforeTreeKill), + (ENGINE_PID, IdentityMismatch), + ], + ); + + assert_eq!(verdicts, vec![ExitedBeforeTreeKill, IdentityMismatch]); + assert!(!verdicts[0].stopped()); + } + /// What the record's processes say together about a root that exited by /// itself. `exits` maps each recorded PID to the verdict `terminate` gives. fn settled( @@ -2143,6 +2229,11 @@ mod tests { assert_eq!(classify_taskkill(Err(error)), TreeKill::Failed); } + #[test] + fn taskkill_is_asked_for_the_pid_its_tree_and_force() { + assert_eq!(taskkill_args(4242), ["/PID", "4242", "/T", "/F"]); + } + #[test] fn only_a_clean_taskkill_exit_is_a_completed_tree_kill_and_128_is_not_found() { use std::io::{Error, ErrorKind}; diff --git a/crates/rocm-core/src/process.rs b/crates/rocm-core/src/process.rs index 80114ef259..68b5833c69 100644 --- a/crates/rocm-core/src/process.rs +++ b/crates/rocm-core/src/process.rs @@ -410,6 +410,24 @@ impl ProcessProbe { #[cfg(any(windows, test))] const ERROR_INVALID_PARAMETER: i32 = 87; +/// Windows `STILL_ACTIVE`: the exit code `GetExitCodeProcess` reports for a +/// process that has not exited. +#[cfg(any(windows, test))] +const STILL_ACTIVE: u32 = 259; + +/// What `GetExitCodeProcess` on a handle that opened says about the process: +/// still active is running, any other exit code is gone, and a failed query is +/// neither. Split out of the FFI, with the open-failure mapping above, so a test +/// runs the whole decision on every host. +#[cfg(any(windows, test))] +pub(crate) const fn classify_exit_query(queried: bool, exit_code: u32) -> ProcessProbe { + match (queried, exit_code) { + (false, _) => ProcessProbe::Unqueryable, + (true, STILL_ACTIVE) => ProcessProbe::Running, + (true, _) => ProcessProbe::Gone, + } +} + /// What a failed `OpenProcess` says about the PID, from its `GetLastError` code. /// /// Only the code for "no such process" counts as gone. `ERROR_ACCESS_DENIED` @@ -488,12 +506,7 @@ pub fn probe_process(pid: u32) -> ProcessProbe { unsafe { CloseHandle(handle); } - match (queried, exit_code) { - (false, _) => ProcessProbe::Unqueryable, - // `STILL_ACTIVE`. - (true, 259) => ProcessProbe::Running, - (true, _) => ProcessProbe::Gone, - } + classify_exit_query(queried, exit_code) } /// Look at `pid`. `kill(pid, 0)` already tells a live process the caller may @@ -1450,6 +1463,17 @@ mod tests { } } + /// An opened handle's exit code: still active is running, any other code is + /// gone, and a failed query (whatever the code it left behind) is neither. + #[test] + fn an_exit_code_query_is_running_gone_or_unqueryable() { + assert_eq!(classify_exit_query(true, 259), ProcessProbe::Running); + assert_eq!(classify_exit_query(true, 0), ProcessProbe::Gone); + assert_eq!(classify_exit_query(true, 1), ProcessProbe::Gone); + assert_eq!(classify_exit_query(false, 0), ProcessProbe::Unqueryable); + assert_eq!(classify_exit_query(false, 259), ProcessProbe::Unqueryable); + } + /// Off Windows, a PID that is running (including one the caller may not /// signal) probes as running and an absent one as gone, matching /// [`process_is_running`]. diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 26ea9e10ad..dd5f2b753c 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -562,6 +562,18 @@ outcome for the stopping process itself; the JSON answer of the sandbox `stop_server` tool does, in `pid_outcomes`, as `self` (never signalled, counted as stopped). +#### Windows host check + +The Windows-only parts of the stop (the `OpenProcess` failure codes, the +creation-time token, the PID hold, `taskkill /T /F` and its exit codes, and the +`System32` resolution) are executed by no CI job, and were not run on a Windows +host when they were written. On a Windows host with a `--features +rocm/e2e-test-hooks` debug build, run +`cargo test -p e2e-cucumber --test e2e -- -n "service-stop-0[1-8]"` and +`-n "service-unqueryable-0"` with `ROCM_CLI_BINARY` set (`service-stop-07` and +the Windows half of `service-stop-08` run only there; `service-unqueryable-02` +is Linux-only), and record the result. + #### A record that stays unverified If the server is gone but its recorded PID has been taken by a process this @@ -581,7 +593,8 @@ possibly running, which is the safe direction: the record never reconciles to command says which process could not be queried and that, if it is still running, it keeps running, and it deletes the record, its log, its engine state and its endpoint key. The server can no longer be restarted afterwards. - `rocm services prune` has no `--force`: it leaves such a record alone. + `rocm services prune` has no `--force`: it leaves such a record alone and lists it + under `Left alone` with this exact command. The rest of this section needs a **vLLM** server running, on Linux or WSL, and two runtimes to move it between. Nothing earlier in this document sets either diff --git a/docs/testing.md b/docs/testing.md index 49d263e5cb..1c861249ce 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2051,7 +2051,7 @@ runs the real routine with the per-PID signal injected, and asserts the persisted record after the first stop and that a second stop signals nothing; `an_unconfirmed_stop_keeps_the_recorded_processes_for_the_next_stop` is the other direction. `service-stop-07` -(`@requires-os:windows`, no GPU) is the live coverage of the Windows tree kill: +(`@requires-os:windows`, no GPU) is the live coverage of the Windows tree kill when it is run on a Windows host, which no CI job does: it records a PowerShell parent that has started a `ping` child, runs `rocm services stop --yes`, and asserts that both processes are gone together with the report (`Local server stopped`, `stopped processes: 1`) and the record (stopped, @@ -2086,8 +2086,16 @@ repeats it against a bystander on Windows only and then stops that bystander with its true token. `service-stop-08` is the live version on every OS: it records a running process under a token no live process has, stops the server, and asserts that the process is still running and the record names no process. -Neither Windows-only test nor the Windows half of `service-stop-08` ran where -this was written; the Windows CI jobs are what exercise them. A stop that cannot +None of the Windows-only paths has been executed on a Windows host for this +change, and no CI job executes them: the Windows build-and-test job runs clippy, +the build, the unit tests and the harness self-tests, not the `service-stop-*` +or `service-unqueryable-*` scenarios. `service-stop-07` (`@requires-os:windows`) +and the Windows half of `service-stop-08` can run only through the harness on a +Windows host, which is a manual step (see "Windows host check" in +[manual-testing.md](manual-testing.md)); on the Linux e2e jobs `-07` is skipped +and `-08` runs its Linux half. The Windows unit tests (`#[cfg(windows)]`) run on +the Windows unit-test job; they were written without a Windows host to run them +on. A stop that cannot verify a process names `ps -p -o pid,args` on Linux and `tasklist /FI "PID eq "` on Windows (`process_check_command`). Three further Windows paths of that stop are tested through injected functions @@ -2154,13 +2162,24 @@ in [manual-testing.md](manual-testing.md); the unit tests `force_removes_a_record_held_by_an_unqueryable_process_and_says_so`, `a_record_held_by_an_unqueryable_process_is_refused_naming_the_forced_removal`, `force_is_refused_for_a_running_process_and_without_a_requested_stop` and -`force_on_a_record_that_is_not_running_is_an_ordinary_removal` cover it with the +`force_on_a_record_that_is_not_running_is_an_ordinary_removal`, +`a_forced_removal_missing_yes_names_the_command_that_completes_it` and +`prune_names_the_forced_removal_for_a_record_held_by_an_unqueryable_process` (prune +lists the record with the command, and running that command clears it) cover it with the probe injected, and `service-unqueryable-01` and `-02` drive the stop, the refusal naming `--force`, and the forced removal through the real binary, with the `e2e-test-hooks` variable `ROCM_E2E_UNQUERYABLE_PIDS` making the listed PIDs read as unqueryable; `-02` is Linux-only because it starts a real process). What no -test here reaches is the Windows API itself: the handle hold, the open-failure -codes and a real `taskkill` failing run only on the Windows lane. +test here reaches is the Windows API itself: the handle hold, the `OpenProcess` +failure codes, `GetProcessTimes` and a real `taskkill` (its exit 128 included) +are executed by no CI job and were not run on a Windows host. What was moved +out of the FFI so that it is unit-tested on Linux and on the Windows unit-test +job: the open-failure code mapping (`classify_open_failure`), the exit-code query +mapping (`classify_exit_query`), the `taskkill` result and arguments +(`classify_taskkill`, `taskkill_args`), the `System32` path building, the +FILETIME join, and the deadline wait (a real child). The ordering of the hold, +the identity read, the tree kill and the termination is tested through injected +functions. The same unqueryable-process rule holds wherever a service's liveness decides something destructive. The deferred key cleanup and the reconciliation to diff --git a/tests/e2e-cucumber/features/service_record_unqueryable.feature b/tests/e2e-cucumber/features/service_record_unqueryable.feature index b006469d16..77ecaf04db 100644 --- a/tests/e2e-cucumber/features/service_record_unqueryable.feature +++ b/tests/e2e-cucumber/features/service_record_unqueryable.feature @@ -12,6 +12,10 @@ Feature: A local server record held by a process rocm cannot query # "cannot be queried" for the PIDs it lists; production builds never read it. # The records are planted as plain JSON, so no scenario needs a GPU. # + # `service-unqueryable-01` has no OS tag and the hook works on Windows too, + # but no CI job runs this feature on Windows: the Windows job does not run + # scenarios. It has run on Linux only. + # # A new feature file, as `service_record_cleanup.feature` explains for its # own: indexes must be sequential per file, and several open branches claim the # next ones in the files that exist. @@ -25,6 +29,9 @@ Feature: A local server record held by a process rocm cannot query When the user removes that local server record while its process cannot be queried Then the CLI refuses the removal and names the forced removal And every file belonging to that record is still there + When the user prunes every record whatever its age while its process cannot be queried + Then the CLI leaves the record alone and names the forced removal + And every file belonging to that record is still there When the user force-removes that local server record while its process cannot be queried Then the CLI removes the record and says its process could not be queried And every file belonging to that record is gone diff --git a/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs b/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs index ec02018e70..40299fee61 100644 --- a/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs @@ -785,6 +785,28 @@ async fn force_remove_unqueryable(world: &mut E2eWorld) { record(world, stdout, stderr, rc); } +#[when("the user prunes every record whatever its age while its process cannot be queried")] +async fn prune_unqueryable(world: &mut E2eWorld) { + script_unqueryable(world); + let (stdout, stderr, rc) = + crate::run_rocm_with_scenario_env(world, &["services", "prune", "--any-age", "--yes"]); + record(world, stdout, stderr, rc); +} + +#[then("the CLI leaves the record alone and names the forced removal")] +async fn prune_leaves_and_names_force(world: &mut E2eWorld) { + assert_succeeded(world); + let combined = combined_output(world); + assert!( + combined.contains(&format!( + "{SERVICE_ID} is " + )) && combined.contains(&format!( + "process {UNQUERYABLE_PID} it recorded could not be queried - remove it with `rocm services remove {SERVICE_ID} --force --yes`" + )), + "{combined}" + ); +} + #[when("the user force-removes that local server record")] async fn force_remove(world: &mut E2eWorld) { let (stdout, stderr, rc) = crate::run_rocm( From 8ec8defc129907224ce9a6ea8a95d966d32be82e Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 20:25:50 +0000 Subject: [PATCH 17/30] fix(rocm): re-apply the failed-launch cleanup refusal in the split launch_transaction module The merge of main took main's split layout of service_launch.rs into a directory, and the change this PR makes to the failed-launch cleanup was not carried into the child modules by that merge commit. This re-applies it where the code now lives: launch_cleanup_refusal and host_records_start_tokens, with the cleanup tradeoff comment and the stop-routine call-site change (an empty termination in place of the removed confirmed_stopped stand-in), and the two tests, in service_launch/launch_transaction.rs; and the process_check_command assertion in service_launch/attach.rs. No behaviour changes beyond what the PR already described. Signed-off-by: Jussi Elo --- apps/rocm/src/service_launch/attach.rs | 8 +- .../src/service_launch/launch_transaction.rs | 83 +++++++++++++++++-- 2 files changed, 83 insertions(+), 8 deletions(-) diff --git a/apps/rocm/src/service_launch/attach.rs b/apps/rocm/src/service_launch/attach.rs index cd1bfe7700..d76a90e8d4 100644 --- a/apps/rocm/src/service_launch/attach.rs +++ b/apps/rocm/src/service_launch/attach.rs @@ -561,7 +561,13 @@ mod tests { finish("attached-unverified", Ok(stop_answer(false, "unverified"))); let error = format!("{:#}", result.expect_err("an unverified stop fails")); assert!(error.contains("not signaled: process 4242"), "{error}"); - assert!(error.contains("check: `ps -p 4242 -o pid,args`"), "{error}"); + assert!( + error.contains(&format!( + "check: `{}`", + crate::services_text::process_check_command(4242) + )), + "{error}" + ); crate::services_text::tests::assert_no_retry_line(&error); assert!(out.is_empty(), "{out}"); assert_eq!(signalled, None); diff --git a/apps/rocm/src/service_launch/launch_transaction.rs b/apps/rocm/src/service_launch/launch_transaction.rs index 13fe90ed1b..dc54ec89ce 100644 --- a/apps/rocm/src/service_launch/launch_transaction.rs +++ b/apps/rocm/src/service_launch/launch_transaction.rs @@ -104,6 +104,30 @@ pub(super) fn resolve_managed_port_in_transaction( /// be published. Short: the child is milliseconds old and has nothing to flush. const MANAGED_LAUNCH_CLEANUP_GRACE: Duration = Duration::from_secs(2); +/// Whether this host records a start token for a process: Linux (the kernel +/// start-time) and Windows (the creation time). Anywhere else +/// `rocm_core::process_start_ticks` has nothing to read. +const fn host_records_start_tokens() -> bool { + cfg!(any(target_os = "linux", windows)) +} + +/// The verdict that ends a failed launch's cleanup before anything is signalled: +/// `Some(Unverified)` for a child whose token was not captured on a host that +/// records tokens, because that child was gone before it could be captured and +/// its pid may now belong to another process. `None` otherwise: a captured token +/// is verified by the termination, and on a host with no tokens there is nothing +/// to verify, so the kill proceeds as it always did there. +const fn launch_cleanup_refusal( + engine_start_ticks: Option, + host_records_tokens: bool, +) -> Option { + if engine_start_ticks.is_none() && host_records_tokens { + Some(rocm_core::TerminationOutcome::Unverified) + } else { + None + } +} + /// Undo a failed managed launch attempt before the launch lock is released, /// so no failed transaction leaves a live reservation or a stray endpoint key. /// @@ -124,12 +148,26 @@ pub(super) fn abandon_failed_managed_launch(paths: &AppPaths, record: &ManagedSe // The token captured at spawn, not a fresh `capture`: when the child // already exited, a fresh one would describe a recycled pid's new owner // and the forced tree kill would hit it. Where start tokens exist - // (Linux), a missing one means the child was gone before it could be - // captured, so the pid is unverifiable and nothing is signalled - // (`terminate_verified` alone would "best-effort proceed"). Windows - // records no token at all, so there the kill proceeds as before. - let outcome = if record.engine_start_ticks.is_none() && cfg!(target_os = "linux") { - rocm_core::TerminationOutcome::Unverified + // (Linux, and Windows, which records the process creation time), a + // missing one means the child was gone before it could be captured, so + // the pid is unverifiable and nothing is signalled (`terminate_verified` + // alone would "best-effort proceed"). A token that was captured is + // verified by `terminate_verified` on both. + // + // The tradeoff on Windows, chosen deliberately: a child whose creation + // time could not be read at capture is not killed, so if `OpenProcess` + // failed for a child that is in fact alive (it is this account's own + // fresh child, so that is not expected), a failed launch leaves its + // engine running and the audit event says `unverified`. The alternative + // is a forced tree kill of a pid that may by now belong to another + // process, which cannot be undone. The spawn primitive returns only the + // pid and closes its handle, so a handle-based identity would mean + // changing it; that is not done here, and this path is not executed on + // Windows by any CI job. + let outcome = if let Some(refusal) = + launch_cleanup_refusal(record.engine_start_ticks, host_records_start_tokens()) + { + refusal } else { rocm_core::terminate_verified( &rocm_core::ProcessIdentity::new(pid, record.engine_start_ticks), @@ -703,7 +741,7 @@ mod tests { // confirms every recorded process gone without touching the test // process itself. crate::service_control::stop_internal_managed_service_with(&paths, "lemonade-a-1", |_| { - crate::service_control::RecordedStopTermination::confirmed_stopped() + Vec::new() })?; let decision = resolve_managed_port_in_transaction( @@ -999,4 +1037,35 @@ mod tests { ); Ok(()) } + + /// A failed launch's cleanup refuses to signal a child whose token was not + /// captured on any host that records tokens. Windows is one of them, since + /// it records the creation time: a missing token there means the child was + /// gone before it could be captured, and its pid may now be another + /// process's, so the kill must not proceed as it did when Windows had none. + #[test] + fn launch_cleanup_refuses_a_child_without_a_token_only_where_tokens_exist() { + use rocm_core::TerminationOutcome::Unverified; + + // A host that records tokens: a missing one is unverifiable, a captured + // one goes on to be verified by the termination. + assert_eq!(launch_cleanup_refusal(None, true), Some(Unverified)); + assert_eq!(launch_cleanup_refusal(Some(7), true), None); + // A host with no tokens has nothing to verify and proceeds as before. + assert_eq!(launch_cleanup_refusal(None, false), None); + assert_eq!(launch_cleanup_refusal(Some(7), false), None); + } + + /// Ties the cleanup's idea of "this host records tokens" to what + /// `process_start_ticks` really does, on the host the test runs on: Linux and + /// Windows read a token for a live process (this one), anywhere else there + /// is none. If a platform gains or loses tokens without the cleanup being + /// told, this fails there. + #[test] + fn the_cleanup_guard_agrees_with_whether_this_host_reads_tokens() { + assert_eq!( + host_records_start_tokens(), + rocm_core::process_start_ticks(std::process::id()).is_some() + ); + } } From 438b353320cc83c0acf7f59b7a3d1c48a4794604 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 20:39:28 +0000 Subject: [PATCH 18/30] docs: select only the scenarios that apply to Windows, and say what the Windows job does run The Windows host check selected service-stop-01 to -08 and service-unqueryable-0, which includes the Linux-only scenarios that need sleep or /proc; -n bypasses the harness's OS filter, so following it would fail for want of the platform. It now selects service-stop-0[178] and service-unqueryable-01 and says why the others are excluded. The blanket claim that no CI job exercises the Windows APIs understated what is configured. Read from the workflow: windows-debug, when Rust files change, runs clippy, the build and cargo test --workspace --all-targets, and that includes two cfg(windows) proc_lifecycle tests that stop a real ping process through the PID hold, the creation-time read, the identity gate and the System32 taskkill. docs/testing.md, docs/manual-testing.md and a code comment now separate that unit-test coverage from what no CI job runs (the stop scenarios, the end-to-end tree kill of a parent with a child, a real unopenable process, taskkill exiting 128 or failing or stalling), without claiming a Windows result for this head. Signed-off-by: Jussi Elo --- .../src/service_launch/launch_transaction.rs | 5 +- docs/manual-testing.md | 25 ++++++---- docs/testing.md | 49 +++++++++++++------ 3 files changed, 52 insertions(+), 27 deletions(-) diff --git a/apps/rocm/src/service_launch/launch_transaction.rs b/apps/rocm/src/service_launch/launch_transaction.rs index dc54ec89ce..abec884c91 100644 --- a/apps/rocm/src/service_launch/launch_transaction.rs +++ b/apps/rocm/src/service_launch/launch_transaction.rs @@ -162,8 +162,9 @@ pub(super) fn abandon_failed_managed_launch(paths: &AppPaths, record: &ManagedSe // is a forced tree kill of a pid that may by now belong to another // process, which cannot be undone. The spawn primitive returns only the // pid and closes its handle, so a handle-based identity would mean - // changing it; that is not done here, and this path is not executed on - // Windows by any CI job. + // changing it; that is not done here. Only the pure refusal decision + // (`launch_cleanup_refusal`) is under a test that runs on Windows; the + // cleanup of a real Windows child is run by no test. let outcome = if let Some(refusal) = launch_cleanup_refusal(record.engine_start_ticks, host_records_start_tokens()) { diff --git a/docs/manual-testing.md b/docs/manual-testing.md index c5b93f8d53..9b4149336b 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -574,15 +574,22 @@ as stopped). #### Windows host check -The Windows-only parts of the stop (the `OpenProcess` failure codes, the -creation-time token, the PID hold, `taskkill /T /F` and its exit codes, and the -`System32` resolution) are executed by no CI job, and were not run on a Windows -host when they were written. On a Windows host with a `--features -rocm/e2e-test-hooks` debug build, run -`cargo test -p e2e-cucumber --test e2e -- -n "service-stop-0[1-8]"` and -`-n "service-unqueryable-0"` with `ROCM_CLI_BINARY` set (`service-stop-07` and -the Windows half of `service-stop-08` run only there; `service-unqueryable-02` -is Linux-only), and record the result. +The Windows build-and-test job runs the workspace unit tests, which include two +`#[cfg(windows)]` tests in `proc_lifecycle.rs` that stop a real `ping` process +through the PID hold, the creation-time read and the System32 `taskkill`. It does +not run any scenario, and nothing there stages the unrun parts: the end-to-end +tree kill of a parent with a child (`service-stop-07`), a real unopenable +process (the `OpenProcess` failure codes), and `taskkill` exiting 128 or failing +or stalling. None of these has been run on a Windows host, and no Windows result +is claimed. On a Windows host with a `--features rocm/e2e-test-hooks` debug +build and `ROCM_CLI_BINARY` set, run the scenarios that apply to Windows: +`cargo test -p e2e-cucumber --test e2e -- -n "service-stop-0[178]"` (`-01` +plants PID 0, `-07` is Windows-only, `-08` runs on every OS) and +`cargo test -p e2e-cucumber --test e2e -- -n "service-unqueryable-01"`, and +record the result. Do not select `service-stop-02` to `-06` or +`service-unqueryable-02`: they are Linux-only (they need `sleep` or `/proc`), +and `-n` bypasses the harness's OS filter, so they would fail for want of the +platform rather than for any defect. #### A record that stays unverified diff --git a/docs/testing.md b/docs/testing.md index 4463585552..0b03ec2426 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2093,16 +2093,29 @@ repeats it against a bystander on Windows only and then stops that bystander with its true token. `service-stop-08` is the live version on every OS: it records a running process under a token no live process has, stops the server, and asserts that the process is still running and the record names no process. -None of the Windows-only paths has been executed on a Windows host for this -change, and no CI job executes them: the Windows build-and-test job runs clippy, -the build, the unit tests and the harness self-tests, not the `service-stop-*` -or `service-unqueryable-*` scenarios. `service-stop-07` (`@requires-os:windows`) -and the Windows half of `service-stop-08` can run only through the harness on a -Windows host, which is a manual step (see "Windows host check" in -[manual-testing.md](manual-testing.md)); on the Linux e2e jobs `-07` is skipped -and `-08` runs its Linux half. The Windows unit tests (`#[cfg(windows)]`) run on -the Windows unit-test job; they were written without a Windows host to run them -on. A stop that cannot +None of the Windows-only paths has been run on a Windows host for this change, +and I am not claiming a Windows result for this head. What the CI configuration +does and does not run there, read from `.github/workflows/ci.yml`: the +`windows-debug` job (`Windows build and test (debug)`), when Rust files change, +runs clippy, the build, `cargo test --workspace --all-targets` and the harness +self-tests. That test run includes unit tests in `proc_lifecycle.rs` that stop a +real process on Windows: `terminate_recorded_pid_leaves_a_recycled_pid_alone_and_stops_the_verified_one` +and `terminate_recorded_pid_stops_a_process_recorded_without_a_token` are +`#[cfg(windows)]` and start a real `ping` child and stop it through +`terminate_recorded_pid`, which takes the PID hold, reads the creation time, +applies the identity gate (a wrong token leaves the process running; a matching +or absent token stops it) and runs the System32 `taskkill /T /F`. The e2e target +is excluded from `cargo test`, so no scenario runs there: the `service-stop-*` +and `service-unqueryable-*` scenarios do not run on that job. +`service-stop-07` (`@requires-os:windows`) and the Windows half of +`service-stop-08` can run only through the harness on a Windows host, which is a +manual step (see "Windows host check" in [manual-testing.md](manual-testing.md)); +on the Linux e2e jobs `-07` is skipped and `-08` runs its Linux half. Not covered +by any CI job: the end-to-end tree kill of a parent with a child (`-07`), the +failure cases that cannot be staged (an `OpenProcess` access-denied process, a +failing or stalled `taskkill`, `taskkill` exiting 128), and the launch-cleanup +branch; the unit tests above stop a `ping` that has no child of its own, so they +do not show a descendant being taken down. A stop that cannot verify a process names `ps -p -o pid,args` on Linux and `tasklist /FI "PID eq "` on Windows (`process_check_command`). Three further Windows paths of that stop are tested through injected functions @@ -2176,11 +2189,15 @@ lists the record with the command, and running that command clears it) cover it probe injected, and `service-unqueryable-01` and `-02` drive the stop, the refusal naming `--force`, and the forced removal through the real binary, with the `e2e-test-hooks` variable `ROCM_E2E_UNQUERYABLE_PIDS` making the listed PIDs -read as unqueryable; `-02` is Linux-only because it starts a real process). What no -test here reaches is the Windows API itself: the handle hold, the `OpenProcess` -failure codes, `GetProcessTimes` and a real `taskkill` (its exit 128 included) -are executed by no CI job and were not run on a Windows host. What was moved -out of the FFI so that it is unit-tested on Linux and on the Windows unit-test +read as unqueryable; `-02` is Linux-only because it starts a real process). On +Windows the happy paths of the PID hold, the creation-time read and the System32 +`taskkill` are exercised by the two `#[cfg(windows)]` unit tests named above when +`windows-debug` runs the workspace tests (no result for this head is claimed +here). What no test reaches there is the failure side of the Windows API: the +`OpenProcess` failure codes against a real unopenable process, `taskkill` +exiting 128 or failing or stalling, and a descendant actually being taken down; +none of those has been run on a Windows host. What was moved out of the FFI so +that it is unit-tested on Linux and, as plain `cargo test`, on the Windows unit job: the open-failure code mapping (`classify_open_failure`), the exit-code query mapping (`classify_exit_query`), the `taskkill` result and arguments (`classify_taskkill`, `taskkill_args`), the `System32` path building, the @@ -2203,7 +2220,7 @@ Three gaps in the scenario coverage are deliberate. `service-stop-07` records no token, so the path a record written by this version takes on Windows (a token recorded, matching, with the tree kill then running) has no scenario; it rests on `what_a_stop_does_for_each_recorded_and_live_token` and the other injected tests, -and on the Windows unit tests that stop a real process with its true token. And +and on the `#[cfg(windows)]` unit tests that stop a real process with its true token. And the `self` row that `rocm` now puts in `pid_outcomes` has no scenario: it appears only in the JSON answer of the hidden `sandbox-run stop_server` and the sandbox tool, never in the text of `rocm services stop`, and it names the stopping From 5f75aa103cbed0ee189ba1e7dc6b82e586f1c9a7 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Mon, 9 Nov 2026 21:53:48 +0000 Subject: [PATCH 19/30] fix(core): do not tree-kill a Windows PID that could not be held The Windows stop opens a handle on the recorded PID so that it cannot change hands between the identity read and taskkill, which takes a whole process tree by PID. When that open failed the code went on anyway: if the identity reads then still matched, nothing reserved the PID and a reuse in the gap would have lost an unrelated process its tree. A PID whose identity matches but which could not be held is now reported unverified, and neither the tree kill nor the termination runs. A PID that is gone, recycled or unreadable keeps the verdict its identity read gives it, and where there is no separate tree-kill step (everywhere except Windows) the missing hold changes nothing, since the termination re-verifies the identity itself. The hold is now an Option so the decision is made on whether it was taken, and the tests drive a failed hold through the same injected seam on every host. Signed-off-by: Jussi Elo --- crates/rocm-core/src/proc_lifecycle.rs | 133 +++++++++++++++++++++---- docs/architecture.md | 2 +- docs/testing.md | 11 +- 3 files changed, 123 insertions(+), 23 deletions(-) diff --git a/crates/rocm-core/src/proc_lifecycle.rs b/crates/rocm-core/src/proc_lifecycle.rs index 1e1f744247..26e192a167 100644 --- a/crates/rocm-core/src/proc_lifecycle.rs +++ b/crates/rocm-core/src/proc_lifecycle.rs @@ -706,18 +706,18 @@ const fn after_tree_kill(tree: TreeKill, outcome: TerminationOutcome) -> Termina /// [`terminate_recorded_pid`] with its platform and system touchpoints supplied /// by the caller, so a test drives the tree-kill step on every host: /// `tree_kill` takes one PID's descendants down (`None` where the platform needs -/// no separate step), `hold` keeps the PID reserved and is dropped when the call -/// returns, `state_of` reads the PID's identity, and `terminate` is the verified -/// termination of the root. +/// no separate step), `hold` keeps the PID reserved, answering `None` when it +/// could not, and is dropped when the call returns, `state_of` reads the PID's +/// identity, and `terminate` is the verified termination of the root. fn terminate_recorded_pid_with( identity: &ProcessIdentity, tree_kill: Option<&dyn Fn(u32) -> TreeKill>, - hold: impl FnOnce(&ProcessIdentity) -> Held, + hold: impl FnOnce(&ProcessIdentity) -> Option, state_of: impl Fn(&ProcessIdentity) -> IdentityState, terminate: impl Fn(&ProcessIdentity) -> TerminationOutcome, ) -> TerminationOutcome { // Before anything reads the identity, and until the end of the call. - let _hold = hold(identity); + let held = hold(identity); if let Some(tree_kill) = tree_kill { // The identity read that decides about the tree kill also decides about // the stop. A refusal here (gone, recycled, unverifiable) is the verdict: @@ -727,6 +727,14 @@ fn terminate_recorded_pid_with( if let Some(refusal) = refusal_before_signalling(state_of(identity)) { return refusal; } + // An identity that matches is only worth acting on while the PID cannot + // change hands before `taskkill` reaches it. Without a hold nothing + // reserves it between this read and the kill, which takes a whole tree by + // PID, so the stop is not confirmed and nothing is signalled. A process + // that could not be opened and also cannot be read was refused above. + if held.is_none() { + return TerminationOutcome::Unverified; + } // Taken while the root is still alive: `taskkill /T` resolves children // through the live parent, so it cannot reach them once the root is gone. let tree = tree_kill(identity.pid); @@ -751,37 +759,41 @@ fn terminate_recorded_pid_with( /// re-verifies it. #[cfg(windows)] struct PidHold { - _handle: Option, + _handle: std::os::windows::io::OwnedHandle, } #[cfg(not(windows))] struct PidHold; -/// Hold `identity`'s PID. A process that cannot be opened is not held, and the -/// identity read that follows then refuses it as gone or unverifiable. +/// Hold `identity`'s PID, or `None` when the process cannot be opened. A stop +/// whose identity matches but whose PID is not held is not carried out, see +/// [`terminate_recorded_pid_with`]. #[cfg(windows)] #[allow(unsafe_code)] // Win32 FFI -fn hold_pid(identity: &ProcessIdentity) -> PidHold { +fn hold_pid(identity: &ProcessIdentity) -> Option { use std::os::windows::io::{FromRawHandle, OwnedHandle}; use windows_sys::Win32::System::Threading::{OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION}; // SAFETY: `OpenProcess` takes only plain values and returns a handle or null; // there is no pointer to keep valid. let handle = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, identity.pid) }; - PidHold { - // SAFETY: this closure runs only for a non-null handle, which - // `OpenProcess` just returned and which nothing else owns or closes, so - // the `OwnedHandle` may take ownership and close it exactly once on drop. - _handle: (!handle.is_null()).then(|| unsafe { OwnedHandle::from_raw_handle(handle) }), + if handle.is_null() { + return None; } + Some(PidHold { + // SAFETY: `handle` is non-null, `OpenProcess` just returned it and nothing + // else owns or closes it, so the `OwnedHandle` may take ownership and + // close it exactly once on drop. + _handle: unsafe { OwnedHandle::from_raw_handle(handle) }, + }) } // The `cfg` sibling calls non-const code, so this arm cannot be `const` without // giving the two platforms different signatures. #[cfg(not(windows))] -#[allow(clippy::missing_const_for_fn)] -fn hold_pid(_identity: &ProcessIdentity) -> PidHold { - PidHold +#[allow(clippy::missing_const_for_fn, clippy::unnecessary_wraps)] +fn hold_pid(_identity: &ProcessIdentity) -> Option { + Some(PidHold) } /// The separate descendant kill this platform needs before the verified @@ -1518,6 +1530,8 @@ mod tests { #[derive(Debug, PartialEq, Eq)] enum Step { Hold(u32), + /// The hold could not be taken: nothing reserves the PID. + HoldFailed(u32), TreeKill(u32), Terminate(u32), Release, @@ -1553,7 +1567,7 @@ mod tests { has_tree_kill.then_some(&tree_kill as &dyn Fn(u32) -> TreeKill), |identity| { steps.borrow_mut().push(Step::Hold(identity.pid)); - RecordedHold(&steps) + Some(RecordedHold(&steps)) }, |_| state, |identity| { @@ -1748,7 +1762,7 @@ mod tests { Some(&tree_kill as &dyn Fn(u32) -> TreeKill), |id| { steps.borrow_mut().push(Step::Hold(id.pid)); - RecordedHold(&steps) + Some(RecordedHold(&steps)) }, |id| classify_live_identity(id.start_ticks, live), |id| { @@ -1796,6 +1810,83 @@ mod tests { assert_eq!(platform_tree_kill().is_some(), cfg!(windows)); } + /// Run `terminate_recorded_pid_with` for a PID whose hold could not be taken + /// (the process could not be opened), the identity reading as `state`. + fn unheld_run( + has_tree_kill: bool, + state: IdentityState, + outcome: TerminationOutcome, + ) -> (TerminationOutcome, Vec) { + let steps = RefCell::new(Vec::new()); + let tree_kill = |pid: u32| { + steps.borrow_mut().push(Step::TreeKill(pid)); + TreeKill::Completed + }; + let verdict = terminate_recorded_pid_with( + &ProcessIdentity::new(SUPERVISOR_PID, None), + has_tree_kill.then_some(&tree_kill as &dyn Fn(u32) -> TreeKill), + |identity| { + steps.borrow_mut().push(Step::HoldFailed(identity.pid)); + None::> + }, + |_| state, + |identity| { + steps.borrow_mut().push(Step::Terminate(identity.pid)); + outcome + }, + ); + (verdict, steps.into_inner()) + } + + #[test] + fn a_tree_kill_is_not_run_for_a_pid_that_could_not_be_held_even_when_its_identity_matches() { + // Nothing reserves the PID between the identity read and `taskkill`, which + // takes a whole tree by PID, so a recycled PID would lose its tree. The + // stop is not confirmed, and neither the tree kill nor the termination runs. + let (verdict, steps) = unheld_run(true, IdentityState::Matches, TerminationOutcome::Forced); + + assert_eq!(verdict, TerminationOutcome::Unverified); + assert!(!verdict.stopped()); + assert_eq!(steps, vec![Step::HoldFailed(SUPERVISOR_PID)]); + } + + #[test] + fn an_unheld_pid_that_is_gone_recycled_or_unverifiable_keeps_its_refusal() { + // The hold only adds a refusal for an identity that matches: a process + // already known to be gone, a stranger, or unreadable keeps the verdict + // the identity read gives it. + for (state, expected) in [ + (IdentityState::Gone, TerminationOutcome::AlreadyGone), + ( + IdentityState::Recycled, + TerminationOutcome::IdentityMismatch, + ), + (IdentityState::Indeterminate, TerminationOutcome::Unverified), + ] { + let (verdict, steps) = unheld_run(true, state, TerminationOutcome::Forced); + + assert_eq!(verdict, expected, "{state:?}"); + assert_eq!(steps, vec![Step::HoldFailed(SUPERVISOR_PID)], "{state:?}"); + } + } + + #[test] + fn without_a_tree_kill_step_an_unheld_pid_is_still_terminated() { + // Elsewhere nothing is held and the termination re-verifies the identity + // itself, so the missing hold must not turn a stop into a refusal. + let (verdict, steps) = + unheld_run(false, IdentityState::Matches, TerminationOutcome::Graceful); + + assert_eq!(verdict, TerminationOutcome::Graceful); + assert_eq!( + steps, + vec![ + Step::HoldFailed(SUPERVISOR_PID), + Step::Terminate(SUPERVISOR_PID) + ] + ); + } + #[test] fn the_pid_is_held_before_its_identity_is_read_and_released_after_the_stop() { // The hold is what makes the creation time that was compared the @@ -1892,7 +1983,7 @@ mod tests { let verdict = terminate_recorded_pid_with( &ProcessIdentity::new(SUPERVISOR_PID, Some(7)), Some(&tree_kill as &dyn Fn(u32) -> TreeKill), - |_| (), + |_| Some(()), |_| { // Indeterminate the first time, matching from then on. reads.set(reads.get() + 1); @@ -2316,7 +2407,7 @@ mod tests { let verdict = terminate_recorded_pid_with( &ProcessIdentity::new(SUPERVISOR_PID, Some(7)), Some(&tree_kill as &dyn Fn(u32) -> TreeKill), - |_| (), + |_| Some(()), state_of, // The verified termination's own gate, with the same read. |id| refusal_before_signalling(state_of(id)).unwrap_or(TerminationOutcome::Forced), diff --git a/docs/architecture.md b/docs/architecture.md index 41c0231ed7..3ebdf1ca71 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -245,7 +245,7 @@ Further `rocm-core` modules, each with its own doc comment except where noted: - [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) — OpenMPI, libatomic and libnuma detection, the `libmpi_cxx` compatibility shim, and the per-library install plans (`build_openmpi_install_plan`, `build_libatomic_install_plan`, `build_libnuma_install_plan`) and hints, needed by vLLM's ROCm wheels. The distro-generic half of that planning lives in [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs); [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) re-exports its public items, so `rocm_core::openmpi::running_as_root`, `plan_for_os_release_read` and the rest keep resolving, and `pub(crate)` re-exports `PackageManager` and `resolve_package_manager` for [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs). - [`os_release.rs`](../crates/rocm-core/src/os_release.rs) — the one `/etc/os-release` parser every reader of that file uses. - [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs) — the planning half of a system-package install that does not depend on which library is installed: `PackageManager` and `resolve_package_manager` (from an os-release `ID`/`ID_LIKE`), `SystemPackageInstallPlan` and `InstallCommand`, `plan_for_os_release_at`/`plan_for_os_release_read` (which take the per-library builder as an argument), `InstallHint`, the root and passwordless-`sudo` checks (`running_as_root`, `can_autoinstall`) and the `apt-get -s` helpers (`apt_simulate_argv`, `parse_apt_simulate_removals`, `is_protected_rocm_package`). It never runs anything and depends on no other module of this crate except [`os_release.rs`](../crates/rocm-core/src/os_release.rs) and [`hardware_root.rs`](../crates/rocm-core/src/hardware_root.rs), which it reads os-release through. The module is private; its public items are re-exported from [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs). It is a different module from [`system_packages.rs`](../apps/rocm/src/system_packages.rs) in `apps/rocm`, which runs these plans with consent and `sudo`. -- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches (`terminate_verified`), and the one managed-stop routine built on it that `rocm services stop` and `rocmd`'s `stop_server` both call: `terminate_recorded_service_pids` walks the processes a service record names (supervisor, then engine, each with its own start-time, the stopping process's own PID reported as `self` and never signalled) and returns one `RecordedPidStop` per PID (each caller, on a stop that confirms every one gone, then clears the record's recorded processes with `ManagedServiceRecord::clear_recorded_processes` in [`service_record.rs`](../crates/rocm-core/src/service_record.rs), so a repeated stop reaches no stale PID; an unconfirmed stop keeps them); `terminate_recorded_pid` terminates one of them, adding on Windows a `taskkill /T /F` of its descendants first, because `terminate_verified` cannot walk a process tree there, and only for a PID that passes the same identity check (`process_start_ticks` is the kernel start-time on Linux and the process creation time on Windows, so a recycled PID is refused before any `taskkill`; a record written before Windows recorded one cannot be refuted), holding the PID open on Windows from before that check until the stop ends so the PID cannot be reused in between, and treating a tree kill that did not complete, or did not finish within `MANAGED_STOP_GRACE` (the `taskkill` is killed and reaped at that deadline), as an unconfirmed stop (`descendants_unconfirmed` when the root itself was seen to exit, `timed_out` only when it outlived its deadline), except that a `taskkill` which found no such process is judged by the root's own re-read, so a root that exited by itself is `exited_before_tree_kill`, which the record-wide settling in `terminate_recorded_service_pids` turns into a stop only when another recorded process (the engine, which is visited as its own role and folded by both callers) is confirmed gone, and starting that `taskkill` from `%SystemRoot%\System32` through `windows_system_tool`; a PID that cannot be queried at all, as against one that is gone, reads as unverifiable unless a readable recorded token differs, in which case that mismatch wins (`ProcessProbe` and `probe_process` in [`process.rs`](../crates/rocm-core/src/process.rs), whose `may_be_running` is the answer for any decision that must not take an unqueryable process for gone); `MANAGED_STOP_GRACE` is the one grace period. The termination is injectable, so a caller's test supplies outcomes it cannot construct and a test of the Windows step runs on every host. +- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches (`terminate_verified`), and the one managed-stop routine built on it that `rocm services stop` and `rocmd`'s `stop_server` both call: `terminate_recorded_service_pids` walks the processes a service record names (supervisor, then engine, each with its own start-time, the stopping process's own PID reported as `self` and never signalled) and returns one `RecordedPidStop` per PID (each caller, on a stop that confirms every one gone, then clears the record's recorded processes with `ManagedServiceRecord::clear_recorded_processes` in [`service_record.rs`](../crates/rocm-core/src/service_record.rs), so a repeated stop reaches no stale PID; an unconfirmed stop keeps them); `terminate_recorded_pid` terminates one of them, adding on Windows a `taskkill /T /F` of its descendants first, because `terminate_verified` cannot walk a process tree there, and only for a PID that passes the same identity check (`process_start_ticks` is the kernel start-time on Linux and the process creation time on Windows, so a recycled PID is refused before any `taskkill`; a record written before Windows recorded one cannot be refuted), holding the PID open on Windows from before that check until the stop ends so the PID cannot be reused in between (a PID that matches but cannot be held is reported unverified and not signalled), and treating a tree kill that did not complete, or did not finish within `MANAGED_STOP_GRACE` (the `taskkill` is killed and reaped at that deadline), as an unconfirmed stop (`descendants_unconfirmed` when the root itself was seen to exit, `timed_out` only when it outlived its deadline), except that a `taskkill` which found no such process is judged by the root's own re-read, so a root that exited by itself is `exited_before_tree_kill`, which the record-wide settling in `terminate_recorded_service_pids` turns into a stop only when another recorded process (the engine, which is visited as its own role and folded by both callers) is confirmed gone, and starting that `taskkill` from `%SystemRoot%\System32` through `windows_system_tool`; a PID that cannot be queried at all, as against one that is gone, reads as unverifiable unless a readable recorded token differs, in which case that mismatch wins (`ProcessProbe` and `probe_process` in [`process.rs`](../crates/rocm-core/src/process.rs), whose `may_be_running` is the answer for any decision that must not take an unqueryable process for gone); `MANAGED_STOP_GRACE` is the one grace period. The termination is injectable, so a caller's test supplies outcomes it cannot construct and a test of the Windows step runs on every host. - [`report.rs`](../crates/rocm-core/src/report.rs) — the content of a Doctor report and the rule deciding whether one may exist; it sends nothing. - [`report_delivery.rs`](../crates/rocm-core/src/report_delivery.rs) — how a report leaves the machine: building its link and deciding whether a browser may be started. - [`runtime.rs`](../crates/rocm-core/src/runtime.rs) — the host-platform layer: `RuntimePlatform`/`RuntimeHost`, OS-specific executable and Python-environment names (including `runtime_python_env_root_for_executable`, the one rule for finding an environment's folder from its interpreter's path: only an interpreter directly inside a `bin` or `Scripts` folder, in any case, has one, and everything else gets `None`; the adopt command, the ComfyUI runtime environment and the vLLM launch all call it rather than keeping a copy, since the engine crate cannot reach `apps/rocm`), default config/data/cache directories, path-text and path-list helpers, and `install_root_has_payload`, the one definition of a non-empty install root (dot-entries such as the runtime marker do not count; a path that does not exist holds nothing; a root that cannot be listed is an error) that tarball extraction, update repair detection and activation validation in `apps/rocm` all ask (no module doc comment). diff --git a/docs/testing.md b/docs/testing.md index 0b03ec2426..549eb86630 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2127,7 +2127,16 @@ instead of already gone (`a_pid_that_cannot_be_queried_is_unverified_not_already `a_failed_open_is_gone_only_for_no_such_process` for the error codes). The PID is held open from before the identity read until the stop returns, so the creation time that was compared is that of the process the kill then reaches -(`the_pid_is_held_before_its_identity_is_read_and_released_after_the_stop`). And a +(`the_pid_is_held_before_its_identity_is_read_and_released_after_the_stop`). A PID +that could not be held (the process could not be opened) while its identity still +reads as matching is not signalled either: nothing would reserve it between the +read and `taskkill`, so the stop is `unverified` and neither the tree kill nor the +termination runs +(`a_tree_kill_is_not_run_for_a_pid_that_could_not_be_held_even_when_its_identity_matches`; +a gone, recycled or unreadable PID keeps its own refusal, +`an_unheld_pid_that_is_gone_recycled_or_unverifiable_keeps_its_refusal`, and +where there is no tree-kill step the missing hold changes nothing, +`without_a_tree_kill_step_an_unheld_pid_is_still_terminated`). And a `taskkill` that failed to start or exited nonzero leaves the stop unconfirmed, whatever the root's own termination says, because only the root's exit is watched on Windows. When the root itself was seen to exit (or was already gone) From 0d664c95df8af4460527c741a84027270eb418d7 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Tue, 10 Nov 2026 00:49:38 +0000 Subject: [PATCH 20/30] fix(services): forget the recorded processes when a listing completes a pending stop A stop that could not confirm a process leaves the record naming it, and the next liveness refresh that finds every recorded process gone completes the stop: it drops the endpoint key and clears the stop marker. It left the PIDs and start tokens in the record, though, and a PID is stale the moment its process exits. A later services stop would then visit it, and a PID recorded without a token cannot be refuted, so if the OS had handed the number to another process, that process would be signalled and, on Windows, so would its whole tree. The deferred cleanup now clears the recorded processes with the same call a confirmed stop makes, and reads a live record as stopped, so the one write the caller makes covers the key, the marker, the processes and the status. It does so only behind the existing check that every recorded process is confirmed gone: a process that may still be running, including one that cannot be queried, stays named, since the record is the only thing that names it. With no PID left for the reconciliation to read, the status is set in the cleanup itself, so a record in any live status still ends as stopped in that refresh. The regression tests check that a process that may be running stays named and is forgotten once confirmed gone, that every live status ends as stopped, and that the real termination routine run over the record a listing wrote visits no PID. One existing test needed real running processes, because its second stop reads the record through that refresh and made-up PIDs are now forgotten; it is now Unix only, like its neighbours that start sleep processes. Signed-off-by: Jussi Elo --- apps/rocm/src/managed_service_status.rs | 73 ++++++++++++++++++- apps/rocm/src/service_control.rs | 97 ++++++++++++++++++++++++- docs/architecture.md | 2 +- docs/testing.md | 11 +++ 4 files changed, 176 insertions(+), 7 deletions(-) diff --git a/apps/rocm/src/managed_service_status.rs b/apps/rocm/src/managed_service_status.rs index 195f8bc79c..8fa9101cbf 100644 --- a/apps/rocm/src/managed_service_status.rs +++ b/apps/rocm/src/managed_service_status.rs @@ -208,8 +208,9 @@ pub(crate) fn recorded_pids_may_be_running( } /// Complete a stop that was requested but could not confirm termination, once -/// the processes are actually observed gone: drop the endpoint key and clear the -/// marker. Returns whether the record changed. +/// the processes are actually observed gone: drop the endpoint key, clear the +/// marker, forget the recorded processes and read the record as stopped. Returns +/// whether the record changed. /// /// This is deliberately keyed on `stop_requested_unix_ms` rather than on "the /// tracked pids are dead". A *crashed* service also has dead pids, but its @@ -241,6 +242,24 @@ fn settle_pending_stop_key_cleanup( } endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); record.stop_requested_unix_ms = None; + // Every recorded process is confirmed gone, so none of them is the + // service's any more, and their PIDs are stale from here on. Left in the + // record, a later `services stop` would signal whatever the OS has handed + // those numbers to, and a PID recorded without a start-time token cannot be + // refuted, so on Windows the tree kill would take that process and its + // subtree. The same forgetting as a stop that confirmed the exit itself + // (`clear_recorded_processes`), and it is done only here, behind the + // confirmed-gone check above: a process that may still be running stays named. + record.clear_recorded_processes(); + // And the status a confirmed stop leaves, set here rather than left to the + // reconciliation below: with no PID recorded that reconciliation has nothing + // to read, so a record still reading `ready` or `running` would stay live. + // Setting it also makes the caller's liveness gate return at once, so the one + // write the caller makes covers the key cleanup, the cleared processes and + // the status together. + if managed_service_is_live(record) { + "stopped".clone_into(&mut record.status); + } true } @@ -998,10 +1017,25 @@ mod tests { paths.ensure()?; let service_id = "svc-unqueryable"; let mut record = pending_stop_record(&paths, service_id)?; + // The dangerous shape: a supervisor recorded without a start-time token, + // and an engine with one. + record.supervisor_pid = 4242; + record.supervisor_start_ticks = None; + record.engine_pid = Some(4243); + record.engine_start_ticks = Some(77); let _ = refresh_managed_service_runtime_liveness_with(&paths, &mut record, |_| { ProcessProbe::Unqueryable }); + assert_eq!( + ( + record.supervisor_pid, + record.engine_pid, + record.engine_start_ticks + ), + (4242, Some(4243), Some(77)), + "a process that may still be running stays named: the record is the only thing that names it" + ); assert!( endpoint_keys::endpoint_key_file_if_present(&paths, service_id).is_some(), @@ -1028,10 +1062,45 @@ mod tests { ); assert_eq!(record.stop_requested_unix_ms, None); assert_eq!(record.status, "stopped"); + assert_eq!( + ( + record.supervisor_pid, + record.supervisor_start_ticks, + record.engine_pid, + record.engine_start_ticks + ), + (0, None, None, None), + "the processes confirmed gone are forgotten, so a later stop cannot reach a recycled PID" + ); let _ = fs::remove_dir_all(root); Ok(()) } + /// The deferred cleanup forgets the PIDs, so the reconciliation after it has + /// none to read: every live status must still end as `stopped`, in the one + /// refresh, rather than a record such as `starting` staying live with no + /// process behind it. + #[test] + fn a_completed_pending_stop_reads_stopped_whatever_live_status_it_had() -> Result<()> { + for status in ["starting", "running", "ready", "recovering"] { + let (root, paths) = test_paths(&format!("liveness-pending-stop-{status}")); + paths.ensure()?; + let mut record = pending_stop_record(&paths, "svc-status")?; + record.status = status.to_owned(); + + let changed = + refresh_managed_service_runtime_liveness_with(&paths, &mut record, |_| { + ProcessProbe::Gone + }); + + assert!(changed, "{status}"); + assert_eq!(record.status, "stopped", "{status}"); + assert_eq!(record.supervisor_pid, 0, "{status}"); + let _ = fs::remove_dir_all(root); + } + Ok(()) + } + #[test] fn a_pending_stop_keeps_its_key_and_marker_while_a_process_runs() -> Result<()> { let (root, paths) = test_paths("liveness-running-pending-stop"); diff --git a/apps/rocm/src/service_control.rs b/apps/rocm/src/service_control.rs index 433e186b8d..a3e48edead 100644 --- a/apps/rocm/src/service_control.rs +++ b/apps/rocm/src/service_control.rs @@ -1583,16 +1583,30 @@ mod tests { assert_eq!(after_second.engine_pid, None); } + // Unix only: the processes must really be running. The second stop reads the + // record through the liveness refresh, which forgets the PIDs of processes it + // observes gone (`a_stop_completed_by_a_later_listing_leaves_no_pid_for_the_next_stop`), + // so made-up PIDs would be forgotten and not kept. + #[cfg(unix)] #[test] fn an_unconfirmed_stop_keeps_the_recorded_processes_for_the_next_stop() { use rocm_core::TerminationOutcome::TimedOut; + let mut supervisor = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn supervisor"); + let mut engine = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn engine"); + let (supervisor_pid, engine_pid) = (supervisor.id(), engine.id()); let (root, paths) = test_paths("services-stop-keeps-pids"); paths.ensure().unwrap(); let service_id = "svc-services-stop-keeps-pids"; let mut record = stop_test_record(&paths, service_id); - record.supervisor_pid = 4242; + record.supervisor_pid = supervisor_pid; record.supervisor_start_ticks = Some(5); - record.engine_pid = Some(4243); + record.engine_pid = Some(engine_pid); record.engine_start_ticks = Some(77); record.write().unwrap(); @@ -1607,14 +1621,18 @@ mod tests { let after_first = raw_manifest(&paths, service_id); let _ = stop_internal_managed_service_with(&paths, service_id, stop).unwrap(); let signalled_in_all = signalled.get(); + let _ = supervisor.kill(); + let _ = supervisor.wait(); + let _ = engine.kill(); + let _ = engine.wait(); let _ = fs::remove_dir_all(root); // Nothing was confirmed gone, so the record still names what survived: // that is the only thing a later stop can find them by. assert_eq!(first["stopped"], false, "{first}"); - assert_eq!(after_first.supervisor_pid, 4242); + assert_eq!(after_first.supervisor_pid, supervisor_pid); assert_eq!(after_first.supervisor_start_ticks, Some(5)); - assert_eq!(after_first.engine_pid, Some(4243)); + assert_eq!(after_first.engine_pid, Some(engine_pid)); assert_eq!(after_first.engine_start_ticks, Some(77)); assert_eq!(signalled_in_all, 4, "each stop tried both processes"); } @@ -2820,6 +2838,77 @@ mod tests { ); } + /// An unconfirmed stop that a later listing completes (every recorded process + /// is observed gone) must leave a record that names no process, as a confirmed + /// stop does: otherwise the next `services stop` signals whatever the OS has + /// since given those PIDs to, and a PID with no start-time token cannot be + /// refuted. The same record, written back and stopped again, visits no PID. + #[test] + fn a_stop_completed_by_a_later_listing_leaves_no_pid_for_the_next_stop() { + let (root, paths) = test_paths("stop-completed-by-listing-clears-pids"); + paths.ensure().unwrap(); + let service_id = "svc-completed-by-listing"; + let mut planted = stop_test_record(&paths, service_id); + planted.supervisor_pid = 4242; + planted.supervisor_start_ticks = None; + planted.engine_pid = Some(4243); + planted.engine_start_ticks = Some(77); + planted.write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + vec![stop_of(4242, rocm_core::TerminationOutcome::Unverified)] + }) + .unwrap(); + assert_eq!(report["stopped"], false, "{report}"); + let (mut record, _) = read_managed_service_manifest(&paths, service_id).unwrap(); + assert_eq!( + record.supervisor_pid, 4242, + "an unconfirmed stop keeps the PIDs" + ); + + // The listing that finds both processes gone completes the stop. + let changed = crate::managed_service_status::refresh_managed_service_runtime_liveness_with( + &paths, + &mut record, + |_| rocm_core::ProcessProbe::Gone, + ); + assert!(changed); + record.write().unwrap(); + let after_listing = raw_manifest(&paths, service_id); + + let visited = std::cell::RefCell::new(Vec::new()); + let second = stop_internal_managed_service_with(&paths, service_id, |record| { + rocm_core::terminate_recorded_service_pids(record, |identity| { + visited.borrow_mut().push(*identity); + rocm_core::TerminationOutcome::Graceful + }) + }) + .unwrap(); + let visited = visited.into_inner(); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + + assert_eq!(after_listing.status, "stopped"); + assert_eq!(after_listing.stop_requested_unix_ms, None); + assert_eq!(key, None, "the deferred cleanup drops the key"); + assert_eq!( + ( + after_listing.supervisor_pid, + after_listing.supervisor_start_ticks, + after_listing.engine_pid, + after_listing.engine_start_ticks + ), + (0, None, None, None) + ); + assert!( + visited.is_empty(), + "a later stop reached a PID the listing had observed gone: {visited:?}" + ); + assert_eq!(second["stopped"], true, "{second}"); + assert_eq!(second["pid_outcomes"], serde_json::json!([]), "{second}"); + } + /// A stop whose recorded process was seen to exit but whose tree kill did not /// complete says so in those words, not as a timeout, and keeps what an /// unconfirmed stop keeps: the claim and the state are checked together. diff --git a/docs/architecture.md b/docs/architecture.md index 210591683f..6f1bb8c3da 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -70,7 +70,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-214): `--device`/`--gpu` parsing (`parse_device_policy`, `parse_gpu_selection`, `parse_gpu_indices_arg`), the managed-launch-lock GPU selection (`select_gpu_indices_under_launch_lock`, `validate_pinned_gpu_index`, `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `busy_gpu_indices`), per-GPU VRAM telemetry (the `GpuVramUsage` type, `gpu_vram_usage`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs`, `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`), and the serve-plan low-VRAM warning (`gpu_low_memory_warning`, `vram_capacity_is_meaningful`, `serve_gpu_low_memory_warning`), plus the best-effort `amd-smi`-based GPU count (`detect_gpu_count`).Both `amd-smi` probes go through `rocm_core::amd_smi_json` (the `/dev/kfd` pre-flight) with this module's own `AMD_SMI_SERVE_PROBE_TIMEOUT`, whose doc records the resulting worst-case `serve` latency. `const fn device_policy_name` sits immediately after this cluster's last function in the original file but stayed in `main.rs` rather than moving — it already had callers outside the cluster (`engine_ipc.rs`, `service_launch.rs`, `serve_cmd.rs`) plus its own test. `GpuVramUsage` (and its fields), `gpu_vram_usage`, `parse_gpu_indices_arg`, `serve_gpu_low_memory_warning`, and `vram_capacity_is_meaningful` are `pub(crate)` and read from [`model_fit.rs`](../apps/rocm/src/model_fit.rs) (`classify_accelerator_memory`, `host_accelerator_memory`) [`main.rs`](../apps/rocm/src/main.rs) (the `dispatch()` `Command::EngineServeHttp` arm) and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) (`collect_serve_notes`); `detect_gpu_count`, `gpu_vram_usage`, `parse_device_policy`, `parse_gpu_selection`, `select_gpu_indices_under_launch_lock`, `serve_gpu_low_memory_warning`, and `validate_pinned_gpu_index` are also `pub(crate)` and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs); `parse_device_policy` is also read from [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs)'s `run_builtin_engine_serve_http` and, as of Phase 6f (ROCMAI-209), from [`service_control.rs`](../apps/rocm/src/service_control.rs)'s `restart_child_argv` (moved out of `main.rs` in that same phase, and since relocated from `services_text.rs`). `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs` (both the Linux and non-Linux `cfg` variants), `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`, `gpu_low_memory_warning`, and `busy_gpu_indices` have no callers outside this module and stay private. The cluster's regression tests live in `gpu_policy.rs`'s own `mod tests`; two small test-only fixture builders (`vram`, `host_gpu`) are duplicated rather than shared, since `serve_cmd.rs`'s own `serve_notes_pair_low_vram_with_the_vllm_utilization_hint` (testing `collect_serve_notes`) needs them too. - [`install_cmd.rs`](../apps/rocm/src/install_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm install`'s command handler (`install()`, for the `sdk` and `driver` targets), the `--version`/`--build-date` selector parsing and display it uses (`therock_install_version_selector`, `therock_install_version_selector_display`) and `sdk_install_request`, which maps the parsed `install sdk` arguments onto a `therock::SdkInstallRequest` and moved here from [`chat.rs`](../apps/rocm/src/chat.rs) with its two tests. `InstallTarget` and `InstallFormat` live in [`cli.rs`](../apps/rocm/src/cli.rs). `install()` is `pub(crate)` and re-imported in `main.rs` for `dispatch()`; `therock_install_version_selector` is `pub(crate)` too, and `chat/mcp_bridge.rs`'s `render_install_sdk_dry_run_for_args` reaches it as `crate::therock_install_version_selector` through the same kind of re-import, which is why `sdk_install_request` lives here and not in `chat.rs`: this module does not depend on `chat.rs`. `sdk_install_request` keeps the `pub(crate)` it had, although only `install()` and the tests call it now. - [`logging.rs`](../apps/rocm/src/logging.rs) — mechanical relocation, no owned types, reached by qualified path. -- [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-211): managed-service liveness classification (`managed_service_is_live`, `refresh_managed_service_runtime_liveness`, `existing_live_managed_service_in`'s idempotency guard) and the dashboard-sidebar status/counts helpers derived from it (`ManagedServiceSidebarCounts`/`managed_service_sidebar_counts`/`local_server_sidebar_status`), plus the engine display-name helper `friendly_engine_label`, read only by [`chat/summaries.rs`](../apps/rocm/src/chat/summaries.rs). `ManagedServiceSidebarCounts` is `pub(crate)` to match the visibility of the `pub(crate)` functions that return/take it; `friendly_engine_label`, `existing_live_managed_service_in`, `recorded_service_pids`, `refresh_managed_service_runtime_liveness`, `local_server_sidebar_status`, `managed_service_sidebar_counts`, and `load_managed_services` are `pub(crate)` for the same reason — each has a confirmed caller outside this module. [`service_records.rs`](../apps/rocm/src/service_records.rs) uses its liveness helpers (`managed_service_is_live`, `recorded_service_pids`) for the in-use check behind service-record removal and pruning, both of which, like the deferred key cleanup and the reconciliation to `stopped` here, take a recorded process for gone only when `probe_process` confirms it (a process that cannot be queried counts as possibly running), and `load_managed_services` for prune planning, while [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs) uses `load_managed_services` for the codex-bridge snapshot; [`service_launch.rs`](../apps/rocm/src/service_launch.rs) is the only caller of the `existing_live_managed_service_in` duplicate-launch guard; [`services_text.rs`](../apps/rocm/src/services_text.rs) reads the sidebar helpers; other modules, including [`chat/turn.rs`](../apps/rocm/src/chat/turn.rs), [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs), read the registry and liveness helpers. +- [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-211): managed-service liveness classification (`managed_service_is_live`, `refresh_managed_service_runtime_liveness`, `existing_live_managed_service_in`'s idempotency guard) and the dashboard-sidebar status/counts helpers derived from it (`ManagedServiceSidebarCounts`/`managed_service_sidebar_counts`/`local_server_sidebar_status`), plus the engine display-name helper `friendly_engine_label`, read only by [`chat/summaries.rs`](../apps/rocm/src/chat/summaries.rs). `ManagedServiceSidebarCounts` is `pub(crate)` to match the visibility of the `pub(crate)` functions that return/take it; `friendly_engine_label`, `existing_live_managed_service_in`, `recorded_service_pids`, `refresh_managed_service_runtime_liveness`, `local_server_sidebar_status`, `managed_service_sidebar_counts`, and `load_managed_services` are `pub(crate)` for the same reason — each has a confirmed caller outside this module. [`service_records.rs`](../apps/rocm/src/service_records.rs) uses its liveness helpers (`managed_service_is_live`, `recorded_service_pids`) for the in-use check behind service-record removal and pruning, both of which, like the deferred key cleanup and the reconciliation to `stopped` here, take a recorded process for gone only when `probe_process` confirms it (a process that cannot be queried counts as possibly running; when the deferred cleanup does complete, it also forgets the recorded processes and reads the record as stopped, so a later stop cannot signal a recycled PID), and `load_managed_services` for prune planning, while [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs) uses `load_managed_services` for the codex-bridge snapshot; [`service_launch.rs`](../apps/rocm/src/service_launch.rs) is the only caller of the `existing_live_managed_service_in` duplicate-launch guard; [`services_text.rs`](../apps/rocm/src/services_text.rs) reads the sidebar helpers; other modules, including [`chat/turn.rs`](../apps/rocm/src/chat/turn.rs), [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs), read the registry and liveness helpers. - [`model_catalog_text.rs`](../apps/rocm/src/model_catalog_text.rs) — mechanical relocation out of `examine_report_text.rs`: the `rocm model` (alias `models`) catalog renderer. `render_model_registry_text_with_context_and_host` prints the curated list grouped by hardware target (`model_catalog_platform_heading`, `model_catalog_header`, `model_recipe_memory_label`, `model_recipe_gpu_fit_label`), and `render_model_registry_verbose_text_with_context_and_host` prints the per-recipe diagnostics behind `rocm model --verbose` (`append_model_recipe_registry_source`, `append_model_recipe_metadata_lines`, `append_model_engine_recipe_settings_lines`, `append_model_recipe_staleness_lines`, `append_model_artifact_lines`, `append_artifact_source_policy_lines`, `artifact_source_policy_label`, `append_model_host_ram_fit_lines`, `append_model_fit_lines`, `append_manual_alternative_lines`, `manual_alternative_recommendations`, `recipe_is_manual_fit`, `append_model_engine_support_lines`, `model_registry_adapter_availability_note`, `recipe_display_ref`, `preferred_engine_action_target`, `model_registry_reason`, plus the small formatters `format_list_or_none`, `format_string_map_or_none`, `format_gib` and `once_per_call`). `format_bytes`, the human-readable size formatter, lives here too. The two renderers and `format_bytes` are `pub(crate)`: [`main.rs`](../apps/rocm/src/main.rs) imports them for the `Command::Model` arm, and [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs) (the verbose renderer), [`storage.rs`](../apps/rocm/src/storage.rs) and [`automations.rs`](../apps/rocm/src/automations.rs) (`format_bytes`) reach them as `crate::`. Every other item, including `model_registry_adapter_availability_note`, is private. The module has no edge to `examine_report_text.rs`; it reaches the engine-availability helpers in [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs), `PlatformRuleOut` and `engine_ruled_out_by_platform_with` at the crate root, and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs)'s `select_serve_engine`. The catalog's regression tests live in this module's own `mod tests`; they keep the names they had in `examine_report_text.rs`. The `format_bytes` property test's saved failure case lives in [`model_catalog_text.txt`](../apps/rocm/proptest-regressions/model_catalog_text.txt), which proptest finds by the test's source file, so the saved case has to follow the test when it moves (see [Property tests](testing.md#property-tests) for how, which depends on whether the whole file moves). - [`model_fit.rs`](../apps/rocm/src/model_fit.rs) — mechanical relocation, owns a `pub(crate)` type: answering whether a host can run a model for `rocm diagnose --model` (`assess_model_on_this_host`, `assess_model_with_host_paths`, `assess_model_for_host`), the accelerator-memory fact it rests on (`host_accelerator_memory`, `classify_accelerator_memory`, `examination_reports_amd_gpu`) and the platform gate that rules an engine out (`PlatformRuleOut`, `engine_ruled_out_by_platform`, `unsupported_here_for`). It is a module of its own, rather than part of the `diagnose` handler, so [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs) and [`model_catalog_text.rs`](../apps/rocm/src/model_catalog_text.rs), which read `PlatformRuleOut` and the platform gate (`engine_ruled_out_by_platform` and `engine_ruled_out_by_platform_with` respectively), do not depend on a command module. It depends on [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) (`select_serve_engine`) and [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs), neither of which depends on it. `assess_model_on_this_host` (called from [`diagnose_cmd.rs`](../apps/rocm/src/diagnose_cmd.rs)), `PlatformRuleOut` and `engine_ruled_out_by_platform` are `pub(crate)`, and [`main.rs`](../apps/rocm/src/main.rs) re-imports the last two so `examine_report_text.rs` reaches them as `crate::` (`model_catalog_text.rs` reaches `PlatformRuleOut` and `engine_ruled_out_by_platform_with` the same way); every other item is private. The cluster's regression tests live in this module's own `mod tests`; the one that re-runs the test binary on a helper test by its full path (`engine_ruled_out_by_platform_gates_aim_on_docker_reachability`) names the helper's new path, `model_fit::tests::docker_unreachable_subprocess_helper`. - [`prompt_text.rs`](../apps/rocm/src/prompt_text.rs) — text extraction from a natural-language request, shared by `chat/intent.rs`'s prompt-intent heuristics (`fallback_rocm_tool_call_for_prompt`, `fallback_config_tool_call_for_prompt`, `install_sdk_chat_approval_for_prompt` and the `prompt_*` predicates) and the freeform planner, in a leaf module that depends only on `therock::RuntimeVersionSelector` (`tool_policy.rs` does not use it): `any_substring`, the install-folder, TheRock build-date and TheRock version readers (`requested_install_prefix_from_prompt`, `requested_therock_build_date_from_prompt`, `requested_therock_version_from_prompt`, with `requested_bare_install_prefix_from_prompt`, `clean_requested_install_prefix`, `looks_like_user_path` and `looks_like_therock_runtime_version` private beside them) from `chat.rs`, and `infer_engine_from_request` from `freeform_planner.rs`. They moved verbatim, together with `infer_engine_from_request`'s regression test, so that `freeform_planner.rs` no longer imports `chat` for them; `chat.rs` re-exports `any_substring`, which [`freeform_answer.rs`](../apps/rocm/src/freeform_answer.rs) still reaches as `chat::any_substring`. diff --git a/docs/testing.md b/docs/testing.md index 8b5f321718..40b27441d0 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2248,6 +2248,17 @@ drives the refresh with an unqueryable PID and then a confirmed-gone one, an unverified stop and then the refresh over the record it wrote, and `a_record_is_in_use_unless_every_recorded_process_is_confirmed_gone` covers the removal guard; each takes the probe as an argument, so they run on every host. +When that refresh completes a pending stop (every recorded process confirmed gone) it +also forgets the recorded processes and reads the record as stopped, as a stop that +confirmed the exit itself does, so a later `rocm services stop` visits no PID: a PID +recorded without a token cannot be refuted, and on Windows the tree kill would take +whatever process now holds the number. A process that may still be running stays +named. `a_pending_stop_keeps_its_key_and_marker_while_a_process_cannot_be_queried` +checks both halves (named while unqueryable, forgotten once gone), +`a_completed_pending_stop_reads_stopped_whatever_live_status_it_had` that every live +status still ends as `stopped` in that one refresh, and +`a_stop_completed_by_a_later_listing_leaves_no_pid_for_the_next_stop` runs the real +termination routine over the record the listing wrote and asserts it visits nothing. Three gaps in the scenario coverage are deliberate. `service-stop-07` records no token, so the path a record written by this version takes on Windows (a token recorded, matching, with the tree kill then running) has no scenario; it rests on From 86737e256f272924bc7c6cd2cc6bc0a72a6aa8df Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Tue, 10 Nov 2026 00:49:53 +0000 Subject: [PATCH 21/30] test(e2e): fail the Windows stop scenario when tasklist fails instead of reading it as an exit The check behind the service-stop tree-kill scenario asked tasklist whether a process was still running and read a missing CSV row as an exit. The command's output is collected even when it exits nonzero, so a tasklist that failed (refused, could not be run as expected) printed no row and the scenario concluded the process had exited, passing while the process might still be running. The read is now a pure function of the exit status and the output: a successful run with a row for the PID is running, a successful run without one is not, and a run that did not exit successfully is an error that the step turns into a failed scenario, naming the PID and what the run printed. A table test covers each case on every host, including the failed runs that print nothing and the ones that print a row or an INFO line. Signed-off-by: Jussi Elo --- docs/testing.md | 6 +- tests/e2e-cucumber/src/lib.rs | 1 + tests/e2e-cucumber/src/tasklist.rs | 83 +++++++++++++++++++ .../tests/e2e/service_stop_steps.rs | 12 ++- 4 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 tests/e2e-cucumber/src/tasklist.rs diff --git a/docs/testing.md b/docs/testing.md index 40b27441d0..74c5ea4cfd 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2278,7 +2278,11 @@ service listing confirms and drops the key. The command is a stock one the host has, pinned by `the_process_check_command_is_the_one_the_host_has`, and `service-stop-07` uses the same `tasklist /FI "PID eq "` filter (in CSV form, which does not depend on the display language) to see whether a process is -running; no test follows the printed command on a Windows host and reads its +running. A `tasklist` that does not exit successfully is no answer: the scenario +fails instead of reading the missing row as an exit +(`tasklist_reports_running` in `tests/e2e-cucumber/src/tasklist.rs`, a pure read of +the status and output covered by `a_failed_run_is_no_answer_whatever_it_printed` on +every host). No test follows the printed command on a Windows host and reads its default table output, because that output is localised. What follows once the process is gone is covered by `a_pending_stop_keeps_its_key_and_marker_while_a_process_cannot_be_queried`. diff --git a/tests/e2e-cucumber/src/lib.rs b/tests/e2e-cucumber/src/lib.rs index dcbbed1ea2..d7571d43ee 100644 --- a/tests/e2e-cucumber/src/lib.rs +++ b/tests/e2e-cucumber/src/lib.rs @@ -30,6 +30,7 @@ pub mod serve_log; pub mod serve_plan; pub mod shared_runtime; pub mod stray_process; +pub mod tasklist; pub mod terminal_size; use std::path::{Path, PathBuf}; diff --git a/tests/e2e-cucumber/src/tasklist.rs b/tests/e2e-cucumber/src/tasklist.rs new file mode 100644 index 0000000000..35aea69c67 --- /dev/null +++ b/tests/e2e-cucumber/src/tasklist.rs @@ -0,0 +1,83 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! What `tasklist` says about one process, read so that a failed query cannot +//! pass for an answer. +//! +//! A scenario that wants to know whether a process a stop was meant to take down +//! is still running asks `tasklist /FI "PID eq N" /FO CSV /NH`. A match prints a +//! CSV row naming the PID and no match prints an INFO line without one, but the +//! command can also fail (it cannot be started under that name, it is refused, it +//! times out) and print nothing useful. Reading "no row" from such a run as "the +//! process exited" lets the scenario pass while the process may still be running, +//! so the exit status is judged before the text. The read is a pure function of +//! the command's status and output, so a table test covers every case on every +//! host. + +/// Whether `tasklist` reported process `pid` running. +/// +/// `success` is whether the command exited successfully and `stdout` is what it +/// printed. `Ok(true)` is a CSV row for `pid`, `Ok(false)` is a successful run +/// with no such row, and `Err` is a run that did not succeed: its output says +/// nothing about the process, so there is no answer. +/// +/// # Errors +/// +/// Returns a message naming the PID and what the failed run printed when the +/// command did not exit successfully. +pub fn tasklist_reports_running( + success: bool, + stdout: &str, + stderr: &str, + pid: u32, +) -> Result { + if !success { + return Err(format!( + "`tasklist` did not exit successfully, so it cannot say whether process {pid} \ + is still running (stdout: {:?}, stderr: {:?})", + stdout.trim(), + stderr.trim() + )); + } + Ok(stdout.contains(&format!("\"{pid}\""))) +} + +#[cfg(test)] +mod tests { + use super::*; + + const ROW: &str = "\"ping.exe\",\"4242\",\"Console\",\"1\",\"3,044 K\"\r\n"; + const NO_MATCH: &str = "INFO: No tasks are running which match the specified criteria.\r\n"; + + #[test] + fn a_successful_run_with_a_row_for_the_pid_is_running() { + assert_eq!(tasklist_reports_running(true, ROW, "", 4242), Ok(true)); + } + + #[test] + fn a_successful_run_without_a_row_for_the_pid_is_not_running() { + assert_eq!( + tasklist_reports_running(true, NO_MATCH, "", 4242), + Ok(false) + ); + // A row for another PID is not a row for this one, nor is a PID that + // only contains it. + assert_eq!(tasklist_reports_running(true, ROW, "", 424), Ok(false)); + assert_eq!(tasklist_reports_running(true, ROW, "", 7), Ok(false)); + } + + #[test] + fn a_failed_run_is_no_answer_whatever_it_printed() { + // No output to read: "no row" must not become "exited". + let err = tasklist_reports_running(false, "", "Access is denied.", 4242) + .expect_err("a failed query is not an answer"); + assert!( + err.contains("4242") && err.contains("Access is denied."), + "{err}" + ); + // Nor does a row printed before the failure make it one. + assert!(tasklist_reports_running(false, ROW, "", 4242).is_err()); + assert!(tasklist_reports_running(false, NO_MATCH, "", 4242).is_err()); + } +} diff --git a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs index d1a2530ddb..2b0e13073f 100644 --- a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs @@ -302,13 +302,21 @@ fn wait_for_descendant_pid(pid_file: &std::path::Path) -> u32 { } /// Whether Windows has a process `pid`, by `tasklist`: a match prints a CSV -/// row naming it, and no match prints an INFO line without one. +/// row naming it, and no match prints an INFO line without one. A `tasklist` +/// that fails has said nothing about the process, so the scenario fails rather +/// than read it as "exited" (`e2e_cucumber::tasklist`). fn windows_process_is_running(pid: u32) -> bool { let output = Command::new("tasklist") .args(["/FI", &format!("PID eq {pid}"), "/FO", "CSV", "/NH"]) .output() .expect("failed to run tasklist"); - String::from_utf8_lossy(&output.stdout).contains(&format!("\"{pid}\"")) + e2e_cucumber::tasklist::tasklist_reports_running( + output.status.success(), + &String::from_utf8_lossy(&output.stdout), + &String::from_utf8_lossy(&output.stderr), + pid, + ) + .unwrap_or_else(|message| panic!("{message}")) } /// Write the record for a ready server whose supervisor is `process`, with From 44a4afdc11bcc0b2bb8b0fcd61e18278a34c38c8 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Tue, 10 Nov 2026 01:52:01 +0000 Subject: [PATCH 22/30] fix(services): count a stop that a listing completed under it as the same stop A stop reads the record, terminates what it names, then re-reads the record and compares the processes it names with the ones it signalled, to tell its own record from a restart's. The deferred cleanup now clears the recorded processes, so a listing that saw every process gone while the stop was still finalizing completed the same stop and left a record naming none. The comparison failed in both rocm and rocmd, and a successful termination was reported as unconfirmed with the stop marker put back, which made services restart refuse. Both finalizations now ask ManagedServiceRecord::is_covered_by_stop_of: a record is covered when it names the same processes, or when it reads stopped and names none, which is what the deferred cleanup of that same stop leaves. A restart is not covered, whether it recorded new processes or is mid-launch with none recorded yet, since that record reads starting and not stopped, so the key it will serve with is kept and its status is not overwritten. The regression tests persist a real liveness refresh between the termination and the final read and assert a confirmed stop with the marker not restored, and that a restart carries on past its own stop that a listing completed. Tests for a relaunch with new processes, and for one mid-launch with nothing recorded, show a real relaunch is still detected, in rocm and in rocmd, and a table covers every record shape. Signed-off-by: Jussi Elo --- apps/rocm/src/service_control.rs | 123 ++++++++++++++++++++++++- apps/rocmd/src/service.rs | 77 +++++++++++++++- crates/rocm-core/src/service_record.rs | 99 ++++++++++++++++++++ docs/testing.md | 18 +++- 4 files changed, 314 insertions(+), 3 deletions(-) diff --git a/apps/rocm/src/service_control.rs b/apps/rocm/src/service_control.rs index a3e48edead..ed296479b0 100644 --- a/apps/rocm/src/service_control.rs +++ b/apps/rocm/src/service_control.rs @@ -252,7 +252,7 @@ fn stop_internal_managed_service_confirmed_with( // supervisor that follows it, so a failed `Stop` leaves the stop unconfirmed. let engine_stop_confirms = !rocm_core::engine_stop_decides_confirmation(&stopping.engine) || engine_stop.is_ok(); - let record_changed = !stopping.names_same_processes(&record); + let record_changed = !record.is_covered_by_stop_of(&stopping); let all_stopped = terminated_all && !record_changed && engine_stop_confirms; if all_stopped { record.status = "stopped".to_owned(); @@ -1118,6 +1118,127 @@ mod tests { assert_eq!(key.as_deref(), Some("key-123"), "the key is kept"); } + /// A liveness refresh that lands after the stop's processes have exited but + /// before the stop reads the record again completes the same stop (key, + /// marker, recorded processes cleared, `stopped`) and writes it. That record + /// names no process, which must not read as a restart: the stop is confirmed, + /// the marker is not put back, and a restart is not refused for it. + #[test] + fn a_listing_that_completes_the_stop_before_finalization_does_not_unconfirm_it() { + use rocm_core::TerminationOutcome::Graceful; + let (root, paths) = test_paths("stop-listing-before-finalization"); + paths.ensure().unwrap(); + let service_id = "svc-stop-listing-before-finalization"; + let mut planted = stop_test_record(&paths, service_id); + planted.supervisor_pid = 4242; + planted.supervisor_start_ticks = None; + planted.engine_pid = Some(4243); + planted.engine_start_ticks = Some(77); + planted.write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |stopping| { + assert!(stopping.stop_requested_unix_ms.is_some()); + // The processes have exited; a concurrent `services` listing sees it, + // completes the pending stop and persists that, before this stop + // reads the record again. + let (mut listed, _) = read_managed_service_manifest(&paths, service_id).unwrap(); + let changed = + crate::managed_service_status::refresh_managed_service_runtime_liveness_with( + &paths, + &mut listed, + |_| rocm_core::ProcessProbe::Gone, + ); + assert!(changed, "premise: the listing completed the stop"); + listed.write().unwrap(); + assert_eq!(raw_manifest(&paths, service_id).supervisor_pid, 0); + vec![stop_of(4242, Graceful), stop_of(4243, Graceful)] + }) + .unwrap(); + + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + assert_eq!(report["stopped"], true, "{report}"); + assert_eq!(on_disk.status, "stopped"); + assert_eq!( + on_disk.stop_requested_unix_ms, None, + "a confirmed stop does not put the marker back" + ); + assert_eq!(on_disk.supervisor_pid, 0); + assert_eq!(key, None, "the key goes with the confirmed stop"); + } + + /// The same race through `restart`: the restart's own stop is completed by a + /// listing before it finalizes, and the restart must carry on past the stop + /// (here to the launch lock, which a holder keeps, so it stops there) + /// instead of refusing with "could not confirm". + #[test] + fn a_restart_is_not_refused_because_a_listing_completed_its_stop() -> Result<()> { + use rocm_core::TerminationOutcome::Graceful; + let (root, paths) = test_paths("restart-listing-before-finalization"); + paths.ensure()?; + let service_id = "svc-restart-listing-before-finalization"; + stop_test_record(&paths, service_id).write()?; + let holder = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + + LAUNCH_LOCK_TIMEOUT_OVERRIDE.with(|cell| cell.set(Some(Duration::from_millis(100)))); + let result = restart_internal_managed_service_with(&paths, service_id, None, |_| { + let (mut listed, _) = read_managed_service_manifest(&paths, service_id).unwrap(); + let _ = crate::managed_service_status::refresh_managed_service_runtime_liveness_with( + &paths, + &mut listed, + |_| rocm_core::ProcessProbe::Gone, + ); + listed.write().unwrap(); + vec![stop_of(999_999_999, Graceful)] + }); + LAUNCH_LOCK_TIMEOUT_OVERRIDE.with(|cell| cell.set(None)); + drop(holder); + let _ = fs::remove_dir_all(root); + + let message = format!( + "{:#}", + result.expect_err("the held launch lock fails the restart") + ); + assert!( + !message.contains("could not confirm"), + "the stop was completed, the restart must not be refused for it: {message}" + ); + assert!(message.contains("stopped but NOT restarted"), "{message}"); + Ok(()) + } + + /// The other half of the same check: a restart that has begun launching but + /// has recorded no process yet (status `starting`, nothing named) is not the + /// stop completing, and the stop must not claim it or drop the key it will + /// serve with. + #[test] + fn stop_does_not_confirm_over_a_restart_that_has_recorded_no_process_yet() { + let (root, paths) = test_paths("stop-restart-in-flight"); + paths.ensure().unwrap(); + let service_id = "svc-stop-restart-in-flight"; + stop_test_record(&paths, service_id).write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + let mut restarting = raw_manifest(&paths, service_id); + restarting.status = "starting".to_owned(); + restarting.clear_recorded_processes(); + restarting.stop_requested_unix_ms = None; + restarting.write().unwrap(); + bare_termination(Vec::new(), true) + }) + .unwrap(); + + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + assert_eq!(report["status"], "starting", "{report}"); + assert_eq!(on_disk.status, "starting", "the restart's status stays"); + assert_eq!(key.as_deref(), Some("key-123"), "the key is kept"); + } + #[test] fn stop_of_a_container_engine_is_unconfirmed_when_the_engine_stop_fails() { // For `aim` the recorded PID is only the supervisor; killing it says diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index b259df59d0..caf05aa48e 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -106,7 +106,7 @@ pub(crate) fn stop_managed_service_via( let engine_stop_confirms = !rocm_core::engine_stop_decides_confirmation(&stopping.engine) || engine_stop.as_ref().is_some_and(Result::is_ok); let all_stopped = stops.iter().all(|stop| stop.stopped()) - && stopping.names_same_processes(&record) + && record.is_covered_by_stop_of(&stopping) && engine_stop_confirms; // A PID whose identity cannot be read at all — `/proc//stat` // unreadable while `kill(pid, 0)` still answers: another uid, `hidepid`, an @@ -855,6 +855,81 @@ mod tests { Ok(()) } + /// A `rocm` listing that sees every process gone completes the pending stop + /// and persists it (marker and key cleared, recorded processes forgotten, + /// `stopped`), possibly before this stop reads the record again. That record + /// names no process, which is not a restart: the stop is still confirmed and + /// the marker is not put back. The write below is exactly the one the + /// listing's cleanup makes (it lives in the `rocm` binary, which this crate + /// cannot call; `rocm` has the test that runs the real refresh). + #[test] + fn a_stop_completed_by_a_concurrent_listing_is_still_confirmed() -> Result<()> { + let (root, paths) = temp_app_paths("stop-listing-completed-it"); + paths.ensure()?; + let service_id = "svc-stop-listing-completed-it"; + let key_path = seed_keyed_service(&paths, service_id, 11455)?; + + let result = stop_managed_service_with(&paths, service_id, |_| { + let mut on_disk = load_service_record(&paths, service_id).expect("record"); + on_disk.clear_recorded_processes(); + on_disk.stop_requested_unix_ms = None; + on_disk.status = "stopped".to_owned(); + on_disk.write().expect("concurrent listing write"); + rocm_core::TerminationOutcome::Graceful + }); + let key_removed = !key_path.exists(); + let reloaded = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let value = result?; + assert_eq!( + value.get("stopped").and_then(Value::as_bool), + Some(true), + "{value}" + ); + let reloaded = reloaded?; + assert_eq!(reloaded.status, "stopped"); + assert_eq!( + reloaded.stop_requested_unix_ms, None, + "a confirmed stop does not put the marker back" + ); + assert!(key_removed); + Ok(()) + } + + /// A restart that has begun launching but recorded no process yet reads + /// `starting` with nothing named: not the stop completing, so the stop does + /// not claim it or drop the key. + #[test] + fn a_stop_does_not_claim_a_restart_that_has_recorded_no_process_yet() -> Result<()> { + let (root, paths) = temp_app_paths("stop-restart-in-flight"); + paths.ensure()?; + let service_id = "svc-stop-restart-in-flight"; + let key_path = seed_keyed_service(&paths, service_id, 11456)?; + + let result = stop_managed_service_with(&paths, service_id, |_| { + let mut on_disk = load_service_record(&paths, service_id).expect("record"); + on_disk.clear_recorded_processes(); + on_disk.stop_requested_unix_ms = None; + on_disk.status = "starting".to_owned(); + on_disk.write().expect("concurrent restart write"); + rocm_core::TerminationOutcome::Graceful + }); + let key_kept = key_path.exists(); + let reloaded = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let value = result?; + assert_eq!( + value.get("stopped").and_then(Value::as_bool), + Some(false), + "{value}" + ); + assert_eq!(reloaded?.status, "starting"); + assert!(key_kept, "the restarting service's key must be kept"); + Ok(()) + } + /// A launcher that is also the server is recorded under both roles. The /// stop handles that PID once, and must report it under both. #[test] diff --git a/crates/rocm-core/src/service_record.rs b/crates/rocm-core/src/service_record.rs index d46bb656e6..cc4e27017f 100644 --- a/crates/rocm-core/src/service_record.rs +++ b/crates/rocm-core/src/service_record.rs @@ -330,6 +330,35 @@ impl ManagedServiceRecord { && self.engine_start_ticks == other.engine_start_ticks } + /// Whether `self`, the record as it is now, is still the record of the + /// service that `stopping`, the copy the stop read before it signalled + /// anything, was stopping: the stop's verdict covers what it names. + /// + /// True when the two name the same processes + /// ([`Self::names_same_processes`]), and also when `self` names none because + /// the same stop was completed under it: a liveness refresh that saw every + /// recorded process gone, with the stop requested, clears the processes and + /// reads the record as `stopped` (the deferred cleanup). A record reading + /// `stopped` that names no process has nothing left that the stop did not + /// handle, so the stop's own confirmation is still true of it. + /// + /// False for a restart that recorded new processes, and equally for one that + /// is mid-launch with none recorded yet: that record reads `starting` (or + /// another live status), not `stopped`, and claiming it stopped, or dropping + /// the key it will serve with, would be false. + #[must_use] + pub fn is_covered_by_stop_of(&self, stopping: &Self) -> bool { + self.names_same_processes(stopping) || (self.status == "stopped" && self.names_no_process()) + } + + /// Whether the record names neither a supervisor nor an engine process. + const fn names_no_process(&self) -> bool { + self.supervisor_pid == 0 + && self.supervisor_start_ticks.is_none() + && self.engine_pid.is_none() + && self.engine_start_ticks.is_none() + } + /// Record `pid` as the engine, together with its start-time token. See /// [`Self::record_supervisor_identity`]. pub fn record_engine_identity(&mut self, pid: u32) { @@ -851,6 +880,76 @@ mod tests { assert!(record.names_same_processes(&record.clone())); } + /// What a stop's verdict still covers, record by record: the same processes, + /// or none because the same stop was completed under it (`stopped`, nothing + /// recorded). A restart's record is never covered: new processes recorded, or + /// still starting with none recorded yet. + #[test] + fn a_stop_is_covered_by_the_same_processes_or_by_its_own_completion_and_not_by_a_relaunch() { + let mut stopping = probe_test_record(11438); + stopping.supervisor_pid = 4242; + stopping.supervisor_start_ticks = None; + stopping.engine_pid = Some(4243); + stopping.engine_start_ticks = Some(77); + stopping.status = "ready".to_owned(); + let record_with = |status: &str, pid: u32, engine: Option| { + let mut record = stopping.clone(); + status.clone_into(&mut record.status); + record.supervisor_pid = pid; + record.supervisor_start_ticks = None; + record.engine_pid = engine; + record.engine_start_ticks = None; + record + }; + // (what the record on disk is now, whether the stop still covers it) + let rows = [ + ("unchanged", stopping.clone(), true), + // The deferred cleanup of this very stop: every process gone, so the + // record is stopped and names none. + ( + "completed under the stop", + record_with("stopped", 0, None), + true, + ), + // A restart that recorded new processes. + ( + "relaunched, ready", + record_with("ready", 5555, Some(5556)), + false, + ), + ( + "relaunched, starting", + record_with("starting", 5555, None), + false, + ), + // A restart mid-launch that has not recorded anything yet: not a stop. + ( + "restarting, nothing recorded", + record_with("starting", 0, None), + false, + ), + ( + "recovering, nothing recorded", + record_with("recovering", 0, None), + false, + ), + ( + "ready, nothing recorded", + record_with("ready", 0, None), + false, + ), + // Stopped, but naming a process the stop did not handle. + ( + "stopped, other process", + record_with("stopped", 5555, None), + false, + ), + ]; + for (name, record, covered) in rows { + assert_eq!(record.is_covered_by_stop_of(&stopping), covered, "{name}"); + } + } + /// Writing a service manifest never writes through a file another writer /// already staged next to it. The occupied names are the ones the retired /// clock-and-pid temp naming would have picked, so a write that still used diff --git a/docs/testing.md b/docs/testing.md index 74c5ea4cfd..cd17987a66 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2311,7 +2311,23 @@ verdict to the JSON; `rocm services stop` exits 1. The stop records the request in the manifest before it signals anything, and applies its result to the manifest as it is once the stop ends, so a write made meanwhile is kept — and a restart that recorded new processes meanwhile makes the stop -unconfirmed rather than having its PIDs cleared. +unconfirmed rather than having its PIDs cleared. What the stop's verdict still +covers is decided by `ManagedServiceRecord::is_covered_by_stop_of`: the same +processes, or none recorded because a liveness refresh that saw every process +gone completed this same stop first (the record reads `stopped` and names +nothing). A restart, whether it has recorded new processes or is mid-launch with +none yet (status `starting`), is not covered. The unit test +`a_stop_is_covered_by_the_same_processes_or_by_its_own_completion_and_not_by_a_relaunch` +is the table; in `rocm`, `a_listing_that_completes_the_stop_before_finalization_does_not_unconfirm_it` +persists a real liveness refresh between the termination and the final read and +asserts a confirmed stop with the marker not restored, +`a_restart_is_not_refused_because_a_listing_completed_its_stop` that a restart then +carries on past its stop, and `stop_does_not_confirm_over_a_restart_that_has_recorded_no_process_yet` +and `stop_does_not_confirm_when_a_restart_recorded_new_processes_meanwhile` that a real +relaunch is still detected; `rocmd` has the same three (`a_stop_completed_by_a_concurrent_listing_is_still_confirmed`, +`a_stop_does_not_claim_a_restart_that_has_recorded_no_process_yet`, +`a_stop_does_not_clear_processes_recorded_while_it_ran`), the first with the listing's +write made directly because the refresh lives in the `rocm` binary. `features/service_stop.feature` (`service-stop-02`, Linux, mock lane) drives the real `rocmd sandbox-tool stop_server` against a live recorded From ceeac3dd8da643ce6bc513efa78e0914527f1c0c Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Tue, 10 Nov 2026 12:38:09 +0000 Subject: [PATCH 23/30] fix(services): clean up a failed launch's child through the shared termination routine The failed-launch cleanup terminated the spawned child with terminate_verified directly. On Windows that cannot walk a process tree, so a failed launch could leave the engine one level below the supervisor running: the defect the shared routine fixed for stop. The cleanup now goes through the same routine a stop uses. rocm-core gains terminate_recorded_pid_within, which is terminate_recorded_pid with the root's grace chosen by the caller, so the cleanup keeps its two second grace for a child that is milliseconds old and has nothing to flush. The identity rule is unchanged and still decided first: a child whose start token was not captured, on a host that records tokens, is not signalled. The termination is supplied through a seam, and a test that runs on every host checks that the cleanup asks about the child's PID with the token captured at spawn, asks about nothing for a child with no token, and marks the record stopped either way. That the production wiring reaches the Windows tree kill is not observable on Linux, where the termination walks the tree itself; it rests on the routine's own tests. Signed-off-by: Jussi Elo --- .../src/service_launch/launch_transaction.rs | 85 ++++++++++++++++--- crates/rocm-core/src/lib.rs | 3 +- crates/rocm-core/src/proc_lifecycle.rs | 15 +++- docs/architecture.md | 2 +- 4 files changed, 92 insertions(+), 13 deletions(-) diff --git a/apps/rocm/src/service_launch/launch_transaction.rs b/apps/rocm/src/service_launch/launch_transaction.rs index abec884c91..4fb530a5b8 100644 --- a/apps/rocm/src/service_launch/launch_transaction.rs +++ b/apps/rocm/src/service_launch/launch_transaction.rs @@ -144,6 +144,23 @@ const fn launch_cleanup_refusal( /// Every step is best-effort and idempotent; cleanup must never mask the /// original failure with a second one. pub(super) fn abandon_failed_managed_launch(paths: &AppPaths, record: &ManagedServiceRecord) { + // The shared routine `rocm services stop` and `rocmd` use, not + // `terminate_verified` alone: on Windows only the routine takes the + // processes under the child down, so a failed launch cannot leave the engine + // one level below the supervisor running. + abandon_failed_managed_launch_with(paths, record, |identity| { + rocm_core::terminate_recorded_pid_within(identity, MANAGED_LAUNCH_CLEANUP_GRACE) + }); +} + +/// [`abandon_failed_managed_launch`] with the termination of the child supplied +/// by the caller, so a test sees what the cleanup asks it to terminate, and +/// whether it asks at all. +fn abandon_failed_managed_launch_with( + paths: &AppPaths, + record: &ManagedServiceRecord, + terminate: impl Fn(&rocm_core::ProcessIdentity) -> rocm_core::TerminationOutcome, +) { if let Some(pid) = record.engine_pid.filter(|pid| *pid != 0) { // The token captured at spawn, not a fresh `capture`: when the child // already exited, a fresh one would describe a recycled pid's new owner @@ -152,7 +169,7 @@ pub(super) fn abandon_failed_managed_launch(paths: &AppPaths, record: &ManagedSe // missing one means the child was gone before it could be captured, so // the pid is unverifiable and nothing is signalled (`terminate_verified` // alone would "best-effort proceed"). A token that was captured is - // verified by `terminate_verified` on both. + // verified by the termination on both. // // The tradeoff on Windows, chosen deliberately: a child whose creation // time could not be read at capture is not killed, so if `OpenProcess` @@ -162,20 +179,19 @@ pub(super) fn abandon_failed_managed_launch(paths: &AppPaths, record: &ManagedSe // is a forced tree kill of a pid that may by now belong to another // process, which cannot be undone. The spawn primitive returns only the // pid and closes its handle, so a handle-based identity would mean - // changing it; that is not done here. Only the pure refusal decision - // (`launch_cleanup_refusal`) is under a test that runs on Windows; the - // cleanup of a real Windows child is run by no test. + // changing it; that is not done here. The refusal decision + // (`launch_cleanup_refusal`) and what the cleanup asks the termination to + // do are under tests that run on Windows; the cleanup of a real Windows + // child is run by no test. let outcome = if let Some(refusal) = launch_cleanup_refusal(record.engine_start_ticks, host_records_start_tokens()) { refusal } else { - rocm_core::terminate_verified( - &rocm_core::ProcessIdentity::new(pid, record.engine_start_ticks), - rocm_core::KillScope::Tree, - MANAGED_LAUNCH_CLEANUP_GRACE, - true, - ) + terminate(&rocm_core::ProcessIdentity::new( + pid, + record.engine_start_ticks, + )) }; record_cli_audit_event( paths, @@ -937,6 +953,55 @@ mod tests { Ok(()) } + /// What the cleanup asks of the termination, through the seam and so on every + /// host: the child's PID with the token captured at spawn, and nothing at all + /// for a child with no token where tokens are recorded. The record is marked + /// stopped either way and the audit event carries the verdict the termination + /// gave. + #[test] + fn abandoned_launch_asks_the_termination_for_the_spawned_child_with_its_token() -> Result<()> { + use std::cell::RefCell; + let cases = [ + // (token captured, whether the cleanup is expected to ask) + (Some(77_u64), true), + (None, !host_records_start_tokens()), + ]; + for (token, expect_call) in cases { + let (root, paths) = test_paths("abandon-launch-asks-termination"); + paths.ensure()?; + let mut record = write_live_service_record( + &paths, + "lemonade-abandon-seam", + "abandon-seam-model", + 11_494, + "running", + )?; + record.engine_pid = Some(4242); + record.engine_start_ticks = token; + let asked = RefCell::new(Vec::new()); + + abandon_failed_managed_launch_with(&paths, &record, |identity| { + asked.borrow_mut().push(*identity); + rocm_core::TerminationOutcome::Forced + }); + + let asked = asked.into_inner(); + let persisted = load_managed_services(&paths)? + .into_iter() + .find(|candidate| candidate.service_id == record.service_id) + .expect("a launch that spawned a child keeps its record"); + let _ = fs::remove_dir_all(root); + let expected = if expect_call { + vec![rocm_core::ProcessIdentity::new(4242, token)] + } else { + Vec::new() + }; + assert_eq!(asked, expected, "token {token:?}"); + assert_eq!(persisted.status, "stopped", "token {token:?}"); + } + Ok(()) + } + #[test] #[cfg(target_os = "linux")] fn abandoned_launch_never_signals_a_pid_that_no_longer_matches_the_spawn_token() -> Result<()> { diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index f19d138e23..1ac81c194f 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -183,7 +183,8 @@ pub use paths::{AppPaths, engine_plugin_dirs}; pub use proc_lifecycle::{ IdentityState, KillScope, MANAGED_STOP_GRACE, ProcessIdentity, RecordedPidOutcome, RecordedPidStop, TerminationOutcome, identity_state, process_start_ticks, - terminate_recorded_pid, terminate_recorded_service_pids, terminate_verified, + terminate_recorded_pid, terminate_recorded_pid_within, terminate_recorded_service_pids, + terminate_verified, }; pub use process::{ FileLock, LoopbackPortLease, MANAGED_LAUNCH_LOCK_TIMEOUT, ProcessProbe, detach_command_session, diff --git a/crates/rocm-core/src/proc_lifecycle.rs b/crates/rocm-core/src/proc_lifecycle.rs index 26e192a167..dbe18a856e 100644 --- a/crates/rocm-core/src/proc_lifecycle.rs +++ b/crates/rocm-core/src/proc_lifecycle.rs @@ -636,12 +636,25 @@ fn settle_roots_that_exited_alone(stops: &mut [RecordedPidStop]) { /// cannot see exit, and reporting the stop confirmed would clear the record and /// the endpoint key while a worker may still hold the device. pub fn terminate_recorded_pid(identity: &ProcessIdentity) -> TerminationOutcome { + terminate_recorded_pid_within(identity, MANAGED_STOP_GRACE) +} + +/// [`terminate_recorded_pid`] with the root's grace chosen by the caller. +/// +/// For a process that has nothing to flush, such as a launch child cleaned up +/// because its record could not be published. Everything else is the same +/// routine: the PID hold, the identity gate, and on Windows the descendant kill, +/// whose own deadline stays [`MANAGED_STOP_GRACE`]. +pub fn terminate_recorded_pid_within( + identity: &ProcessIdentity, + grace: std::time::Duration, +) -> TerminationOutcome { terminate_recorded_pid_with( identity, platform_tree_kill(), hold_pid, identity_state, - |identity| terminate_verified(identity, KillScope::Tree, MANAGED_STOP_GRACE, true), + |identity| terminate_verified(identity, KillScope::Tree, grace, true), ) } diff --git a/docs/architecture.md b/docs/architecture.md index 6f1bb8c3da..b1200cda0d 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -95,7 +95,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`service_hints.rs`](../apps/rocm/src/service_hints.rs) — shared CLI-output component: the `rocm services stop|restart|remove --yes` hint text, shared by [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs), [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs), [`service_launch.rs`](../apps/rocm/src/service_launch.rs), [`service_records.rs`](../apps/rocm/src/service_records.rs), [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs), and [`services_text.rs`](../apps/rocm/src/services_text.rs) so they cannot go out of sync with each other, as `serve_summary.rs`'s stop row did from the day it was added — #417. It also holds the helpers that map a service tool name to the command word, verb and past tense those hints and messages use (`service_action_command`, `service_action_verb`, `service_action_past_tense`) and to its `SandboxToolArg` (`sandbox_tool_arg_from_service_tool`); [`services_text.rs`](../apps/rocm/src/services_text.rs) reads `service_action_past_tense` from here rather than through the crate root. It quotes the service id with `shell_quote`, so a hint names exactly the argv it runs. - [`service_launch.rs`](../apps/rocm/src/service_launch.rs) — mechanical relocation, since split into a directory of child modules, owns no code of its own beyond its declarations: spawning/attaching to a managed service process and streaming its log, the port allocation transaction and failed-launch cleanup (under the shared launch lock), plus the endpoint-key/auth checks that gate a public bind, divided among the children listed below. The file declares the children, a `test_support` module compiled only for tests (the fixtures more than one child's tests share), and `pub(crate) use` re-exports of the items other modules call, so [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`automations.rs`](../apps/rocm/src/automations.rs), [`service_control.rs`](../apps/rocm/src/service_control.rs) and [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs) still reach them as `crate::service_launch::name`; [`main.rs`](../apps/rocm/src/main.rs) does not. An item that sibling children share is `pub(super)` and reached through `super::`; everything else is private to the child that owns it. `ServicesCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs) rather than moving alongside, and `services()` lives in [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs); the record removal and pruning it dispatches to now lives in [`service_records.rs`](../apps/rocm/src/service_records.rs), and only stop/restart are also reachable from the chat/LLM sandbox tool-calling path. This module does not import [`service_control.rs`](../apps/rocm/src/service_control.rs), which imports it: `run_attached_service` takes the stop for a Ctrl-C as a parameter, and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) passes `run_internal_sandbox_tool`. - [`service_launch/endpoint_auth.rs`](../apps/rocm/src/service_launch/endpoint_auth.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, no owned types: bind-host validation and the endpoint API key policy. `validate_bind_host` refuses a non-loopback host without `--allow-public-bind` (`is_loopback_host` is the inverse of `rocm_engine_protocol::is_public_bind_host`, which `rocmd` shares); `resolve_endpoint_auth` decides the key a launch uses (none for a loopback bind unless a key is required, otherwise the supplied key trimmed or a generated one, refusing an empty key or one with a control character); `drop_orphaned_endpoint_key_on_already_running` clears a key staged for an attempt that spawned nothing; `ensure_public_bind_engine_supported` refuses a public bind with Windows managed Lemonade, whose key cannot be enforced; and `ensure_public_service_has_endpoint_key` is the guard a respawn runs so a public (or `--require-api-key`) service whose key file is gone is not reopened without authentication. All six are `pub(crate)` and re-exported by the parent: [`service_control.rs`](../apps/rocm/src/service_control.rs) reads `ensure_public_service_has_endpoint_key`, [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs) reads `is_loopback_host`, and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) reads the rest. -- [`service_launch/launch_transaction.rs`](../apps/rocm/src/service_launch/launch_transaction.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, owns types: the part of a managed launch that decides before anything spawns. `resolve_managed_port_in_transaction` refreshes liveness, answers an equivalent live service before leasing any port (`ManagedPortDecision`), and otherwise leases a concrete port against both the ports live records still reserve (`live_managed_services`, `reserved_service_ports`) and the real OS through [`serve_port.rs`](../apps/rocm/src/serve_port.rs); a caller must hold the launch lock across it and the record write that follows. `abandon_failed_managed_launch` undoes a failed attempt before that lock is released: a spawned child is terminated first and its record kept, marked stopped, while an attempt that never spawned has every artifact removed. A spawned child whose start token was not captured, on a host that records tokens (Linux and Windows, which records the process creation time), is not signalled (`launch_cleanup_refusal`): it was gone before it could be captured and its pid may belong to another process. `managed_service_reuse_refusal` (with `ManagedReuseRefusal`) is the one answer to whether a live matching service can serve a request (recipe, `--require-api-key`, device policy), and `no_usable_gpu_refusal` words the no-usable-GPU refusal once for `serve()`'s pre-flight and for the spawn. `managed_service_reuse_refusal` and `no_usable_gpu_refusal` are `pub(crate)`, re-exported, and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), and `ManagedReuseRefusal` is `pub(crate)` too because the first returns it (only [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs) matches on it); `ManagedPortDecision`, `live_managed_services`, `resolve_managed_port_in_transaction` and `abandon_failed_managed_launch` are `pub(super)` for [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs). +- [`service_launch/launch_transaction.rs`](../apps/rocm/src/service_launch/launch_transaction.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, owns types: the part of a managed launch that decides before anything spawns. `resolve_managed_port_in_transaction` refreshes liveness, answers an equivalent live service before leasing any port (`ManagedPortDecision`), and otherwise leases a concrete port against both the ports live records still reserve (`live_managed_services`, `reserved_service_ports`) and the real OS through [`serve_port.rs`](../apps/rocm/src/serve_port.rs); a caller must hold the launch lock across it and the record write that follows. `abandon_failed_managed_launch` undoes a failed attempt before that lock is released: a spawned child is terminated first, through the same `rocm_core::terminate_recorded_pid_within` routine a stop uses (so on Windows its descendants are taken down too), and its record kept, marked stopped, while an attempt that never spawned has every artifact removed. A spawned child whose start token was not captured, on a host that records tokens (Linux and Windows, which records the process creation time), is not signalled (`launch_cleanup_refusal`): it was gone before it could be captured and its pid may belong to another process. `managed_service_reuse_refusal` (with `ManagedReuseRefusal`) is the one answer to whether a live matching service can serve a request (recipe, `--require-api-key`, device policy), and `no_usable_gpu_refusal` words the no-usable-GPU refusal once for `serve()`'s pre-flight and for the spawn. `managed_service_reuse_refusal` and `no_usable_gpu_refusal` are `pub(crate)`, re-exported, and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), and `ManagedReuseRefusal` is `pub(crate)` too because the first returns it (only [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs) matches on it); `ManagedPortDecision`, `live_managed_services`, `resolve_managed_port_in_transaction` and `abandon_failed_managed_launch` are `pub(super)` for [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs). - [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, owns types: the detached engine child that the background and attached serve paths share. The process and stdio plumbing (`detach_background_command`, `attach_background_stdio`, `managed_service_process_command`, `managed_engine_startup_failure_detail`) is `pub(crate)` and, off Windows, re-exported: [`service_control.rs`](../apps/rocm/src/service_control.rs) reads all four for a restart and [`service_launch/helper_autostart.rs`](../apps/rocm/src/service_launch/helper_autostart.rs) reads the first three. On Windows `managed_engine_startup_failure_detail` is re-exported for test builds only, because the [`services_text.rs`](../apps/rocm/src/services_text.rs) log-tail tests call it through `crate::service_launch` there too. `ManagedLaunchReport` (`pub(crate)`) is the engine-neutral result of a launch, and `ManagedSpawn` (`pub(super)`) says whether the launch found a live service or spawned a child. `spawn_managed_engine_child` is the whole serialized transaction under the launch lock (duplicate detection, port lease, manifest and log creation, argv preparation, the bounded immediate-exit check and record publication) and returns before any readiness wait; `launch_managed_engine_child_locked` is its side-effecting half, split out so that every failure after the record's first write goes through `abandon_failed_managed_launch`. The tests reach the spawn through a test-only, thread-scoped hook (`TEST_SPAWN_HOOK`, `pub(super)` so the tests in [`service_launch/start.rs`](../apps/rocm/src/service_launch/start.rs) can observe the state at the instant of the spawn). - [`service_launch/readiness.rs`](../apps/rocm/src/service_launch/readiness.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, owns private items: waiting on a service after it is launched or found live. `wait_for_record_ready` and `wait_on_existing_service` run the wait (one wait is bounded by `READINESS_WAIT`, and `wait_past_readiness_budget` is what `rocm serve --wait` uses to keep polling past it until the server answers or stops being live), `service_pid_is_alive`, `record_still_live` and `service_still_live` decide whether the tracked process is still there, `settle_exited_service` demotes the manifest of one that is not, and `died_during_wait` and `record_wait_outcome` end a wait, writing its outcome back without discarding what changed on disk meanwhile. `e2e_simulate_oom_launch` and `simulate_oom_managed_launch` are the end-to-end fault-injection seam behind the `e2e-oom-fault-injection` feature: the first is a `const fn` returning false without the feature, is `pub(crate)` and re-exported, and is read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs); the second fabricates the on-disk state of a launch that ran out of memory. `wait_for_record_ready`, `wait_on_existing_service`, `service_pid_is_alive`, `died_during_wait`, `record_wait_outcome` and `simulate_oom_managed_launch` are `pub(super)` for [`service_launch/start.rs`](../apps/rocm/src/service_launch/start.rs); the rest are private. - [`service_launch/start.rs`](../apps/rocm/src/service_launch/start.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, no owned types: the background launch path. `start_managed_service` runs the spawn transaction, releases the launch lock once the record is published, waits for HTTP readiness and returns a `ManagedLaunchReport`; if the launch fails before a record exists it drops the endpoint key `serve()` staged for the attempt. `print_managed_launch_plain` renders the report line by line for output that is not interactive, and `wait_begin_client_config` returns the one-time client configuration for a generated key when a wait begins, built by `render_endpoint_client_config` (`pub(super)`, since [`service_launch/attach.rs`](../apps/rocm/src/service_launch/attach.rs) prints it too). The three functions other modules call are `pub(crate)`, re-exported, and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs). From 4d24e0d08cbf745a1caf355a4ef5ea5696adc495 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Tue, 10 Nov 2026 12:40:17 +0000 Subject: [PATCH 24/30] fix(services): do not complete a container engine's pending stop from PID evidence, and finish a failed record as stopped A listing that sees every recorded process gone completes a pending stop. For a container-backed engine that is not evidence: the supervisor can be gone while the container, started by a detached docker run, still runs, and only the engine Stop decides that stop. The deferred cleanup now leaves such a record alone, with its key, marker and recorded processes, so the next services stop asks the engine again. A process engine with the same evidence still completes. When the cleanup does complete, it now sets stopped whatever the record read before, as a confirmed stop does. It set it only for a live status, so a failed record came back with its processes cleared and still failed. is_covered_by_stop_of accepts a record with no process only when it reads stopped, so a stop still finalizing reported itself unconfirmed, put the marker back and blocked a restart. The listing-before-finalization regression test now covers a failed record. Also pinned, with no behaviour change: a ready record held by a process this account cannot open is not stuck. --force is refused for it until a stop has been requested and the refusal names that stop, after which the forced removal works. The test runs the two commands in order, and docs/testing.md and docs/manual-testing.md state the order and the container-engine limit. Signed-off-by: Jussi Elo --- apps/rocm/src/managed_service_status.rs | 71 ++++++++++++++++--- apps/rocm/src/service_control.rs | 90 ++++++++++++++----------- apps/rocm/src/service_records.rs | 50 ++++++++++++++ docs/manual-testing.md | 4 ++ docs/testing.md | 28 +++++++- 5 files changed, 190 insertions(+), 53 deletions(-) diff --git a/apps/rocm/src/managed_service_status.rs b/apps/rocm/src/managed_service_status.rs index 8fa9101cbf..613e3f2db0 100644 --- a/apps/rocm/src/managed_service_status.rs +++ b/apps/rocm/src/managed_service_status.rs @@ -225,6 +225,10 @@ pub(crate) fn recorded_pids_may_be_running( /// "stopped" by another route still gets its deferred cleanup rather than /// stranding the key. /// +/// A container-backed engine never completes this way: its supervisor can exit +/// while the container runs, so only a stop whose engine `Stop` confirms may end +/// it. +/// /// "Observed gone" means every recorded process is *confirmed* gone: a process /// that could not be queried (on Windows, a live one this process may not open) /// is not gone, and the stop that reported it unverified must keep its key and @@ -237,6 +241,14 @@ fn settle_pending_stop_key_cleanup( if record.stop_requested_unix_ms.is_none() { return false; } + // A container-backed engine (`engine_stop_decides_confirmation`) is stopped by + // its engine `Stop`, not by the recorded supervisor: that process can be gone + // while the container still runs, so PID evidence alone cannot say the stop + // completed. The stop that could not confirm keeps its key, marker and + // recorded processes, and the next `services stop` asks the engine again. + if rocm_core::engine_stop_decides_confirmation(&record.engine) { + return false; + } if recorded_pids_may_be_running(record, probe) { return false; } @@ -251,15 +263,17 @@ fn settle_pending_stop_key_cleanup( // (`clear_recorded_processes`), and it is done only here, behind the // confirmed-gone check above: a process that may still be running stays named. record.clear_recorded_processes(); - // And the status a confirmed stop leaves, set here rather than left to the - // reconciliation below: with no PID recorded that reconciliation has nothing - // to read, so a record still reading `ready` or `running` would stay live. - // Setting it also makes the caller's liveness gate return at once, so the one - // write the caller makes covers the key cleanup, the cleared processes and - // the status together. - if managed_service_is_live(record) { - "stopped".clone_into(&mut record.status); - } + // And the status a confirmed stop leaves, whatever the record read before: + // `stopped`, as `rocm services stop` and `rocmd`'s stop set on a confirmed + // stop for a `failed` record too. Set here rather than left to the + // reconciliation below, which has no PID to read once they are cleared, so a + // `ready` or `running` record would stay live; and not only for a live status, + // because a `failed` record with its processes cleared is not the `stopped` + // record `is_covered_by_stop_of` accepts as this same stop completing, so a + // stop still finalizing would report itself unconfirmed. Setting it also makes + // the caller's liveness gate return at once, so the one write the caller makes + // covers the key cleanup, the cleared processes and the status together. + "stopped".clone_into(&mut record.status); true } @@ -1079,10 +1093,45 @@ mod tests { /// The deferred cleanup forgets the PIDs, so the reconciliation after it has /// none to read: every live status must still end as `stopped`, in the one /// refresh, rather than a record such as `starting` staying live with no - /// process behind it. + /// process behind it, and a `failed` one must too, as a confirmed stop makes it. + /// A container-backed engine's supervisor can be gone while the container + /// still runs, and its stop is decided by the engine `Stop`, which the + /// unconfirmed stop did not get. A listing that sees the supervisor gone must + /// not complete that stop: key, marker and recorded processes stay for the + /// next `services stop`. A process engine with the same PID evidence does + /// complete (the sibling tests above). + #[test] + fn a_pending_stop_of_a_container_engine_is_not_completed_by_a_gone_supervisor() -> Result<()> { + assert!(rocm_core::engine_stop_decides_confirmation("aim")); + let (root, paths) = test_paths("liveness-container-pending-stop"); + paths.ensure()?; + let service_id = "svc-container"; + let mut record = pending_stop_record(&paths, service_id)?; + "aim".clone_into(&mut record.engine); + record.supervisor_pid = 4242; + record.supervisor_start_ticks = Some(5); + + let _ = refresh_managed_service_runtime_liveness_with(&paths, &mut record, |_| { + ProcessProbe::Gone + }); + + assert!( + endpoint_keys::endpoint_key_file_if_present(&paths, service_id).is_some(), + "the key stays: the container may still be serving with it" + ); + assert_eq!(record.stop_requested_unix_ms, Some(1)); + assert_eq!( + (record.supervisor_pid, record.supervisor_start_ticks), + (4242, Some(5)), + "the record keeps naming what the stop could not confirm" + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + #[test] fn a_completed_pending_stop_reads_stopped_whatever_live_status_it_had() -> Result<()> { - for status in ["starting", "running", "ready", "recovering"] { + for status in ["starting", "running", "ready", "recovering", "failed"] { let (root, paths) = test_paths(&format!("liveness-pending-stop-{status}")); paths.ensure()?; let mut record = pending_stop_record(&paths, "svc-status")?; diff --git a/apps/rocm/src/service_control.rs b/apps/rocm/src/service_control.rs index ed296479b0..95b5bc0292 100644 --- a/apps/rocm/src/service_control.rs +++ b/apps/rocm/src/service_control.rs @@ -1122,51 +1122,63 @@ mod tests { /// before the stop reads the record again completes the same stop (key, /// marker, recorded processes cleared, `stopped`) and writes it. That record /// names no process, which must not read as a restart: the stop is confirmed, - /// the marker is not put back, and a restart is not refused for it. + /// the marker is not put back, and a restart is not refused for it. A record + /// that read `failed` is completed to `stopped` too, as a confirmed stop makes + /// it, so it is covered as well and not left reading `failed` with no process. #[test] fn a_listing_that_completes_the_stop_before_finalization_does_not_unconfirm_it() { use rocm_core::TerminationOutcome::Graceful; - let (root, paths) = test_paths("stop-listing-before-finalization"); - paths.ensure().unwrap(); - let service_id = "svc-stop-listing-before-finalization"; - let mut planted = stop_test_record(&paths, service_id); - planted.supervisor_pid = 4242; - planted.supervisor_start_ticks = None; - planted.engine_pid = Some(4243); - planted.engine_start_ticks = Some(77); - planted.write().unwrap(); - endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + for status in ["ready", "failed"] { + let (root, paths) = test_paths(&format!("stop-listing-before-finalization-{status}")); + paths.ensure().unwrap(); + let service_id = "svc-stop-listing-before-finalization"; + let mut planted = stop_test_record(&paths, service_id); + planted.status = status.to_owned(); + planted.supervisor_pid = 4242; + planted.supervisor_start_ticks = None; + planted.engine_pid = Some(4243); + planted.engine_start_ticks = Some(77); + planted.write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); - let report = stop_internal_managed_service_with(&paths, service_id, |stopping| { - assert!(stopping.stop_requested_unix_ms.is_some()); - // The processes have exited; a concurrent `services` listing sees it, - // completes the pending stop and persists that, before this stop - // reads the record again. - let (mut listed, _) = read_managed_service_manifest(&paths, service_id).unwrap(); - let changed = - crate::managed_service_status::refresh_managed_service_runtime_liveness_with( - &paths, - &mut listed, - |_| rocm_core::ProcessProbe::Gone, + let report = stop_internal_managed_service_with(&paths, service_id, |stopping| { + assert!(stopping.stop_requested_unix_ms.is_some()); + // The processes have exited; a concurrent `services` listing sees + // it, completes the pending stop and persists that, before this + // stop reads the record again. + let (mut listed, _) = read_managed_service_manifest(&paths, service_id).unwrap(); + // The fake PIDs read as gone to the stop's own initial refresh, + // which may already have demoted the status: put back the status + // the listing under test starts from. + status.clone_into(&mut listed.status); + let changed = + crate::managed_service_status::refresh_managed_service_runtime_liveness_with( + &paths, + &mut listed, + |_| rocm_core::ProcessProbe::Gone, + ); + assert!( + changed, + "premise: the listing completed the stop ({status})" ); - assert!(changed, "premise: the listing completed the stop"); - listed.write().unwrap(); - assert_eq!(raw_manifest(&paths, service_id).supervisor_pid, 0); - vec![stop_of(4242, Graceful), stop_of(4243, Graceful)] - }) - .unwrap(); + listed.write().unwrap(); + assert_eq!(raw_manifest(&paths, service_id).supervisor_pid, 0); + vec![stop_of(4242, Graceful), stop_of(4243, Graceful)] + }) + .unwrap(); - let on_disk = raw_manifest(&paths, service_id); - let key = endpoint_keys::endpoint_api_key(&paths, service_id); - let _ = fs::remove_dir_all(root); - assert_eq!(report["stopped"], true, "{report}"); - assert_eq!(on_disk.status, "stopped"); - assert_eq!( - on_disk.stop_requested_unix_ms, None, - "a confirmed stop does not put the marker back" - ); - assert_eq!(on_disk.supervisor_pid, 0); - assert_eq!(key, None, "the key goes with the confirmed stop"); + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + assert_eq!(report["stopped"], true, "{status}: {report}"); + assert_eq!(on_disk.status, "stopped", "{status}"); + assert_eq!( + on_disk.stop_requested_unix_ms, None, + "{status}: a confirmed stop does not put the marker back" + ); + assert_eq!(on_disk.supervisor_pid, 0, "{status}"); + assert_eq!(key, None, "{status}: the key goes with the confirmed stop"); + } } /// The same race through `restart`: the restart's own stop is completed by a diff --git a/apps/rocm/src/service_records.rs b/apps/rocm/src/service_records.rs index 90f6ff9ebf..1142e3d72e 100644 --- a/apps/rocm/src/service_records.rs +++ b/apps/rocm/src/service_records.rs @@ -2093,6 +2093,56 @@ mod tests { Ok(()) } + /// A `ready` record whose dead server's PID is now held by a process this + /// account cannot open, and that nobody has tried to stop: `--force` is + /// refused (it never forces a record nobody tried to stop), and the refusal + /// names `services stop`. That stop cannot verify the process, reports it + /// unverified and records the request, after which `--force` removes the + /// record. So the record is not stuck: the way out is those two commands, and + /// this is the intended shape, not a gap. + #[test] + fn a_ready_record_held_by_an_unqueryable_process_is_cleared_by_stop_then_force() -> Result<()> { + let (root, paths) = test_paths("services-remove-ready-unqueryable-chain"); + paths.ensure()?; + held_record(&paths, "svc-ready-held", false)?; + let pid = std::process::id(); + + let first = + remove_managed_service_record_with(&paths, "svc-ready-held", true, true, |_| { + ProcessProbe::Unqueryable + }) + .map(|removed| removed.text) + .expect_err("a record nobody tried to stop is not forced"); + let report = crate::service_control::stop_internal_managed_service_with( + &paths, + "svc-ready-held", + |_| { + vec![rocm_core::RecordedPidStop { + pid, + role: "supervisor_and_engine", + outcome: rocm_core::RecordedPidOutcome::Terminated( + rocm_core::TerminationOutcome::Unverified, + ), + }] + }, + )?; + let second = + remove_managed_service_record_with(&paths, "svc-ready-held", true, true, |_| { + ProcessProbe::Unqueryable + })?; + let left = files_left(&paths, "svc-ready-held"); + let _ = fs::remove_dir_all(root); + + assert!( + format!("{first:#}").contains("rocm services stop svc-ready-held --yes"), + "{first:#}" + ); + assert_eq!(report["stopped"], false, "{report}"); + assert!(second.forced); + assert_eq!(left, 0, "the forced removal took the record and its files"); + Ok(()) + } + /// `--force` never goes past a process that is confirmed running, and it /// needs a stop to have been requested: the refusal says which, and names the /// stop that is the way forward. diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 81ad07753b..f25b416e66 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -646,6 +646,10 @@ possibly running, which is the safe direction: the record never reconciles to state and its endpoint key. The server can no longer be restarted afterwards. `rocm services prune` has no `--force`: it leaves such a record alone and lists it under `Left alone` with this exact command. + If the record still reads `ready` and nobody has run `rocm services stop` on it, + `--force` is refused and the refusal names that stop first; run it (it reports + the process unverified and records the request), then the forced removal. This is + the intended order, not a gap: `--force` never removes a record nobody tried to stop. The rest of this section needs a **vLLM** server running, on Linux or WSL, and two runtimes to move it between. Nothing earlier in this document sets either diff --git a/docs/testing.md b/docs/testing.md index cd17987a66..5a892b582a 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2137,7 +2137,15 @@ on the Linux e2e jobs `-07` is skipped and `-08` runs its Linux half. Not covere by any CI job: the end-to-end tree kill of a parent with a child (`-07`), the failure cases that cannot be staged (an `OpenProcess` access-denied process, a failing or stalled `taskkill`, `taskkill` exiting 128), and the launch-cleanup -branch; the unit tests above stop a `ping` that has no child of its own, so they +branch run against a real Windows child. The failed-launch cleanup terminates its +child through the same routine as a stop (`terminate_recorded_pid_within`, so on +Windows the tree kill runs too), and what it asks of the termination is tested +through an injected function on every host +(`abandoned_launch_asks_the_termination_for_the_spawned_child_with_its_token`: +the child's PID with the token captured at spawn, and nothing at all for a child +with no token where tokens are recorded); that the production wiring reaches the tree +kill is not observable on Linux, where the termination walks the tree itself, so it +rests on the routine's own tests and on review. The unit tests above stop a `ping` that has no child of its own, so they do not show a descendant being taken down. A stop that cannot verify a process names `ps -p -o pid,args` on Linux and `tasklist /FI "PID eq "` on Windows (`process_check_command`). @@ -2248,6 +2256,12 @@ drives the refresh with an unqueryable PID and then a confirmed-gone one, an unverified stop and then the refresh over the record it wrote, and `a_record_is_in_use_unless_every_recorded_process_is_confirmed_gone` covers the removal guard; each takes the probe as an argument, so they run on every host. +A pending stop of a container-backed engine (`engine_stop_decides_confirmation`, +today `aim`) is never completed that way: its supervisor can be gone while the +container still runs, and only the engine `Stop` decides that stop, so a listing +does not complete it from PID evidence. The key, the marker and the recorded processes stay until a later +`services stop` is confirmed +(`a_pending_stop_of_a_container_engine_is_not_completed_by_a_gone_supervisor`). When that refresh completes a pending stop (every recorded process confirmed gone) it also forgets the recorded processes and reads the record as stopped, as a stop that confirmed the exit itself does, so a later `rocm services stop` visits no PID: a PID @@ -2256,7 +2270,9 @@ whatever process now holds the number. A process that may still be running stays named. `a_pending_stop_keeps_its_key_and_marker_while_a_process_cannot_be_queried` checks both halves (named while unqueryable, forgotten once gone), `a_completed_pending_stop_reads_stopped_whatever_live_status_it_had` that every live -status still ends as `stopped` in that one refresh, and +status, and `failed`, still ends as `stopped` in that one refresh (a `failed` record +cleared but left `failed` would not be covered by `is_covered_by_stop_of`, and the stop +finalizing would report itself unconfirmed), and `a_stop_completed_by_a_later_listing_leaves_no_pid_for_the_next_stop` runs the real termination routine over the record the listing wrote and asserts it visits nothing. Three gaps in the scenario coverage are deliberate. `service-stop-07` records no @@ -2273,7 +2289,13 @@ that will read it exists. `a_stop_reports_its_own_pid_as_self` and Third, the `check:` line an unverified stop prints is a diagnostic, not a remediation: it shows whether the process is still running, and what clears the condition is that process ending (or, for a record held by a process `rocm` -cannot open, `rocm services remove --force --yes`), after which a stop or a +cannot open, `rocm services remove --force --yes`; for a record still reading +`ready` that nobody has tried to stop, `--force` is refused and names +`rocm services stop --yes` first, which reports the process unverified and +records the request, so the clearing is those two commands, on purpose: +`--force` never removes a record nobody tried to stop, and +`a_ready_record_held_by_an_unqueryable_process_is_cleared_by_stop_then_force` +runs the chain), after which a stop or a service listing confirms and drops the key. The command is a stock one the host has, pinned by `the_process_check_command_is_the_one_the_host_has`, and `service-stop-07` uses the same `tasklist /FI "PID eq "` filter (in CSV From 78d9c6b5d2a8a99c1b93f9667479527aa97e0550 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Tue, 10 Nov 2026 13:41:49 +0000 Subject: [PATCH 25/30] docs(cli): shorten the services help text this change added, to stay within the module size limit cli.rs is checked by check-module-size, which now blocks, and the doc comments added for services stop, remove, remove --force and prune pushed it six lines over the 1500 limit. The same facts are kept in fewer lines: stop still names a process that did not exit or whose child processes may still be running and the check command for one it could not verify, remove says a process rocm cannot query counts as running on Windows, --force states its three conditions and that a running process keeps running, and prune still lists the forced removal for a record held only by such a process. README.md and the docs carry the longer wording. Signed-off-by: Jussi Elo --- apps/rocm/src/cli.rs | 39 ++++++++++++++++----------------------- 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index 362d1ddc9c..3947096377 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -1139,13 +1139,10 @@ pub(crate) enum ServicesCommand { /// confirmed gone. Otherwise it reports on stderr that the stop was not /// confirmed and exits with an error, keeping the record, which still names /// the processes, and any endpoint key, which the server may still be - /// enforcing. It names each process that did not exit (on Windows also one - /// that exited, or was already gone, when taking the processes under it - /// down did not complete or found nothing and no other recorded process is - /// confirmed gone), and for one it could not verify (its recorded - /// start-time can no longer be read back, or, on Windows, it cannot be - /// queried at all) it signals nothing and names an operating-system - /// command to check on it. It does not suggest running the stop again. + /// enforcing. It names each process that did not exit (on Windows, also one + /// whose child processes may still be running), and for one it could not + /// verify it signals nothing and names an operating-system command to check + /// on it. It does not suggest running the stop again. Stop { /// Service id from `rocm services list`. service_id: String, @@ -1167,12 +1164,11 @@ pub(crate) enum ServicesCommand { }, /// Delete one local server record and its files. /// - /// Only works on a record that is not running: stop the server first. A - /// server counts as running while any process it recorded may be running, - /// which on Windows includes one `rocm` cannot query (a live process the - /// account may not open). The record's details file, log, engine state file, - /// and endpoint key file are all deleted, so its log can no longer be read - /// and it can no longer be restarted. + /// Only works on a record that is not running: stop the server first (on + /// Windows a process `rocm` cannot query counts as running). The record's + /// details file, log, engine state file, and endpoint key file are all + /// deleted, so its log can no longer be read and it can no longer be + /// restarted. Remove { /// Service id from `rocm services list --all`. service_id: String, @@ -1181,22 +1177,19 @@ pub(crate) enum ServicesCommand { yes: bool, /// Remove a record whose recorded process `rocm` cannot query. /// - /// For the one case with no other way out: a stop was requested, no - /// recorded process is confirmed running, and at least one could not be - /// queried (on Windows, a live process the account may not open, which - /// can be a dead server's PID taken by another program). The record, its - /// log, engine state and endpoint key are deleted, the command says - /// which process could not be queried, and a process that is still - /// running keeps running. Refused for any process confirmed running. + /// Only when a stop was requested, no recorded process is confirmed + /// running, and at least one could not be queried (on Windows, a live + /// process the account may not open). Deletes the record, log, engine + /// state and endpoint key, and says which process; if it is still + /// running it keeps running. Refused for a process confirmed running. #[arg(long)] force: bool, }, /// Delete local server records that are no longer running. /// /// Running servers are always left alone; a record held only by a process - /// `rocm` cannot query (on Windows) is listed with the `remove --force` - /// command that clears it. Leftover files whose record is already gone are - /// cleaned up too. + /// `rocm` cannot query (Windows) is listed with the `remove --force` command. + /// Leftover files whose record is already gone are cleaned up too. /// /// Waits for a managed launch already under way to publish its record /// before reading the directory, and waits as long as that launch takes. From bd168e90c83bb09c0f987e69e42ee39dc5e6172f Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Tue, 10 Nov 2026 14:34:00 +0000 Subject: [PATCH 26/30] refactor(cli): move the clap value enums out of cli.rs into cli_args.rs cli.rs is checked by check-module-size, which blocks, and main keeps adding lines to it, so the services help text this branch adds repeatedly put it over the 1500 line limit. Trimming the help text again would only buy a few lines until the next merge. The clap value enums behind --framework, --format, --provider, --device and the similar arguments (FrameworkArg, InstallFormat, Provider, WatcherModeArg, TelemetryModeArg, DevicePolicyArg, SandboxToolArg, PermissionsModeArg) and their conversions are self-contained, so they move as they are to cli_args.rs, and cli.rs re-exports each one so every module keeps reaching them as crate::cli::Provider and so on. Nothing else changes: attributes, help text, aliases and declaration order are as before. cli.rs goes from over the limit to about 1348 production lines, leaving real headroom, and docs/architecture.md has the new file's entry. Signed-off-by: Jussi Elo --- apps/rocm/src/cli.rs | 169 ++---------------------------------- apps/rocm/src/cli_args.rs | 176 ++++++++++++++++++++++++++++++++++++++ apps/rocm/src/main.rs | 1 + docs/architecture.md | 3 +- 4 files changed, 185 insertions(+), 164 deletions(-) create mode 100644 apps/rocm/src/cli_args.rs diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index a7cb1b350a..310474b111 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -12,13 +12,14 @@ //! the completion builder, and re-imports these types so every other module //! keeps reaching them as `crate::Command`, `crate::Provider` and so on. +pub(crate) use crate::cli_args::{ + DevicePolicyArg, FrameworkArg, InstallFormat, PermissionsModeArg, Provider, SandboxToolArg, + TelemetryModeArg, WatcherModeArg, +}; use crate::service_records::DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS; use crate::{bootstrap, chat, comfyui, remote, storage}; -use clap::{Parser, Subcommand, ValueEnum}; -use rocm_core::{ - DEFAULT_LOCAL_HOST, PERMISSIONS_MODE_ASK, PERMISSIONS_MODE_FULL_ACCESS, TELEMETRY_MODE_LOCAL, - TELEMETRY_MODE_OFF, WatcherMode, -}; +use clap::{Parser, Subcommand}; +use rocm_core::DEFAULT_LOCAL_HOST; use std::path::PathBuf; /// User-selectable serving engines, in the order shown in `--help` and shell @@ -1345,164 +1346,6 @@ pub(crate) enum SetupCommand { }, } -/// Which framework `rocm examine` should probe. -/// -/// The CLI-facing mirror of [`rocm_core::FrameworkProbe`], which cannot derive -/// `ValueEnum` itself without pulling clap into `rocm-core`. Every variant of -/// that enum existed and was reachable in the library, but no caller ever passed -/// anything but the default, so the choice could not be made from the command -/// line at all. -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -pub(crate) enum FrameworkArg { - /// Try PyTorch first, then llama.cpp (the default). - Auto, - /// Probe PyTorch only. - Pytorch, - /// Probe llama.cpp only. - #[value(name = "llama-cpp", alias = "llamacpp")] - LlamaCpp, - /// Probe no framework at all - fastest, and enough for GPU and driver questions. - Skip, -} - -impl From for rocm_core::FrameworkProbe { - fn from(value: FrameworkArg) -> Self { - match value { - FrameworkArg::Auto => Self::Auto, - FrameworkArg::Pytorch => Self::PyTorch, - FrameworkArg::LlamaCpp => Self::LlamaCpp, - FrameworkArg::Skip => Self::Skip, - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -pub(crate) enum InstallFormat { - /// Python wheel packages (recommended; smaller, pip-installable). - Wheel, - /// Self-contained tarball archive of the ROCm SDK. - Tarball, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -pub(crate) enum Provider { - /// A local OpenAI-compatible server started via `rocm serve` (no API key). - Local, - /// Anthropic Claude API (requires an API key). - Anthropic, - /// OpenAI API or OpenAI-compatible endpoint (requires an API key). - Openai, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -pub(crate) enum WatcherModeArg { - /// Only report findings; never take action. - Observe, - /// Report findings and propose changes for you to approve. - Propose, - /// Apply changes automatically within a contained, reversible scope. - Contained, -} - -impl From for WatcherMode { - fn from(value: WatcherModeArg) -> Self { - match value { - WatcherModeArg::Observe => Self::Observe, - WatcherModeArg::Propose => Self::Propose, - WatcherModeArg::Contained => Self::Contained, - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -pub(crate) enum TelemetryModeArg { - /// Collect GPU telemetry on this machine only; nothing leaves the device. - Local, - /// Disable telemetry collection entirely. - Off, -} - -/// User-facing `rocm serve --device` policy. -/// -/// Modeled as a clap `ValueEnum` so invalid input is rejected by clap's usage -/// validation (exit code 2, with the valid choices listed) rather than a generic -/// application error (exit code 1) — matching how every other enum-style CLI -/// argument behaves. The historical aliases (`auto`/`gpu` for `gpu_required`, -/// `cpu` for `cpu_only`) stay accepted for backward compatibility but are hidden -/// from the advertised list. -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -#[value(rename_all = "snake_case")] -pub(crate) enum DevicePolicyArg { - /// Require a ROCm GPU; fail if none is usable (the default). - #[value(alias = "auto", alias = "gpu")] - GpuRequired, - /// Accepted for compatibility; behaves identically to `gpu_required` - /// (rocm serve has no CPU fallback path). - GpuPreferred, - /// Accepted but always rejected: rocm serve has no CPU fallback path, so - /// this exits with an error. Hidden from `--help`, shell completions and the - /// invalid-value suggestion list so it is never advertised as usable, while - /// still parsing so the deliberate rejection message is preserved. - #[value(alias = "cpu", hide = true)] - CpuOnly, -} - -impl DevicePolicyArg { - /// Canonical policy string understood by [`crate::gpu_policy::parse_device_policy`]. Routing the - /// parsed variant back through the string keeps `parse_device_policy` the - /// single choke point that enforces the `cpu_only` rejection across every - /// caller, rather than adding a second conversion path that could bypass it. - pub(crate) const fn as_policy_str(self) -> &'static str { - match self { - Self::GpuRequired => "gpu_required", - Self::GpuPreferred => "gpu_preferred", - Self::CpuOnly => "cpu_only", - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -#[value(rename_all = "snake_case")] -pub(crate) enum SandboxToolArg { - ListServers, - RestartServer, - StopServer, -} - -impl SandboxToolArg { - pub(crate) const fn as_cli_value(self) -> &'static str { - match self { - Self::ListServers => "list_servers", - Self::RestartServer => "restart_server", - Self::StopServer => "stop_server", - } - } -} - -impl TelemetryModeArg { - pub(crate) const fn as_str(self) -> &'static str { - match self { - Self::Local => TELEMETRY_MODE_LOCAL, - Self::Off => TELEMETRY_MODE_OFF, - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -pub(crate) enum PermissionsModeArg { - FullAccess, - Ask, -} - -impl PermissionsModeArg { - pub(crate) const fn as_str(self) -> &'static str { - match self { - Self::FullAccess => PERMISSIONS_MODE_FULL_ACCESS, - Self::Ask => PERMISSIONS_MODE_ASK, - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/apps/rocm/src/cli_args.rs b/apps/rocm/src/cli_args.rs new file mode 100644 index 0000000000..a11fd1b520 --- /dev/null +++ b/apps/rocm/src/cli_args.rs @@ -0,0 +1,176 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The clap value enums behind the `--framework`, `--format`, `--provider`, +//! `--mode`, `--device`, `--telemetry` and `--permissions` style arguments, and +//! the conversions from them to the library types they stand for. +//! +//! Mechanically relocated from `cli.rs` with no behavior change: attributes, +//! help text, aliases and declaration order are unchanged. `cli.rs` re-exports +//! each type, so every module keeps reaching them as `crate::cli::Provider` and +//! so on. + +use clap::ValueEnum; +use rocm_core::{ + PERMISSIONS_MODE_ASK, PERMISSIONS_MODE_FULL_ACCESS, TELEMETRY_MODE_LOCAL, TELEMETRY_MODE_OFF, + WatcherMode, +}; + +/// Which framework `rocm examine` should probe. +/// +/// The CLI-facing mirror of [`rocm_core::FrameworkProbe`], which cannot derive +/// `ValueEnum` itself without pulling clap into `rocm-core`. Every variant of +/// that enum existed and was reachable in the library, but no caller ever passed +/// anything but the default, so the choice could not be made from the command +/// line at all. +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +pub(crate) enum FrameworkArg { + /// Try PyTorch first, then llama.cpp (the default). + Auto, + /// Probe PyTorch only. + Pytorch, + /// Probe llama.cpp only. + #[value(name = "llama-cpp", alias = "llamacpp")] + LlamaCpp, + /// Probe no framework at all - fastest, and enough for GPU and driver questions. + Skip, +} + +impl From for rocm_core::FrameworkProbe { + fn from(value: FrameworkArg) -> Self { + match value { + FrameworkArg::Auto => Self::Auto, + FrameworkArg::Pytorch => Self::PyTorch, + FrameworkArg::LlamaCpp => Self::LlamaCpp, + FrameworkArg::Skip => Self::Skip, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +pub(crate) enum InstallFormat { + /// Python wheel packages (recommended; smaller, pip-installable). + Wheel, + /// Self-contained tarball archive of the ROCm SDK. + Tarball, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +pub(crate) enum Provider { + /// A local OpenAI-compatible server started via `rocm serve` (no API key). + Local, + /// Anthropic Claude API (requires an API key). + Anthropic, + /// OpenAI API or OpenAI-compatible endpoint (requires an API key). + Openai, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +pub(crate) enum WatcherModeArg { + /// Only report findings; never take action. + Observe, + /// Report findings and propose changes for you to approve. + Propose, + /// Apply changes automatically within a contained, reversible scope. + Contained, +} + +impl From for WatcherMode { + fn from(value: WatcherModeArg) -> Self { + match value { + WatcherModeArg::Observe => Self::Observe, + WatcherModeArg::Propose => Self::Propose, + WatcherModeArg::Contained => Self::Contained, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +pub(crate) enum TelemetryModeArg { + /// Collect GPU telemetry on this machine only; nothing leaves the device. + Local, + /// Disable telemetry collection entirely. + Off, +} + +/// User-facing `rocm serve --device` policy. +/// +/// Modeled as a clap `ValueEnum` so invalid input is rejected by clap's usage +/// validation (exit code 2, with the valid choices listed) rather than a generic +/// application error (exit code 1) — matching how every other enum-style CLI +/// argument behaves. The historical aliases (`auto`/`gpu` for `gpu_required`, +/// `cpu` for `cpu_only`) stay accepted for backward compatibility but are hidden +/// from the advertised list. +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +#[value(rename_all = "snake_case")] +pub(crate) enum DevicePolicyArg { + /// Require a ROCm GPU; fail if none is usable (the default). + #[value(alias = "auto", alias = "gpu")] + GpuRequired, + /// Accepted for compatibility; behaves identically to `gpu_required` + /// (rocm serve has no CPU fallback path). + GpuPreferred, + /// Accepted but always rejected: rocm serve has no CPU fallback path, so + /// this exits with an error. Hidden from `--help`, shell completions and the + /// invalid-value suggestion list so it is never advertised as usable, while + /// still parsing so the deliberate rejection message is preserved. + #[value(alias = "cpu", hide = true)] + CpuOnly, +} + +impl DevicePolicyArg { + /// Canonical policy string understood by [`crate::gpu_policy::parse_device_policy`]. Routing the + /// parsed variant back through the string keeps `parse_device_policy` the + /// single choke point that enforces the `cpu_only` rejection across every + /// caller, rather than adding a second conversion path that could bypass it. + pub(crate) const fn as_policy_str(self) -> &'static str { + match self { + Self::GpuRequired => "gpu_required", + Self::GpuPreferred => "gpu_preferred", + Self::CpuOnly => "cpu_only", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +#[value(rename_all = "snake_case")] +pub(crate) enum SandboxToolArg { + ListServers, + RestartServer, + StopServer, +} + +impl SandboxToolArg { + pub(crate) const fn as_cli_value(self) -> &'static str { + match self { + Self::ListServers => "list_servers", + Self::RestartServer => "restart_server", + Self::StopServer => "stop_server", + } + } +} + +impl TelemetryModeArg { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::Local => TELEMETRY_MODE_LOCAL, + Self::Off => TELEMETRY_MODE_OFF, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +pub(crate) enum PermissionsModeArg { + FullAccess, + Ask, +} + +impl PermissionsModeArg { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::FullAccess => PERMISSIONS_MODE_FULL_ACCESS, + Self::Ask => PERMISSIONS_MODE_ASK, + } + } +} diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 52a3614f76..21ce37a6d0 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -17,6 +17,7 @@ mod chat; mod chat_cmd; mod chat_host_facts; mod cli; +mod cli_args; mod cli_audit; mod cli_progress; mod cli_report; diff --git a/docs/architecture.md b/docs/architecture.md index 5056903927..fb1cb45472 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -44,7 +44,8 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`chat/intent.rs`](../apps/rocm/src/chat/intent.rs) — mechanical relocation out of `chat.rs`, a leaf child of `chat.rs`: the prompt-intent heuristics, which decide from a user's latest message alone what it asks for. `fallback_rocm_tool_call_for_prompt` maps a prompt to the read-only or approval-gated tool call the assistant proposes without asking the local model (with `fallback_config_tool_call_for_prompt`, `fallback_engine_list_tool_call` and `fallback_services_list_tool_call` behind it), `prompt_can_use_read_only_without_local_assistant` says whether a status question can be answered before any model server is up, and `install_sdk_chat_approval_for_prompt` (with `install_sdk_without_prefix_chat_approval`) turns an install or setup request into the `rocm install sdk` approval request. The phrase lists live in the `prompt_asks_running_or_status`, `prompt_asks_engine_install_state`, `prompt_mentions_serving_engine_or_service`, `prompt_requests_rocm_install_or_setup` and `prompt_requests_install_action` predicates, and `latest_user_chat_message` cuts the newest message out of the conversation text the chat builds around it, so older words never steer these heuristics; it moved here with the heuristics because three of them call it, and leaving it with the turn loop would make the two modules call each other. Ten of the functions are `pub(super)`, called from `chat/turn.rs`; the other three are private. Its edges out are [`prompt_text.rs`](../apps/rocm/src/prompt_text.rs), [`rocm_argv.rs`](../apps/rocm/src/rocm_argv.rs), `tool_policy.rs` (`ChatToolApprovalRequest`), `freeform_planner.rs` (`format_structured_tool_call`) and `providers` (the tool-call type); nothing in it calls back into `chat/turn.rs` or `chat.rs`. Its tests, the `fallback_tool_call_*` family and the install-intent checks that call the heuristics directly, live in its own `mod tests`, with their original names. - [`chat_cmd.rs`](../apps/rocm/src/chat_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm chat`'s command handler (`run_chat_command`, the body of the `Command::Chat` arm of `dispatch()`), which hands off to the dashboard chat (`dash::run_chat`) in a terminal with no `--prompt` and otherwise prints the reply or status screen through `chat::render_chat_prompt_text`/`chat::render_chat_text`. It is a module of its own, not part of [`chat.rs`](../apps/rocm/src/chat.rs), because it calls into both `chat.rs` and [`dash.rs`](../apps/rocm/src/dash.rs) while `dash.rs` already calls into `chat.rs`; this module depends on both and neither depends on it. `Provider` lives in [`cli.rs`](../apps/rocm/src/cli.rs) and `ChatInferenceParams` stays in [`main.rs`](../apps/rocm/src/main.rs); `run_chat_command` is `pub(crate)` and re-imported there for `dispatch()`. Four source-text tests in this module's own `mod tests` read this file and assert on the handler's text, so a rename of the function or of the calls they look for has to be made there too. - [`chat_host_facts.rs`](../apps/rocm/src/chat_host_facts.rs) — full domain extraction. -- [`cli.rs`](../apps/rocm/src/cli.rs) — the `rocm` command line definition: `Cli`, `Command`, the subcommand enums nested under it (`BenchCommand`, `InstallTarget`, `EnginesCommand`, `RuntimesCommand`, `StorageCommand`, `ComfyuiCommand`, `ServicesCommand`, `AutomationsCommand`, `ConfigCommand`, `SetupCommand`) and the clap value enums behind `--provider`, `--device`, `--framework` and similar arguments (`Provider`, `DevicePolicyArg`, `FrameworkArg`, `InstallFormat`, `WatcherModeArg`, `TelemetryModeArg`, `SandboxToolArg`, `PermissionsModeArg`), with `SUPPORTED_ENGINES` (the `--engine` choices) and the conversions from a value enum to its domain type. Like [`apps/rocmd`](../apps/rocmd/src/cli.rs)'s `cli.rs`, it holds definitions only: the `dispatch()` that matches on `Command` stays in [`main.rs`](../apps/rocm/src/main.rs). Declaration order sets the order of every nested `--help` command list (the top-level list is sorted alphabetically by `cli_command()` in `main.rs`), so reordering variants reorders help output. Its items are `pub(crate)` and re-imported in `main.rs`, so other modules keep reaching them as `crate::Command`, `crate::Provider` and so on; [`bootstrap.rs`](../apps/rocm/src/bootstrap.rs) and [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs) own their enums and appear in `Command` as a single variant each. The tests that parse the tree or render its help live in its own `mod tests`. +- [`cli.rs`](../apps/rocm/src/cli.rs) — the `rocm` command line definition: `Cli`, `Command`, the subcommand enums nested under it (`BenchCommand`, `InstallTarget`, `EnginesCommand`, `RuntimesCommand`, `StorageCommand`, `ComfyuiCommand`, `ServicesCommand`, `AutomationsCommand`, `ConfigCommand`, `SetupCommand`) and re-exports the clap value enums behind `--provider`, `--device`, `--framework` and similar arguments, which [`cli_args.rs`](../apps/rocm/src/cli_args.rs) defines, with `SUPPORTED_ENGINES` (the `--engine` choices). Like [`apps/rocmd`](../apps/rocmd/src/cli.rs)'s `cli.rs`, it holds definitions only: the `dispatch()` that matches on `Command` stays in [`main.rs`](../apps/rocm/src/main.rs). Declaration order sets the order of every nested `--help` command list (the top-level list is sorted alphabetically by `cli_command()` in `main.rs`), so reordering variants reorders help output. Its items are `pub(crate)` and re-imported in `main.rs`, so other modules keep reaching them as `crate::Command`, `crate::Provider` and so on; [`bootstrap.rs`](../apps/rocm/src/bootstrap.rs) and [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs) own their enums and appear in `Command` as a single variant each. The tests that parse the tree or render its help live in its own `mod tests`. +- [`cli_args.rs`](../apps/rocm/src/cli_args.rs) — mechanical relocation out of [`cli.rs`](../apps/rocm/src/cli.rs), owns types: the clap value enums behind `--provider`, `--device`, `--framework` and similar arguments (`Provider`, `DevicePolicyArg`, `FrameworkArg`, `InstallFormat`, `WatcherModeArg`, `TelemetryModeArg`, `SandboxToolArg`, `PermissionsModeArg`) and the conversions from a value enum to its domain type. `cli.rs` re-exports every one, so other modules keep reaching them as `crate::cli::Provider` and so on. Moved so `cli.rs`, which every new subcommand grows, stays under the module size limit. - [`cli_audit.rs`](../apps/rocm/src/cli_audit.rs) — mechanical relocation, no owned types: the CLI audit and lifecycle log (`record_cli_audit_event`, plus the helpers behind it that write `logs/cli-lifecycle.log` and the per-action files under `logs/cli/`). Its production code depends only on `rocm-core`. `record_cli_audit_event`, `cli_lifecycle_log_path` and `sanitize_log_value` are `pub(crate)` and reached from other modules as `crate::` through a re-import in [`main.rs`](../apps/rocm/src/main.rs); every other item is private. The two regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::`. - [`cli_progress.rs`](../apps/rocm/src/cli_progress.rs) — shared CLI-output component (`Spinner`, `AnimatedSpinner`). - [`cli_report.rs`](../apps/rocm/src/cli_report.rs) — shared CLI-output component (`ActionReport`, `write_marked_row`, `JsonOrText`/`print_json_or_text`). From 2ca770d0af4cd931fd536e14bb156ea3539f3872 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Tue, 10 Nov 2026 14:56:35 +0000 Subject: [PATCH 27/30] fix(services): keep a container engine's unresolved stop protected, tell an empty relaunch from the stop, and resolve system tools without the environment An unresolved stop of a container-backed engine (a stop was requested and the engine Stop decides confirmation, today aim) is now protected everywhere, not only in the deferred cleanup. The liveness refresh does not read such a record as stopped from the supervisor's PID, and the in-use check behind services remove and prune counts it as in use, so a listing followed by remove or prune no longer deletes a record and key whose container may still be running. One predicate, container_stop_unresolved, is the single reading. is_covered_by_stop_of no longer treats an empty relaunch as the stop. When the stopping snapshot names no process, a record naming none is covered only while it still carries the stop's marker or reads stopped; an empty starting or recovering relaunch has cleared the marker, so the stop does not confirm over it, overwrite its status or delete its key. The deferred cleanup goes on to forget the recorded processes and read the record as stopped only for a record that named at least one process. A starting record with a stale marker gets its key and marker cleaned, as before, and is otherwise left alone. The two unconfirmed tree-kill lines now say that rocm does not keep the warning, so a later stop or listing finds the process gone and reports the stop confirmed, and the test runs that later stop to check the sentence. windows_system_tool reads the system directory with GetSystemDirectoryW instead of building it from the SystemRoot environment variable, which a caller can redirect. The pure path logic keeps its seam and its test. Signed-off-by: Jussi Elo --- apps/rocm/src/managed_service_status.rs | 52 ++++++++++++++++++- apps/rocm/src/service_control.rs | 58 +++++++++++++++++++++ apps/rocm/src/service_records.rs | 57 +++++++++++++++++++-- apps/rocm/src/services_text.rs | 4 +- crates/rocm-core/src/process.rs | 67 ++++++++++++++++--------- crates/rocm-core/src/service_record.rs | 59 ++++++++++++++++++++-- docs/architecture.md | 4 +- docs/testing.md | 29 +++++++++-- 8 files changed, 291 insertions(+), 39 deletions(-) diff --git a/apps/rocm/src/managed_service_status.rs b/apps/rocm/src/managed_service_status.rs index 613e3f2db0..42b7637cc7 100644 --- a/apps/rocm/src/managed_service_status.rs +++ b/apps/rocm/src/managed_service_status.rs @@ -207,6 +207,19 @@ pub(crate) fn recorded_pids_may_be_running( .any(|pid| probe(*pid).may_be_running()) } +/// Whether the record is a container-backed engine's stop that was requested and +/// has not been confirmed. +/// +/// Such a stop is decided by the engine `Stop` (`engine_stop_decides_confirmation`), +/// not by the recorded supervisor, which can be gone while the container still +/// runs. Until a `services stop` confirms it, nothing here may read the record as +/// stopped, complete its cleanup, or let `remove`/`prune` delete it, on the +/// evidence of the supervisor's PID alone. +pub(crate) fn container_stop_unresolved(record: &ManagedServiceRecord) -> bool { + record.stop_requested_unix_ms.is_some() + && rocm_core::engine_stop_decides_confirmation(&record.engine) +} + /// Complete a stop that was requested but could not confirm termination, once /// the processes are actually observed gone: drop the endpoint key, clear the /// marker, forget the recorded processes and read the record as stopped. Returns @@ -246,7 +259,7 @@ fn settle_pending_stop_key_cleanup( // while the container still runs, so PID evidence alone cannot say the stop // completed. The stop that could not confirm keeps its key, marker and // recorded processes, and the next `services stop` asks the engine again. - if rocm_core::engine_stop_decides_confirmation(&record.engine) { + if container_stop_unresolved(record) { return false; } if recorded_pids_may_be_running(record, probe) { @@ -254,6 +267,12 @@ fn settle_pending_stop_key_cleanup( } endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); record.stop_requested_unix_ms = None; + // A record that named no process has no stop to complete: the key and the + // stale marker go, as they always did, but it is not read as stopped. It may + // be a launch that has not recorded its supervisor yet. + if recorded_service_pids(record).is_empty() { + return true; + } // Every recorded process is confirmed gone, so none of them is the // service's any more, and their PIDs are stale from here on. Left in the // record, a later `services stop` would signal whatever the OS has handed @@ -338,6 +357,11 @@ pub(crate) fn refresh_managed_service_runtime_liveness_with( return settled_pending_stop || outcome.record_changed; } + // An unresolved container stop is not read as stopped from the supervisor's + // PID, whether or not the endpoint answers: only the engine `Stop` decides it. + if container_stop_unresolved(record) { + return settled_pending_stop || outcome.record_changed; + } let tracked_pids = recorded_service_pids(record); let has_tracked_pid = !tracked_pids.is_empty(); let has_live_pid = recorded_pids_may_be_running(record, &probe); @@ -1129,6 +1153,32 @@ mod tests { Ok(()) } + /// A record that names no process has no stop to complete: a `starting` + /// record with a stale stop marker gets the marker and key cleaned, as before, + /// but is not read as stopped, since it may be a launch that has not recorded + /// its supervisor yet. + #[test] + fn a_stale_stop_marker_on_a_record_naming_no_process_is_cleaned_without_reading_it_stopped() + -> Result<()> { + let (root, paths) = test_paths("liveness-pending-stop-no-process"); + paths.ensure()?; + let service_id = "svc-no-process"; + let mut record = pending_stop_record(&paths, service_id)?; + "starting".clone_into(&mut record.status); + record.clear_recorded_processes(); + + let changed = refresh_managed_service_runtime_liveness_with(&paths, &mut record, |_| { + ProcessProbe::Gone + }); + + assert!(changed, "the stale marker is consumed and persisted"); + assert_eq!(record.stop_requested_unix_ms, None); + assert!(endpoint_keys::endpoint_key_file_if_present(&paths, service_id).is_none()); + assert_eq!(record.status, "starting", "it is not read as stopped"); + let _ = fs::remove_dir_all(root); + Ok(()) + } + #[test] fn a_completed_pending_stop_reads_stopped_whatever_live_status_it_had() -> Result<()> { for status in ["starting", "running", "ready", "recovering", "failed"] { diff --git a/apps/rocm/src/service_control.rs b/apps/rocm/src/service_control.rs index 95b5bc0292..6719c4827d 100644 --- a/apps/rocm/src/service_control.rs +++ b/apps/rocm/src/service_control.rs @@ -1251,6 +1251,46 @@ mod tests { assert_eq!(key.as_deref(), Some("key-123"), "the key is kept"); } + /// The same as above when the record being stopped names no process either: + /// a relaunch's empty `starting` record then equals the stopping snapshot's + /// processes, which must not make it the stop's own. Its marker is cleared and + /// it reads `starting`, so the stop does not claim it, overwrite its status or + /// delete the key it will serve with. A stop of an empty record that nobody + /// relaunched is still confirmed. + #[test] + fn stop_does_not_confirm_over_an_empty_relaunch_of_an_empty_record() { + let (root, paths) = test_paths("stop-empty-relaunch"); + paths.ensure().unwrap(); + let service_id = "svc-stop-empty-relaunch"; + let mut empty = stop_test_record(&paths, service_id); + empty.clear_recorded_processes(); + empty.write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + let mut relaunch = raw_manifest(&paths, service_id); + relaunch.status = "starting".to_owned(); + relaunch.stop_requested_unix_ms = None; + relaunch.write().unwrap(); + bare_termination(Vec::new(), true) + }) + .unwrap(); + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + + // And with nobody relaunching, the stop of the same empty record is a stop. + let untouched = stop_internal_managed_service_with(&paths, service_id, |_| { + bare_termination(Vec::new(), true) + }) + .unwrap(); + let _ = fs::remove_dir_all(root); + + assert_eq!(report["stopped"], false, "{report}"); + assert_eq!(on_disk.status, "starting", "the relaunch's status stays"); + assert_eq!(key.as_deref(), Some("key-123"), "its key is kept"); + assert_eq!(untouched["stopped"], true, "{untouched}"); + } + #[test] fn stop_of_a_container_engine_is_unconfirmed_when_the_engine_stop_fails() { // For `aim` the recorded PID is only the supervisor; killing it says @@ -3064,6 +3104,16 @@ mod tests { let outcome = finish_service_action(&paths, "stop_server", service_id, &answer); let on_disk = raw_manifest(&paths, service_id); let key = endpoint_keys::endpoint_api_key(&paths, service_id); + // What the report says about the warning, checked against what happens + // next: a later stop finds the root gone and, with no new evidence about + // what was under it, reports the stop confirmed. + let later = stop_internal_managed_service_with(&paths, service_id, |_| { + vec![stop_of( + 999_999_999, + rocm_core::TerminationOutcome::AlreadyGone, + )] + }) + .unwrap(); let _ = fs::remove_dir_all(root); assert_eq!(answer["status"], "stop_unconfirmed", "{answer}"); @@ -3074,6 +3124,14 @@ mod tests { ), "{rendered}" ); + assert!( + rendered.contains("rocm does not keep this warning, so a later stop or listing will find the process gone and report the stop confirmed"), + "the report must not imply the warning outlives this run: {rendered}" + ); + assert_eq!( + later["stopped"], true, + "the claim the report makes is true: {later}" + ); assert!( !rendered.contains("still running: process") && !rendered.contains("did not exit after it was signaled"), diff --git a/apps/rocm/src/service_records.rs b/apps/rocm/src/service_records.rs index 1142e3d72e..2220426374 100644 --- a/apps/rocm/src/service_records.rs +++ b/apps/rocm/src/service_records.rs @@ -39,8 +39,8 @@ use anyhow::{Context, Result, bail}; use rocm_core::{AppPaths, ManagedServiceRecord, ProcessProbe}; use crate::managed_service_status::{ - load_managed_services, managed_service_is_live, recorded_pids_may_be_running, - recorded_service_pids, + container_stop_unresolved, load_managed_services, managed_service_is_live, + recorded_pids_may_be_running, recorded_service_pids, }; use crate::services_text::{load_managed_service, validate_service_id}; use crate::{cli_progress, cli_report, endpoint_keys, service_hints}; @@ -178,7 +178,9 @@ fn managed_service_record_is_in_use_with( record: &ManagedServiceRecord, probe: impl Fn(u32) -> ProcessProbe, ) -> bool { - managed_service_is_live(record) || recorded_pids_may_be_running(record, probe) + managed_service_is_live(record) + || container_stop_unresolved(record) + || recorded_pids_may_be_running(record, probe) } /// The recorded processes `--force` may remove a record over: `Some` only when @@ -2093,6 +2095,55 @@ mod tests { Ok(()) } + /// An `aim` stop whose engine `Stop` failed leaves the stop pending, and its + /// supervisor can be gone while the container still runs. Neither the liveness + /// refresh nor `remove` and `prune` may then treat the record as stopped and + /// delete it, with the key the container may still be enforcing: the refresh + /// leaves the status alone, `remove` refuses it, and `prune` skips it, all + /// with the supervisor confirmed gone. A process engine in the same state is + /// an ordinary stopped record that is removed. + #[test] + fn a_pending_container_stop_survives_refresh_remove_and_prune() -> Result<()> { + let gone = |_: u32| ProcessProbe::Gone; + let (root, paths) = test_paths("services-container-pending-stop"); + paths.ensure()?; + // A record already reading `stopped` (an earlier reconciliation wrote it) + // is the case only the in-use guard protects. + for (engine, status, id, protected) in [ + ("aim", "ready", "svc-container", true), + ("aim", "stopped", "svc-container-stopped", true), + ("vllm", "ready", "svc-process", false), + ] { + let mut record = plant_service_record(&paths, id, status, 999_999_999)?; + record.engine = engine.to_owned(); + record.stop_requested_unix_ms = Some(1); + record.write()?; + + let mut refreshed = record.clone(); + let _ = crate::managed_service_status::refresh_managed_service_runtime_liveness_with( + &paths, + &mut refreshed, + gone, + ); + let removal = remove_managed_service_record_with(&paths, id, true, false, gone); + let pruned = prune_managed_service_records_with(&paths, 0, false, true, &gone)?; + let left = files_left(&paths, id); + + if protected { + assert_eq!(refreshed.status, status, "{engine}: refresh"); + assert_eq!(refreshed.supervisor_pid, 999_999_999, "{engine}: refresh"); + assert!(removal.is_err(), "{engine}: remove must refuse"); + assert_eq!(pruned.removed_records, 0, "{engine}: prune"); + assert_eq!(left, 3, "{engine}: nothing was deleted"); + } else { + assert!(removal.is_ok() || pruned.removed_records == 1, "{engine}"); + assert_eq!(left, 0, "{engine}: an ordinary stopped record goes"); + } + } + let _ = fs::remove_dir_all(root); + Ok(()) + } + /// A `ready` record whose dead server's PID is now held by a process this /// account cannot open, and that nobody has tried to stop: `--force` is /// refused (it never forces a record nobody tried to stop), and the refusal diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index cf876cc9f7..fb0e97f29c 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -1150,13 +1150,13 @@ fn render_unconfirmed_service_stop(output: &serde_json::Value) -> String { for (pid, role) in &descendants_unconfirmed { let _ = writeln!( text, - " not confirmed: process {pid} ({role}) exited, but a process under it may still be running, because taking the processes under it down did not complete" + " not confirmed: process {pid} ({role}) exited, but a process under it may still be running, because taking the processes under it down did not complete; rocm does not keep this warning, so a later stop or listing will find the process gone and report the stop confirmed" ); } for (pid, role) in &exited_before_tree_kill { let _ = writeln!( text, - " not confirmed: process {pid} ({role}) was already gone when the processes under it were to be taken down, and nothing else the server recorded vouches for them, so one of them may still be running" + " not confirmed: process {pid} ({role}) was already gone when the processes under it were to be taken down, and nothing else the server recorded vouches for them, so one of them may still be running; rocm does not keep this warning, so a later stop or listing will find the process gone and report the stop confirmed" ); } for (pid, role) in &unverified { diff --git a/crates/rocm-core/src/process.rs b/crates/rocm-core/src/process.rs index 68b5833c69..f7744ba489 100644 --- a/crates/rocm-core/src/process.rs +++ b/crates/rocm-core/src/process.rs @@ -442,24 +442,50 @@ pub(crate) const fn classify_open_failure(code: Option) -> ProcessProbe { } } -/// The path of a Windows system tool, `%SystemRoot%\System32\`. +/// The path of a Windows system tool in the system directory (`System32`). /// -/// So a directory earlier on `PATH` cannot stand in for it; the bare name when -/// `SystemRoot` is not set. `name` includes the extension (`taskkill.exe`). +/// The directory comes from `GetSystemDirectoryW`, not from the environment, so +/// neither a directory earlier on `PATH` nor a `SystemRoot` the caller controls +/// can stand in for it; the bare name when the directory cannot be read (and off +/// Windows). `name` includes the extension (`taskkill.exe`). #[must_use] pub fn windows_system_tool(name: &str) -> PathBuf { - system_tool_path(std::env::var_os("SystemRoot"), name) + system_tool_path(windows_system_directory(), name) } -/// [`windows_system_tool`] for a given `SystemRoot`, so a test builds the path -/// on every host. -fn system_tool_path(system_root: Option, name: &str) -> PathBuf { - match system_root.filter(|root| !root.is_empty()) { - Some(root) => PathBuf::from(root).join("System32").join(name), +/// [`windows_system_tool`] for a given system directory, so a test builds the +/// path on every host. +fn system_tool_path(system_dir: Option, name: &str) -> PathBuf { + match system_dir.filter(|dir| !dir.as_os_str().is_empty()) { + Some(dir) => dir.join(name), None => PathBuf::from(name), } } +/// The Windows system directory, read with `GetSystemDirectoryW`. +#[cfg(windows)] +#[allow(unsafe_code)] // Win32 FFI +fn windows_system_directory() -> Option { + use std::os::windows::ffi::OsStringExt; + use windows_sys::Win32::System::SystemInformation::GetSystemDirectoryW; + + const MAX_PATH: u32 = 260; + let mut buffer = [0_u16; MAX_PATH as usize]; + // SAFETY: `buffer` is a live local valid for `MAX_PATH` wide characters, and + // that is the size passed, so the call cannot write past it. + let length = unsafe { GetSystemDirectoryW(buffer.as_mut_ptr(), MAX_PATH) } as usize; + // Zero is a failure; a length that does not fit means the path was too long + // for the buffer, which is not a directory to trust either. + (length > 0 && length < buffer.len()) + .then(|| PathBuf::from(std::ffi::OsString::from_wide(&buffer[..length]))) +} + +#[cfg(not(windows))] +#[allow(clippy::missing_const_for_fn)] // keeps the signature of the Windows arm +fn windows_system_directory() -> Option { + None +} + /// E2E hook: `ROCM_E2E_UNQUERYABLE_PIDS`, a comma-separated list of PIDs, makes /// [`probe_process`] answer [`ProcessProbe::Unqueryable`] for each of them. /// @@ -1415,19 +1441,14 @@ mod tests { use crate::test_support::workspace_test_artifact_dir; use std::process::Stdio; - /// A system tool is started from `System32` under `SystemRoot`, never by a - /// bare name that `PATH` could resolve to something else; with no - /// `SystemRoot` it falls back to the bare name. + /// A system tool is started from the system directory, never by a bare name + /// that `PATH` could resolve to something else; with no directory it falls + /// back to the bare name. #[test] - fn a_system_tool_is_resolved_under_system32_of_the_system_root() { - let path = system_tool_path( - Some(std::ffi::OsString::from("C:\\Windows")), - "taskkill.exe", - ); - let mut expected = PathBuf::from("C:\\Windows"); - expected.push("System32"); - expected.push("taskkill.exe"); - assert_eq!(path, expected); + fn a_system_tool_is_resolved_under_the_system_directory() { + let dir = PathBuf::from("C:\\Windows").join("System32"); + let path = system_tool_path(Some(dir.clone()), "taskkill.exe"); + assert_eq!(path, dir.join("taskkill.exe")); assert!(path.ends_with(Path::new("System32").join("taskkill.exe"))); assert!(!path.parent().unwrap().as_os_str().is_empty()); @@ -1436,9 +1457,9 @@ mod tests { PathBuf::from("tasklist.exe") ); assert_eq!( - system_tool_path(Some(std::ffi::OsString::new()), "tasklist.exe"), + system_tool_path(Some(PathBuf::new()), "tasklist.exe"), PathBuf::from("tasklist.exe"), - "an empty SystemRoot is no SystemRoot" + "an empty directory is no directory" ); } diff --git a/crates/rocm-core/src/service_record.rs b/crates/rocm-core/src/service_record.rs index cc4e27017f..429cece2e8 100644 --- a/crates/rocm-core/src/service_record.rs +++ b/crates/rocm-core/src/service_record.rs @@ -344,11 +344,22 @@ impl ManagedServiceRecord { /// /// False for a restart that recorded new processes, and equally for one that /// is mid-launch with none recorded yet: that record reads `starting` (or - /// another live status), not `stopped`, and claiming it stopped, or dropping - /// the key it will serve with, would be false. + /// another live status) with the stop marker cleared, not `stopped`, and + /// claiming it stopped, or dropping the key it will serve with, would be + /// false. That holds when the stopping snapshot itself names no process too: + /// an empty record is covered only while it still carries the stop's marker, + /// or once it reads `stopped`. #[must_use] pub fn is_covered_by_stop_of(&self, stopping: &Self) -> bool { - self.names_same_processes(stopping) || (self.status == "stopped" && self.names_no_process()) + if self.status == "stopped" && self.names_no_process() { + return true; + } + // The same processes. When they are none, that equality is also what a + // relaunch that has not recorded its supervisor yet looks like, so it + // proves nothing: the stop's own marker is what tells the two apart, since + // a restart clears it and a stop that is still finalizing has it on disk. + self.names_same_processes(stopping) + && (!stopping.names_no_process() || self.stop_requested_unix_ms.is_some()) } /// Whether the record names neither a supervisor nor an engine process. @@ -948,6 +959,48 @@ mod tests { for (name, record, covered) in rows { assert_eq!(record.is_covered_by_stop_of(&stopping), covered, "{name}"); } + + // The same table from a stopping snapshot that names no process: an + // untouched record (the stop's own marker still on it, or completed to + // `stopped`) is that stop; an empty relaunch that has cleared the marker is + // not, and neither is one that has recorded processes. + let mut empty = stopping.clone(); + empty.clear_recorded_processes(); + empty.stop_requested_unix_ms = Some(1); + let with_marker = |status: &str, marker: Option, pid: u32| { + let mut record = empty.clone(); + status.clone_into(&mut record.status); + record.stop_requested_unix_ms = marker; + record.supervisor_pid = pid; + record + }; + let rows = [ + ("empty, untouched", with_marker("ready", Some(1), 0), true), + ("empty, stopped", with_marker("stopped", None, 0), true), + ( + "empty, starting relaunch", + with_marker("starting", None, 0), + false, + ), + ( + "empty, recovering relaunch", + with_marker("recovering", None, 0), + false, + ), + ( + "empty, ready relaunch", + with_marker("ready", None, 0), + false, + ), + ( + "relaunched with a process", + with_marker("starting", None, 5555), + false, + ), + ]; + for (name, record, covered) in rows { + assert_eq!(record.is_covered_by_stop_of(&empty), covered, "{name}"); + } } /// Writing a service manifest never writes through a file another writer diff --git a/docs/architecture.md b/docs/architecture.md index fb1cb45472..7b59fa6b6f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -71,7 +71,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-214): `--device`/`--gpu` parsing (`parse_device_policy`, `parse_gpu_selection`, `parse_gpu_indices_arg`), the managed-launch-lock GPU selection (`select_gpu_indices_under_launch_lock`, `validate_pinned_gpu_index`, `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `busy_gpu_indices`), per-GPU VRAM telemetry (the `GpuVramUsage` type, `gpu_vram_usage`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs`, `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`), and the serve-plan low-VRAM warning (`gpu_low_memory_warning`, `vram_capacity_is_meaningful`, `serve_gpu_low_memory_warning`), plus the best-effort `amd-smi`-based GPU count (`detect_gpu_count`).Both `amd-smi` probes go through `rocm_core::amd_smi_json` (the `/dev/kfd` pre-flight) with this module's own `AMD_SMI_SERVE_PROBE_TIMEOUT`, whose doc records the resulting worst-case `serve` latency. `const fn device_policy_name` sits immediately after this cluster's last function in the original file but stayed in `main.rs` rather than moving — it already had callers outside the cluster (`engine_ipc.rs`, `service_launch.rs`, `serve_cmd.rs`) plus its own test. `GpuVramUsage` (and its fields), `gpu_vram_usage`, `parse_gpu_indices_arg`, `serve_gpu_low_memory_warning`, and `vram_capacity_is_meaningful` are `pub(crate)` and read from [`model_fit.rs`](../apps/rocm/src/model_fit.rs) (`classify_accelerator_memory`, `host_accelerator_memory`) [`main.rs`](../apps/rocm/src/main.rs) (the `dispatch()` `Command::EngineServeHttp` arm) and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) (`collect_serve_notes`); `detect_gpu_count`, `gpu_vram_usage`, `parse_device_policy`, `parse_gpu_selection`, `select_gpu_indices_under_launch_lock`, `serve_gpu_low_memory_warning`, and `validate_pinned_gpu_index` are also `pub(crate)` and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs); `parse_device_policy` is also read from [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs)'s `run_builtin_engine_serve_http` and, as of Phase 6f (ROCMAI-209), from [`service_control.rs`](../apps/rocm/src/service_control.rs)'s `restart_child_argv` (moved out of `main.rs` in that same phase, and since relocated from `services_text.rs`). `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs` (both the Linux and non-Linux `cfg` variants), `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`, `gpu_low_memory_warning`, and `busy_gpu_indices` have no callers outside this module and stay private. The cluster's regression tests live in `gpu_policy.rs`'s own `mod tests`; two small test-only fixture builders (`vram`, `host_gpu`) are duplicated rather than shared, since `serve_cmd.rs`'s own `serve_notes_pair_low_vram_with_the_vllm_utilization_hint` (testing `collect_serve_notes`) needs them too. - [`install_cmd.rs`](../apps/rocm/src/install_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm install`'s command handler (`install()`, for the `sdk` and `driver` targets), the `--version`/`--build-date` selector parsing and display it uses (`therock_install_version_selector`, `therock_install_version_selector_display`) and `sdk_install_request`, which maps the parsed `install sdk` arguments onto a `therock::SdkInstallRequest` and moved here from [`chat.rs`](../apps/rocm/src/chat.rs) with its two tests. `InstallTarget` and `InstallFormat` live in [`cli.rs`](../apps/rocm/src/cli.rs). `install()` is `pub(crate)` and re-imported in `main.rs` for `dispatch()`; `therock_install_version_selector` is `pub(crate)` too, and `chat/mcp_bridge.rs`'s `render_install_sdk_dry_run_for_args` reaches it as `crate::therock_install_version_selector` through the same kind of re-import, which is why `sdk_install_request` lives here and not in `chat.rs`: this module does not depend on `chat.rs`. `sdk_install_request` keeps the `pub(crate)` it had, although only `install()` and the tests call it now. - [`logging.rs`](../apps/rocm/src/logging.rs) — mechanical relocation, no owned types, reached by qualified path. -- [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-211): managed-service liveness classification (`managed_service_is_live`, `refresh_managed_service_runtime_liveness`, `existing_live_managed_service_in`'s idempotency guard) and the dashboard-sidebar status/counts helpers derived from it (`ManagedServiceSidebarCounts`/`managed_service_sidebar_counts`/`local_server_sidebar_status`), plus the engine display-name helper `friendly_engine_label`, read only by [`chat/summaries.rs`](../apps/rocm/src/chat/summaries.rs). `ManagedServiceSidebarCounts` is `pub(crate)` to match the visibility of the `pub(crate)` functions that return/take it; `friendly_engine_label`, `existing_live_managed_service_in`, `recorded_service_pids`, `refresh_managed_service_runtime_liveness`, `local_server_sidebar_status`, `managed_service_sidebar_counts`, and `load_managed_services` are `pub(crate)` for the same reason — each has a confirmed caller outside this module. [`service_records.rs`](../apps/rocm/src/service_records.rs) uses its liveness helpers (`managed_service_is_live`, `recorded_service_pids`) for the in-use check behind service-record removal and pruning, both of which, like the deferred key cleanup and the reconciliation to `stopped` here, take a recorded process for gone only when `probe_process` confirms it (a process that cannot be queried counts as possibly running; when the deferred cleanup does complete, it also forgets the recorded processes and reads the record as stopped, so a later stop cannot signal a recycled PID), and `load_managed_services` for prune planning, while [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs) uses `load_managed_services` for the codex-bridge snapshot; [`service_launch.rs`](../apps/rocm/src/service_launch.rs) is the only caller of the `existing_live_managed_service_in` duplicate-launch guard; [`services_text.rs`](../apps/rocm/src/services_text.rs) reads the sidebar helpers; other modules, including [`chat/turn.rs`](../apps/rocm/src/chat/turn.rs), [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs), read the registry and liveness helpers. +- [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-211): managed-service liveness classification (`managed_service_is_live`, `refresh_managed_service_runtime_liveness`, `existing_live_managed_service_in`'s idempotency guard) and the dashboard-sidebar status/counts helpers derived from it (`ManagedServiceSidebarCounts`/`managed_service_sidebar_counts`/`local_server_sidebar_status`), plus the engine display-name helper `friendly_engine_label`, read only by [`chat/summaries.rs`](../apps/rocm/src/chat/summaries.rs). `ManagedServiceSidebarCounts` is `pub(crate)` to match the visibility of the `pub(crate)` functions that return/take it; `friendly_engine_label`, `existing_live_managed_service_in`, `recorded_service_pids`, `refresh_managed_service_runtime_liveness`, `local_server_sidebar_status`, `managed_service_sidebar_counts`, and `load_managed_services` are `pub(crate)` for the same reason — each has a confirmed caller outside this module. [`service_records.rs`](../apps/rocm/src/service_records.rs) uses its liveness helpers (`managed_service_is_live`, `recorded_service_pids`) for the in-use check behind service-record removal and pruning, both of which, like the deferred key cleanup and the reconciliation to `stopped` here, take a recorded process for gone only when `probe_process` confirms it (a process that cannot be queried counts as possibly running; when the deferred cleanup does complete, it also forgets the recorded processes and reads the record as stopped, so a later stop cannot signal a recycled PID; an unconfirmed stop of a container-backed engine, decided by the engine `Stop`, is never completed or read as stopped from the supervisor's PID, and `container_stop_unresolved` makes the in-use check treat it as in use), and `load_managed_services` for prune planning, while [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs) uses `load_managed_services` for the codex-bridge snapshot; [`service_launch.rs`](../apps/rocm/src/service_launch.rs) is the only caller of the `existing_live_managed_service_in` duplicate-launch guard; [`services_text.rs`](../apps/rocm/src/services_text.rs) reads the sidebar helpers; other modules, including [`chat/turn.rs`](../apps/rocm/src/chat/turn.rs), [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs), read the registry and liveness helpers. - [`model_catalog_text.rs`](../apps/rocm/src/model_catalog_text.rs) — mechanical relocation out of `examine_report_text.rs`: the `rocm model` (alias `models`) catalog renderer. `render_model_registry_text_with_context_and_host` prints the curated list grouped by hardware target (`model_catalog_platform_heading`, `model_catalog_header`, `model_recipe_memory_label`, `model_recipe_gpu_fit_label`), and `render_model_registry_verbose_text_with_context_and_host` prints the per-recipe diagnostics behind `rocm model --verbose` (`append_model_recipe_registry_source`, `append_model_recipe_metadata_lines`, `append_model_engine_recipe_settings_lines`, `append_model_recipe_staleness_lines`, `append_model_artifact_lines`, `append_artifact_source_policy_lines`, `artifact_source_policy_label`, `append_model_host_ram_fit_lines`, `append_model_fit_lines`, `append_manual_alternative_lines`, `manual_alternative_recommendations`, `recipe_is_manual_fit`, `append_model_engine_support_lines`, `model_registry_adapter_availability_note`, `recipe_display_ref`, `preferred_engine_action_target`, `model_registry_reason`, plus the small formatters `format_list_or_none`, `format_string_map_or_none`, `format_gib` and `once_per_call`). `format_bytes`, the human-readable size formatter, lives here too. The two renderers and `format_bytes` are `pub(crate)`: [`main.rs`](../apps/rocm/src/main.rs) imports them for the `Command::Model` arm, and [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs) (the verbose renderer), [`storage.rs`](../apps/rocm/src/storage.rs) and [`automations.rs`](../apps/rocm/src/automations.rs) (`format_bytes`) reach them as `crate::`. Every other item, including `model_registry_adapter_availability_note`, is private. The module has no edge to `examine_report_text.rs`; it reaches the engine-availability helpers in [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs), `PlatformRuleOut` and `engine_ruled_out_by_platform_with` at the crate root, and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs)'s `select_serve_engine`. The catalog's regression tests live in this module's own `mod tests`; they keep the names they had in `examine_report_text.rs`. The `format_bytes` property test's saved failure case lives in [`model_catalog_text.txt`](../apps/rocm/proptest-regressions/model_catalog_text.txt), which proptest finds by the test's source file, so the saved case has to follow the test when it moves (see [Property tests](testing.md#property-tests) for how, which depends on whether the whole file moves). - [`model_fit.rs`](../apps/rocm/src/model_fit.rs) — mechanical relocation, owns a `pub(crate)` type: answering whether a host can run a model for `rocm diagnose --model` (`assess_model_on_this_host`, `assess_model_with_host_paths`, `assess_model_for_host`), the accelerator-memory fact it rests on (`host_accelerator_memory`, `classify_accelerator_memory`, `examination_reports_amd_gpu`) and the platform gate that rules an engine out (`PlatformRuleOut`, `engine_ruled_out_by_platform`, `unsupported_here_for`). It is a module of its own, rather than part of the `diagnose` handler, so [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs) and [`model_catalog_text.rs`](../apps/rocm/src/model_catalog_text.rs), which read `PlatformRuleOut` and the platform gate (`engine_ruled_out_by_platform` and `engine_ruled_out_by_platform_with` respectively), do not depend on a command module. It depends on [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) (`select_serve_engine`) and [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs), neither of which depends on it. `assess_model_on_this_host` (called from [`diagnose_cmd.rs`](../apps/rocm/src/diagnose_cmd.rs)), `PlatformRuleOut` and `engine_ruled_out_by_platform` are `pub(crate)`, and [`main.rs`](../apps/rocm/src/main.rs) re-imports the last two so `examine_report_text.rs` reaches them as `crate::` (`model_catalog_text.rs` reaches `PlatformRuleOut` and `engine_ruled_out_by_platform_with` the same way); every other item is private. The cluster's regression tests live in this module's own `mod tests`; the one that re-runs the test binary on a helper test by its full path (`engine_ruled_out_by_platform_gates_aim_on_docker_reachability`) names the helper's new path, `model_fit::tests::docker_unreachable_subprocess_helper`. - [`prompt_text.rs`](../apps/rocm/src/prompt_text.rs) — text extraction from a natural-language request, shared by `chat/intent.rs`'s prompt-intent heuristics (`fallback_rocm_tool_call_for_prompt`, `fallback_config_tool_call_for_prompt`, `install_sdk_chat_approval_for_prompt` and the `prompt_*` predicates) and the freeform planner, in a leaf module that depends only on `therock::RuntimeVersionSelector` (`tool_policy.rs` does not use it): `any_substring`, the install-folder, TheRock build-date and TheRock version readers (`requested_install_prefix_from_prompt`, `requested_therock_build_date_from_prompt`, `requested_therock_version_from_prompt`, with `requested_bare_install_prefix_from_prompt`, `clean_requested_install_prefix`, `looks_like_user_path` and `looks_like_therock_runtime_version` private beside them) from `chat.rs`, and `infer_engine_from_request` from `freeform_planner.rs`. They moved verbatim, together with `infer_engine_from_request`'s regression test, so that `freeform_planner.rs` no longer imports `chat` for them; `chat.rs` re-exports `any_substring`, which [`freeform_answer.rs`](../apps/rocm/src/freeform_answer.rs) still reaches as `chat::any_substring`. @@ -246,7 +246,7 @@ Further `rocm-core` modules, each with its own doc comment except where noted: - [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) — OpenMPI, libatomic and libnuma detection, the `libmpi_cxx` compatibility shim, and the per-library install plans (`build_openmpi_install_plan`, `build_libatomic_install_plan`, `build_libnuma_install_plan`) and hints, needed by vLLM's ROCm wheels. The distro-generic half of that planning lives in [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs); [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) re-exports its public items, so `rocm_core::openmpi::running_as_root`, `plan_for_os_release_read` and the rest keep resolving, and `pub(crate)` re-exports `PackageManager` and `resolve_package_manager` for [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs). - [`os_release.rs`](../crates/rocm-core/src/os_release.rs) — the one `/etc/os-release` parser every reader of that file uses. - [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs) — the planning half of a system-package install that does not depend on which library is installed: `PackageManager` and `resolve_package_manager` (from an os-release `ID`/`ID_LIKE`), `SystemPackageInstallPlan` and `InstallCommand`, `plan_for_os_release_at`/`plan_for_os_release_read` (which take the per-library builder as an argument), `InstallHint`, the root and passwordless-`sudo` checks (`running_as_root`, `can_autoinstall`) and the `apt-get -s` helpers (`apt_simulate_argv`, `parse_apt_simulate_removals`, `is_protected_rocm_package`). It never runs anything and depends on no other module of this crate except [`os_release.rs`](../crates/rocm-core/src/os_release.rs) and [`hardware_root.rs`](../crates/rocm-core/src/hardware_root.rs), which it reads os-release through. The module is private; its public items are re-exported from [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs). It is a different module from [`system_packages.rs`](../apps/rocm/src/system_packages.rs) in `apps/rocm`, which runs these plans with consent and `sudo`. -- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches (`terminate_verified`), and the one managed-stop routine built on it that `rocm services stop` and `rocmd`'s `stop_server` both call: `terminate_recorded_service_pids` walks the processes a service record names (supervisor, then engine, each with its own start-time, the stopping process's own PID reported as `self` and never signalled) and returns one `RecordedPidStop` per PID (each caller, on a stop that confirms every one gone, then clears the record's recorded processes with `ManagedServiceRecord::clear_recorded_processes` in [`service_record.rs`](../crates/rocm-core/src/service_record.rs), so a repeated stop reaches no stale PID; an unconfirmed stop keeps them); `terminate_recorded_pid` terminates one of them, adding on Windows a `taskkill /T /F` of its descendants first, because `terminate_verified` cannot walk a process tree there, and only for a PID that passes the same identity check (`process_start_ticks` is the kernel start-time on Linux and the process creation time on Windows, so a recycled PID is refused before any `taskkill`; a record written before Windows recorded one cannot be refuted), holding the PID open on Windows from before that check until the stop ends so the PID cannot be reused in between (a PID that matches but cannot be held is reported unverified and not signalled), and treating a tree kill that did not complete, or did not finish within `MANAGED_STOP_GRACE` (the `taskkill` is killed and reaped at that deadline), as an unconfirmed stop (`descendants_unconfirmed` when the root itself was seen to exit, `timed_out` only when it outlived its deadline), except that a `taskkill` which found no such process is judged by the root's own re-read, so a root that exited by itself is `exited_before_tree_kill`, which the record-wide settling in `terminate_recorded_service_pids` turns into a stop only when another recorded process (the engine, which is visited as its own role and folded by both callers) is confirmed gone, and starting that `taskkill` from `%SystemRoot%\System32` through `windows_system_tool`; a PID that cannot be queried at all, as against one that is gone, reads as unverifiable unless a readable recorded token differs, in which case that mismatch wins (`ProcessProbe` and `probe_process` in [`process.rs`](../crates/rocm-core/src/process.rs), whose `may_be_running` is the answer for any decision that must not take an unqueryable process for gone); `MANAGED_STOP_GRACE` is the one grace period. The termination is injectable, so a caller's test supplies outcomes it cannot construct and a test of the Windows step runs on every host. +- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches (`terminate_verified`), and the one managed-stop routine built on it that `rocm services stop` and `rocmd`'s `stop_server` both call: `terminate_recorded_service_pids` walks the processes a service record names (supervisor, then engine, each with its own start-time, the stopping process's own PID reported as `self` and never signalled) and returns one `RecordedPidStop` per PID (each caller, on a stop that confirms every one gone, then clears the record's recorded processes with `ManagedServiceRecord::clear_recorded_processes` in [`service_record.rs`](../crates/rocm-core/src/service_record.rs), so a repeated stop reaches no stale PID; an unconfirmed stop keeps them); `terminate_recorded_pid` terminates one of them, adding on Windows a `taskkill /T /F` of its descendants first, because `terminate_verified` cannot walk a process tree there, and only for a PID that passes the same identity check (`process_start_ticks` is the kernel start-time on Linux and the process creation time on Windows, so a recycled PID is refused before any `taskkill`; a record written before Windows recorded one cannot be refuted), holding the PID open on Windows from before that check until the stop ends so the PID cannot be reused in between (a PID that matches but cannot be held is reported unverified and not signalled), and treating a tree kill that did not complete, or did not finish within `MANAGED_STOP_GRACE` (the `taskkill` is killed and reaped at that deadline), as an unconfirmed stop (`descendants_unconfirmed` when the root itself was seen to exit, `timed_out` only when it outlived its deadline), except that a `taskkill` which found no such process is judged by the root's own re-read, so a root that exited by itself is `exited_before_tree_kill`, which the record-wide settling in `terminate_recorded_service_pids` turns into a stop only when another recorded process (the engine, which is visited as its own role and folded by both callers) is confirmed gone, and starting that `taskkill` from the system directory (`GetSystemDirectoryW`, not the `SystemRoot` variable) through `windows_system_tool`; a PID that cannot be queried at all, as against one that is gone, reads as unverifiable unless a readable recorded token differs, in which case that mismatch wins (`ProcessProbe` and `probe_process` in [`process.rs`](../crates/rocm-core/src/process.rs), whose `may_be_running` is the answer for any decision that must not take an unqueryable process for gone); `MANAGED_STOP_GRACE` is the one grace period. The termination is injectable, so a caller's test supplies outcomes it cannot construct and a test of the Windows step runs on every host. - [`report.rs`](../crates/rocm-core/src/report.rs) — the content of a Doctor report and the rule deciding whether one may exist; it sends nothing. - [`report_delivery.rs`](../crates/rocm-core/src/report_delivery.rs) — how a report leaves the machine: building its link and deciding whether a browser may be started. - [`runtime.rs`](../crates/rocm-core/src/runtime.rs) — the host-platform layer: `RuntimePlatform`/`RuntimeHost`, OS-specific executable and Python-environment names (including `runtime_python_env_root_for_executable`, the one rule for finding an environment's folder from its interpreter's path: only an interpreter directly inside a `bin` or `Scripts` folder, in any case, has one, and everything else gets `None`; the adopt command, the ComfyUI runtime environment and the vLLM launch all call it rather than keeping a copy, since the engine crate cannot reach `apps/rocm`), default config/data/cache directories, path-text and path-list helpers, and `install_root_has_payload`, the one definition of a non-empty install root (dot-entries such as the runtime marker do not count; a path that does not exist holds nothing; a root that cannot be listed is an error) that tarball extraction, update repair detection and activation validation in `apps/rocm` all ask (no module doc comment). diff --git a/docs/testing.md b/docs/testing.md index 1d309eb2de..8f776ddd6c 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2192,7 +2192,11 @@ process under it may still be running; it is not `timed_out`, which is kept for a root that outlived its deadline. `rocm services stop` prints it as a `not confirmed:` line, and `a_stop_with_an_unconfirmed_tree_kill_says_what_happened_and_keeps_the_service` asserts that line, that no `still running` line is printed, and the record, stop -request and key that are kept +request and key that are kept. The warning lasts one run, and the line says so: the +next `rocm services stop` finds the root gone and reports the stop confirmed, and +a listing completes the pending stop the same way, neither with new evidence about +the processes under the root (the same test runs that second stop and asserts it); +keeping it durably would need a marker in the record, which is not done (`a_tree_kill_that_did_not_complete_leaves_the_stop_unconfirmed`, `only_a_clean_taskkill_exit_is_a_completed_tree_kill_and_128_is_not_found`). A `taskkill` that exits 128 (process not found) is judged by the root's own re-read: @@ -2224,9 +2228,12 @@ elapsed time, the reaped child and the failed verdict; `a_helper_that_exits_in_t and `a_helper_that_cannot_be_started_is_an_error` are its neighbours). The report and the record a `timed_out` stop leaves are covered by `an_unconfirmed_stop_keeps_the_recorded_processes_for_the_next_stop`. The -`taskkill` is started from `%SystemRoot%\System32`, not by a bare name `PATH` -could resolve (`a_system_tool_is_resolved_under_system32_of_the_system_root` -builds the path on every host; ComfyUI's own `taskkill` and `tasklist` calls use +`taskkill` is started from the system directory that `GetSystemDirectoryW` +returns, not by a bare name `PATH` could resolve and not from the `SystemRoot` +environment variable a caller could redirect +(`a_system_tool_is_resolved_under_the_system_directory` builds the path from a +given directory on every host; the Windows call itself is run only by the +Windows job's tests that start `taskkill`; ComfyUI's own `taskkill` and `tasklist` calls use the same helper). A readable recorded token that differs outranks a failed query (`a_readable_token_mismatch_outranks_a_failed_query`), but the creation time and @@ -2278,6 +2285,15 @@ container still runs, and only the engine `Stop` decides that stop, so a listing does not complete it from PID evidence. The key, the marker and the recorded processes stay until a later `services stop` is confirmed (`a_pending_stop_of_a_container_engine_is_not_completed_by_a_gone_supervisor`). +The same holds past the cleanup: the refresh does not read such a record as +`stopped` from the supervisor's PID, and `remove` and `prune` count it as in use +(`container_stop_unresolved`), so a listing followed by `remove --yes` or +`prune --yes` deletes neither the record nor the key the container may still be +enforcing (`a_pending_container_stop_survives_refresh_remove_and_prune`, for a +record reading `ready` and one already reading `stopped`; a process engine in the +same state is removed). A record that named no process at all, such as a `starting` +one with a stale marker, has its key and marker cleaned as before but is not read as +`stopped` (`a_stale_stop_marker_on_a_record_naming_no_process_is_cleaned_without_reading_it_stopped`). When that refresh completes a pending stop (every recorded process confirmed gone) it also forgets the recorded processes and reads the record as stopped, as a stop that confirmed the exit itself does, so a later `rocm services stop` visits no PID: a PID @@ -2356,7 +2372,10 @@ gone completed this same stop first (the record reads `stopped` and names nothing). A restart, whether it has recorded new processes or is mid-launch with none yet (status `starting`), is not covered. The unit test `a_stop_is_covered_by_the_same_processes_or_by_its_own_completion_and_not_by_a_relaunch` -is the table; in `rocm`, `a_listing_that_completes_the_stop_before_finalization_does_not_unconfirm_it` +is the table, and it runs again from a stopping snapshot that names no process (an +empty record is that stop only while it carries the stop's marker or reads +`stopped`; an empty `starting` relaunch has cleared the marker and is not covered, +`stop_does_not_confirm_over_an_empty_relaunch_of_an_empty_record`); in `rocm`, `a_listing_that_completes_the_stop_before_finalization_does_not_unconfirm_it` persists a real liveness refresh between the termination and the final read and asserts a confirmed stop with the marker not restored, `a_restart_is_not_refused_because_a_listing_completed_its_stop` that a restart then From e04c9db6d547258fa99df909d07bfcdbae868ea2 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Tue, 10 Nov 2026 18:08:47 +0000 Subject: [PATCH 28/30] fix(services): keep the evidence of a stop's own marker when a listing consumes it off an empty record A stop that begins on a record naming no process writes its marker and has nothing to signal. A concurrent listing finds the marker with no process to wait for and clears it, but leaves a starting or failed record at its status, so the stop's re-read saw an empty record with no marker, which is also what a relaunch that has not recorded its supervisor yet looks like. The stop reported itself unconfirmed, and so did the restart built on it. The refresh now keeps the consumed marker's value in settled_stop_unix_ms, and is_covered_by_stop_of accepts an empty record whose stamp equals the stop's own request stamp. A restart clears the stamp, so a real empty relaunch is still not covered. Signed-off-by: Jussi Elo --- apps/rocm/src/managed_service_status.rs | 17 ++++- apps/rocm/src/service_control.rs | 96 +++++++++++++++++++++++++ apps/rocmd/src/service.rs | 41 +++++++++++ crates/rocm-core/src/service_record.rs | 74 ++++++++++++++++++- docs/testing.md | 20 ++++-- 5 files changed, 239 insertions(+), 9 deletions(-) diff --git a/apps/rocm/src/managed_service_status.rs b/apps/rocm/src/managed_service_status.rs index 42b7637cc7..15deaef215 100644 --- a/apps/rocm/src/managed_service_status.rs +++ b/apps/rocm/src/managed_service_status.rs @@ -266,11 +266,15 @@ fn settle_pending_stop_key_cleanup( return false; } endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); - record.stop_requested_unix_ms = None; + let consumed_marker = record.stop_requested_unix_ms.take(); // A record that named no process has no stop to complete: the key and the // stale marker go, as they always did, but it is not read as stopped. It may - // be a launch that has not recorded its supervisor yet. + // be a launch that has not recorded its supervisor yet. The marker's value + // stays on the record (`settled_stop_unix_ms`), because a stop that began on + // this empty record and finds it with no marker would otherwise read this + // cleanup as a relaunch and report itself unconfirmed. if recorded_service_pids(record).is_empty() { + record.settled_stop_unix_ms = consumed_marker; return true; } // Every recorded process is confirmed gone, so none of them is the @@ -1175,6 +1179,11 @@ mod tests { assert_eq!(record.stop_requested_unix_ms, None); assert!(endpoint_keys::endpoint_key_file_if_present(&paths, service_id).is_none()); assert_eq!(record.status, "starting", "it is not read as stopped"); + assert_eq!( + record.settled_stop_unix_ms, + Some(1), + "the consumed marker's stamp stays, so a stop that began on this record can tell its own marker was consumed from a relaunch having cleared it" + ); let _ = fs::remove_dir_all(root); Ok(()) } @@ -1195,6 +1204,10 @@ mod tests { assert!(changed, "{status}"); assert_eq!(record.status, "stopped", "{status}"); assert_eq!(record.supervisor_pid, 0, "{status}"); + assert_eq!( + record.settled_stop_unix_ms, None, + "{status}: a record that names processes reads stopped and needs no stamp" + ); let _ = fs::remove_dir_all(root); } Ok(()) diff --git a/apps/rocm/src/service_control.rs b/apps/rocm/src/service_control.rs index 6719c4827d..b2e65e2d4e 100644 --- a/apps/rocm/src/service_control.rs +++ b/apps/rocm/src/service_control.rs @@ -1291,6 +1291,102 @@ mod tests { assert_eq!(untouched["stopped"], true, "{untouched}"); } + /// The interleaving the empty-relaunch guard above must not mistake for a + /// relaunch: the record being stopped names no process and reads `starting` or + /// `failed` (a liveness refresh leaves both as they are), and a concurrent + /// listing finds the stop marker with nothing recorded to wait for and clears + /// it before the stop reads the record again. Nobody relaunched, so the stop is + /// confirmed as it is when no listing intervenes (the last block), and the + /// record reads `stopped` afterwards with its key gone. + #[test] + fn a_listing_clearing_the_marker_of_an_empty_stop_does_not_make_it_unconfirmed() { + for status in ["starting", "failed"] { + let (root, paths) = test_paths(&format!("stop-empty-listing-{status}")); + paths.ensure().unwrap(); + let service_id = "svc-stop-empty-listing"; + let mut empty = stop_test_record(&paths, service_id); + status.clone_into(&mut empty.status); + empty.clear_recorded_processes(); + empty.write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |stopping| { + assert!(stopping.stop_requested_unix_ms.is_some()); + // The listing under test: it reads the record with the stop's + // marker on it, finds no process to wait for, and persists the + // cleanup before this stop reads the record again. + let (mut listed, _) = read_managed_service_manifest(&paths, service_id).unwrap(); + let changed = + crate::managed_service_status::refresh_managed_service_runtime_liveness_with( + &paths, + &mut listed, + |_| rocm_core::ProcessProbe::Gone, + ); + assert!( + changed, + "premise: the listing cleaned the marker ({status})" + ); + listed.write().unwrap(); + let after_listing = raw_manifest(&paths, service_id); + assert_eq!(after_listing.stop_requested_unix_ms, None, "{status}"); + assert_eq!(after_listing.status, status, "premise: status unchanged"); + bare_termination(Vec::new(), true) + }) + .unwrap(); + + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + assert_eq!(report["stopped"], true, "{status}: {report}"); + assert_eq!(on_disk.status, "stopped", "{status}"); + assert_eq!(on_disk.stop_requested_unix_ms, None, "{status}"); + assert_eq!(key, None, "{status}: the key goes with the confirmed stop"); + } + } + + /// The same interleaving followed by a real relaunch: the listing consumed the + /// marker, then the daemon restarted the empty record (a restart counts itself, + /// clears the marker and reads `recovering` or `starting`). The stamp the + /// listing left is the stop's own, but the relaunch wipes it, so the stop still + /// does not claim the relaunch's record. + #[test] + fn a_relaunch_after_a_listing_consumed_the_marker_is_still_not_the_stop() { + let (root, paths) = test_paths("stop-empty-listing-then-relaunch"); + paths.ensure().unwrap(); + let service_id = "svc-stop-empty-listing-then-relaunch"; + let mut empty = stop_test_record(&paths, service_id); + "starting".clone_into(&mut empty.status); + empty.clear_recorded_processes(); + empty.write().unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + let (mut listed, _) = read_managed_service_manifest(&paths, service_id).unwrap(); + let _ = crate::managed_service_status::refresh_managed_service_runtime_liveness_with( + &paths, + &mut listed, + |_| rocm_core::ProcessProbe::Gone, + ); + listed.write().unwrap(); + let mut relaunch = raw_manifest(&paths, service_id); + assert!( + relaunch.settled_stop_unix_ms.is_some(), + "premise: the listing left the stop's stamp" + ); + "recovering".clone_into(&mut relaunch.status); + relaunch.reset_for_restart(); + relaunch.stop_requested_unix_ms = None; + relaunch.write().unwrap(); + bare_termination(Vec::new(), true) + }) + .unwrap(); + + let on_disk = raw_manifest(&paths, service_id); + let _ = fs::remove_dir_all(root); + assert_eq!(report["stopped"], false, "{report}"); + assert_eq!(on_disk.status, "recovering", "the relaunch's status stays"); + assert_eq!(on_disk.restart_count, 1); + } + #[test] fn stop_of_a_container_engine_is_unconfirmed_when_the_engine_stop_fails() { // For `aim` the recorded PID is only the supervisor; killing it says diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index caf05aa48e..1b199b022b 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -897,6 +897,47 @@ mod tests { Ok(()) } + /// The listing's cleanup of an empty record (see above) leaves a `starting` or + /// `failed` record at its status with the marker gone and the marker's stamp + /// kept: that is the stop's own marker consumed, not a relaunch, so the stop + /// is confirmed. `rocm` has the test that runs the real refresh. + #[test] + fn a_stop_of_an_empty_record_whose_marker_a_listing_consumed_is_still_confirmed() -> Result<()> + { + for status in ["starting", "failed"] { + let (root, paths) = temp_app_paths(&format!("stop-empty-listing-{status}")); + paths.ensure()?; + let service_id = "svc-stop-empty-listing"; + let key_path = seed_keyed_service(&paths, service_id, 11457)?; + let mut empty = load_service_record(&paths, service_id)?; + empty.clear_recorded_processes(); + status.clone_into(&mut empty.status); + empty.write()?; + + let result = stop_managed_service_with(&paths, service_id, |_| { + let mut on_disk = load_service_record(&paths, service_id).expect("record"); + on_disk.settled_stop_unix_ms = on_disk.stop_requested_unix_ms.take(); + on_disk.write().expect("concurrent listing write"); + rocm_core::TerminationOutcome::Graceful + }); + let key_removed = !key_path.exists(); + let reloaded = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let value = result?; + assert_eq!( + value.get("stopped").and_then(Value::as_bool), + Some(true), + "{status}: {value}" + ); + let reloaded = reloaded?; + assert_eq!(reloaded.status, "stopped", "{status}"); + assert_eq!(reloaded.stop_requested_unix_ms, None, "{status}"); + assert!(key_removed, "{status}"); + } + Ok(()) + } + /// A restart that has begun launching but recorded no process yet reads /// `starting` with nothing named: not the stop completing, so the stop does /// not claim it or drop the key. diff --git a/crates/rocm-core/src/service_record.rs b/crates/rocm-core/src/service_record.rs index 429cece2e8..1943af3c1f 100644 --- a/crates/rocm-core/src/service_record.rs +++ b/crates/rocm-core/src/service_record.rs @@ -109,6 +109,20 @@ pub struct ManagedServiceRecord { /// existed, which is the safe default (keep the key). #[serde(default)] pub stop_requested_unix_ms: Option, + /// The stop marker a liveness refresh cleared off a record that named no + /// process, kept as the evidence that it was cleared and not by a relaunch. + /// + /// A record naming no process has nothing for a pending stop to wait on, so + /// the refresh drops the marker (and the key) at once. A `starting` or + /// `failed` record is left at that status, and a stop that began on such a + /// record would then find it with no marker and nothing recorded, which is + /// also what a relaunch that has not recorded its supervisor yet looks like. + /// This holds the marker's value (the stop's own request stamp), so the stop + /// can tell its own marker having been consumed from a relaunch having + /// cleared it ([`Self::is_covered_by_stop_of`]). Cleared by a restart + /// ([`Self::reset_for_restart`]); absent from records that never had one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub settled_stop_unix_ms: Option, /// When the last inference probe was attempted. Throttles re-probing of a /// service that is listed but still loading — see /// [`crate::INFERENCE_PROBE_RETRY_INTERVAL`]. Absent on records written before @@ -254,6 +268,7 @@ impl ManagedServiceRecord { restart_count: 0, last_restart_unix_ms: None, stop_requested_unix_ms: None, + settled_stop_unix_ms: None, startup_phase: None, inference_verified_at_unix_ms: None, inference_probe_attempted_at_unix_ms: None, @@ -298,6 +313,8 @@ impl ManagedServiceRecord { // died with the previous run. self.anthropic_endpoint_url = None; self.anthropic_endpoint_mode = None; + // A relaunch is no longer the stop that was settled under it. + self.settled_stop_unix_ms = None; self.restart_count = self.restart_count.saturating_add(1); self.last_restart_unix_ms = Some(unix_time_millis()); } @@ -348,7 +365,9 @@ impl ManagedServiceRecord { /// claiming it stopped, or dropping the key it will serve with, would be /// false. That holds when the stopping snapshot itself names no process too: /// an empty record is covered only while it still carries the stop's marker, - /// or once it reads `stopped`. + /// once it reads `stopped`, or once a liveness refresh consumed that very + /// marker ([`Self::settled_stop_unix_ms`]), which leaves a `starting` or + /// `failed` record at its status with the marker gone. #[must_use] pub fn is_covered_by_stop_of(&self, stopping: &Self) -> bool { if self.status == "stopped" && self.names_no_process() { @@ -358,8 +377,13 @@ impl ManagedServiceRecord { // relaunch that has not recorded its supervisor yet looks like, so it // proves nothing: the stop's own marker is what tells the two apart, since // a restart clears it and a stop that is still finalizing has it on disk. + // A marker that is gone is the stop's own only when the refresh that + // consumed it left its stamp behind; a restart wipes that stamp. self.names_same_processes(stopping) - && (!stopping.names_no_process() || self.stop_requested_unix_ms.is_some()) + && (!stopping.names_no_process() + || self.stop_requested_unix_ms.is_some() + || (stopping.stop_requested_unix_ms.is_some() + && self.settled_stop_unix_ms == stopping.stop_requested_unix_ms)) } /// Whether the record names neither a supervisor nor an engine process. @@ -974,9 +998,35 @@ mod tests { record.supervisor_pid = pid; record }; + // What a liveness refresh leaves when it consumes the stop's marker off an + // empty record: the marker gone, the status as it was, the marker's value + // kept as `settled_stop_unix_ms`. + let settled = |status: &str, stamp: Option| { + let mut record = with_marker(status, None, 0); + record.settled_stop_unix_ms = stamp; + record + }; let rows = [ ("empty, untouched", with_marker("ready", Some(1), 0), true), ("empty, stopped", with_marker("stopped", None, 0), true), + // The stop's own marker consumed by a listing: not a relaunch. + ( + "empty, starting, marker consumed by a listing", + settled("starting", Some(1)), + true, + ), + ( + "empty, failed, marker consumed by a listing", + settled("failed", Some(1)), + true, + ), + // A stamp that is not this stop's (an earlier stop's), or none at all, + // is no evidence about this one. + ( + "empty, starting, another stop's marker consumed", + settled("starting", Some(2)), + false, + ), ( "empty, starting relaunch", with_marker("starting", None, 0), @@ -1001,6 +1051,26 @@ mod tests { for (name, record, covered) in rows { assert_eq!(record.is_covered_by_stop_of(&empty), covered, "{name}"); } + + // Nothing is learned from two absent stamps: a snapshot without a marker + // of its own is not matched by a record that never consumed one. + let mut unmarked = empty.clone(); + unmarked.stop_requested_unix_ms = None; + assert!( + !settled("starting", None).is_covered_by_stop_of(&unmarked), + "None must not equal None as evidence" + ); + } + + /// A restart supersedes the stop that was settled under it: the stamp a + /// listing left behind does not outlive the relaunch, or a stop that began + /// before it would read the relaunch's empty record as its own completion. + #[test] + fn restarting_forgets_a_settled_stop() { + let mut record = probe_test_record(11439); + record.settled_stop_unix_ms = Some(7); + record.reset_for_restart(); + assert_eq!(record.settled_stop_unix_ms, None); } /// Writing a service manifest never writes through a file another writer diff --git a/docs/testing.md b/docs/testing.md index 74c9bde41c..282416d671 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2392,9 +2392,17 @@ nothing). A restart, whether it has recorded new processes or is mid-launch with none yet (status `starting`), is not covered. The unit test `a_stop_is_covered_by_the_same_processes_or_by_its_own_completion_and_not_by_a_relaunch` is the table, and it runs again from a stopping snapshot that names no process (an -empty record is that stop only while it carries the stop's marker or reads -`stopped`; an empty `starting` relaunch has cleared the marker and is not covered, -`stop_does_not_confirm_over_an_empty_relaunch_of_an_empty_record`); in `rocm`, `a_listing_that_completes_the_stop_before_finalization_does_not_unconfirm_it` +empty record is that stop only while it carries the stop's marker, reads +`stopped`, or a liveness refresh consumed that marker and left its stamp in +`settled_stop_unix_ms`; an empty `starting` relaunch has cleared the marker +without leaving the stamp and is not covered, +`stop_does_not_confirm_over_an_empty_relaunch_of_an_empty_record`); in `rocm`, +`a_listing_clearing_the_marker_of_an_empty_stop_does_not_make_it_unconfirmed` runs the +real refresh between the termination and the final read of an empty `starting` and an +empty `failed` record (which the refresh leaves at their status) and asserts a confirmed +stop, `a_relaunch_after_a_listing_consumed_the_marker_is_still_not_the_stop` that a +restart following that listing is still not covered (it wipes the stamp), and +`a_listing_that_completes_the_stop_before_finalization_does_not_unconfirm_it` persists a real liveness refresh between the termination and the final read and asserts a confirmed stop with the marker not restored, `a_restart_is_not_refused_because_a_listing_completed_its_stop` that a restart then @@ -2402,8 +2410,10 @@ carries on past its stop, and `stop_does_not_confirm_over_a_restart_that_has_rec and `stop_does_not_confirm_when_a_restart_recorded_new_processes_meanwhile` that a real relaunch is still detected; `rocmd` has the same three (`a_stop_completed_by_a_concurrent_listing_is_still_confirmed`, `a_stop_does_not_claim_a_restart_that_has_recorded_no_process_yet`, -`a_stop_does_not_clear_processes_recorded_while_it_ran`), the first with the listing's -write made directly because the refresh lives in the `rocm` binary. +`a_stop_does_not_clear_processes_recorded_while_it_ran`) plus +`a_stop_of_an_empty_record_whose_marker_a_listing_consumed_is_still_confirmed`, the +listing's write made directly in those that need it because the refresh lives in the +`rocm` binary. `features/service_stop.feature` (`service-stop-02`, Linux, mock lane) drives the real `rocmd sandbox-tool stop_server` against a live recorded From b4672813cc6427d481d9ffdb9812960f79f27e79 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Tue, 10 Nov 2026 18:46:52 +0000 Subject: [PATCH 29/30] test(rocmd): drive the listing's rewrite from the engine stop callback, where an empty record can reach it The test of a stop whose marker a concurrent listing consumed put the simulated listing in the per-PID termination callback. The record names no process, so that callback never ran and the test passed whether or not the settled stamp was honoured. The rewrite now happens in the engine stop callback, which does run for a record naming no process, the termination callback is unreachable, and the test asserts that the interleaving occurred. Signed-off-by: Jussi Elo --- apps/rocmd/src/service.rs | 31 +++++++++++++++++++++++-------- docs/testing.md | 3 ++- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index 1b199b022b..c0579a668c 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -900,7 +900,10 @@ mod tests { /// The listing's cleanup of an empty record (see above) leaves a `starting` or /// `failed` record at its status with the marker gone and the marker's stamp /// kept: that is the stop's own marker consumed, not a relaunch, so the stop - /// is confirmed. `rocm` has the test that runs the real refresh. + /// is confirmed. A record naming no process has no PID to terminate, so the + /// per-PID callback never runs; the engine `Stop` does (for every engine but + /// lemonade), and that is where the listing's write is made, with an assertion + /// that it happened. `rocm` has the test that runs the real refresh. #[test] fn a_stop_of_an_empty_record_whose_marker_a_listing_consumed_is_still_confirmed() -> Result<()> { @@ -913,17 +916,29 @@ mod tests { empty.clear_recorded_processes(); status.clone_into(&mut empty.status); empty.write()?; - - let result = stop_managed_service_with(&paths, service_id, |_| { - let mut on_disk = load_service_record(&paths, service_id).expect("record"); - on_disk.settled_stop_unix_ms = on_disk.stop_requested_unix_ms.take(); - on_disk.write().expect("concurrent listing write"); - rocm_core::TerminationOutcome::Graceful - }); + let interleaved = std::cell::Cell::new(false); + + let result = stop_managed_service_via( + &paths, + service_id, + |_| unreachable!("a record naming no process has no PID to terminate"), + |_, _| { + let mut on_disk = load_service_record(&paths, service_id).expect("record"); + assert!(on_disk.stop_requested_unix_ms.is_some(), "{status}"); + on_disk.settled_stop_unix_ms = on_disk.stop_requested_unix_ms.take(); + on_disk.write().expect("concurrent listing write"); + interleaved.set(true); + Ok(StopResponse { + stopped: true, + graceful: true, + }) + }, + ); let key_removed = !key_path.exists(); let reloaded = load_service_record(&paths, service_id); fs::remove_dir_all(root).ok(); + assert!(interleaved.get(), "{status}: premise: the listing ran"); let value = result?; assert_eq!( value.get("stopped").and_then(Value::as_bool), diff --git a/docs/testing.md b/docs/testing.md index 476d793340..c1b852d590 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2577,7 +2577,8 @@ relaunch is still detected; `rocmd` has the same three (`a_stop_completed_by_a_c `a_stop_does_not_clear_processes_recorded_while_it_ran`) plus `a_stop_of_an_empty_record_whose_marker_a_listing_consumed_is_still_confirmed`, the listing's write made directly in those that need it because the refresh lives in the -`rocm` binary. +`rocm` binary (in the last one it is made in the engine `Stop` callback, the only +step that runs for a record naming no PID, and the test asserts it ran). `features/service_stop.feature` (`service-stop-02`, Linux, mock lane) drives the real `rocmd sandbox-tool stop_server` against a live recorded From 937e30bdfa9c4af0c3e8b1ab37e4eb92d6c9ba2d Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Tue, 10 Nov 2026 18:47:48 +0000 Subject: [PATCH 30/30] fix(services): keep an older stop from claiming the empty record a restart leaves before it publishes Between its own confirmed stop and the publication of its child, a restart leaves an empty stopped record, restores the endpoint key and spawns. That record is also what an older stop's own completion looks like, so an older stop still finalizing took it for its own, wrote stopped over it and deleted the key the restart had just restored for its child. The restart now stamps a relaunch generation (relaunch_started_unix_ms) on the record before it restores the key or spawns. A stop covers a record only in the generation it read, and publishing the child clears the stamp. The restart's own writes also refuse a record whose generation moved since it read it. Signed-off-by: Jussi Elo --- apps/rocm/src/service_control.rs | 126 ++++++++++++++++++++++++- crates/rocm-core/src/service_record.rs | 76 ++++++++++++++- docs/testing.md | 11 ++- 3 files changed, 208 insertions(+), 5 deletions(-) diff --git a/apps/rocm/src/service_control.rs b/apps/rocm/src/service_control.rs index 993c87db83..d2e8410696 100644 --- a/apps/rocm/src/service_control.rs +++ b/apps/rocm/src/service_control.rs @@ -506,6 +506,26 @@ fn restart_internal_managed_service_inner( stop_report["signaled_pids"] ); } + // Publish that a relaunch has begun before anything is restored or spawned. + // From here until `record_restarted_child` the record is an empty `stopped` + // manifest, which an older stop still finalizing would take for its own + // completion and delete the restored key under; the generation stamp is what + // keeps `is_covered_by_stop_of` from covering it. Applied to the record as it + // is on disk now, guarded on it still being what the confirmed stop left. + let (confirmed_record, _) = read_managed_service_manifest(paths, service_id)?; + if update_restarting_record( + paths, + &confirmed_record, + ManagedServiceRecord::begin_relaunch, + )? + .is_none() + { + bail!( + "`{service_id}` was stopped but NOT restarted: it changed on disk while the \ + restart was beginning; run `{}` to try again", + service_hints::service_action_hint("restart", service_id) + ); + } if let Some(key) = preserved_endpoint_key.as_deref() { endpoint_keys::store_endpoint_api_key(paths, service_id, key)?; } @@ -670,8 +690,8 @@ fn restart_internal_managed_service_inner( /// against for the supervisor's writes. /// /// Nothing is written, and `None` returned, when the record no longer reads -/// as `expected` left it — it is gone, names other processes, or has another -/// status: whoever wrote since holds a truer view than this restart. `update` +/// as `expected` left it — it is gone, names other processes, has another +/// status, or is in another relaunch generation: whoever wrote since holds a truer view than this restart. `update` /// never touches the stop marker, so a stop asked for meanwhile survives. fn update_restarting_record( paths: &AppPaths, @@ -682,7 +702,10 @@ fn update_restarting_record( return Ok(None); } let (mut current, _) = read_managed_service_manifest(paths, &expected.service_id)?; - if !current.names_same_processes(expected) || current.status != expected.status { + if !current.names_same_processes(expected) + || current.status != expected.status + || current.relaunch_started_unix_ms != expected.relaunch_started_unix_ms + { return Ok(None); } update(&mut current); @@ -2047,6 +2070,49 @@ mod tests { assert_eq!(on_disk.supervisor_pid, std::process::id()); } + /// The relaunch generation a restart stamps lives only until the restart + /// publishes: recording the child clears it, and a record whose generation + /// moved since the restart read it (another restart began, or this one's + /// stamp is gone) is not written over, as one with other processes is not. + #[test] + fn a_restart_publishing_its_child_ends_the_relaunch_generation_and_respects_a_moved_one() { + let (root, paths) = test_paths("restart-relaunch-generation"); + paths.ensure().unwrap(); + let service_id = "svc-restart-relaunch-generation"; + let mut stopped = stop_test_record(&paths, service_id); + stopped.status = "stopped".to_owned(); + stopped.clear_recorded_processes(); + stopped.begin_relaunch(); + stopped.write().unwrap(); + let stopped = raw_manifest(&paths, service_id); + assert!(stopped.relaunch_started_unix_ms.is_some(), "premise"); + + // Another restart began meanwhile: the generation on disk is not the one + // this restart read. + let mut moved = stopped.clone(); + moved.begin_relaunch(); + moved.write().unwrap(); + assert!( + moved.relaunch_started_unix_ms > stopped.relaunch_started_unix_ms, + "a new generation is always distinct from the one before it" + ); + let refused = record_restarted_child(&paths, &stopped, None, std::process::id()); + let untouched = raw_manifest(&paths, service_id); + + // With the generation as the restart read it, publication goes through. + stopped.write().unwrap(); + let started = record_restarted_child(&paths, &stopped, None, std::process::id()).unwrap(); + let on_disk = raw_manifest(&paths, service_id); + let _ = fs::remove_dir_all(root); + + assert!(refused.is_err(), "a moved generation is not written over"); + assert_eq!(untouched.status, "stopped"); + assert_eq!(untouched.supervisor_pid, 0, "no child was recorded"); + assert_eq!(started.relaunch_started_unix_ms, None); + assert_eq!(on_disk.relaunch_started_unix_ms, None); + assert_eq!(on_disk.supervisor_pid, std::process::id()); + } + #[test] fn restart_reports_a_child_it_could_not_record_and_writes_nothing() { // Between the restart's stop and its spawn, another restart recorded @@ -2301,6 +2367,60 @@ mod tests { Ok(()) } + /// An older stop finalizing while a restart is between its own confirmed stop + /// and the publication of its child. The record is then an empty `stopped` + /// manifest, which is also what an older stop's own completion looks like, but + /// the restart has restored the endpoint key and is about to serve with it. + /// The older stop (here the outer one, whose termination step runs the whole + /// restart, so its snapshot predates it) must neither claim that record nor + /// delete the key. The restart's spawn is failed on purpose, which leaves the + /// record exactly as it is at the spawn: published nothing, key restored. + #[cfg(not(windows))] + #[test] + fn an_older_stop_finalizing_before_a_restart_publishes_does_not_delete_its_key() -> Result<()> { + let (root, paths) = test_paths("older-stop-during-restart"); + paths.ensure()?; + let service_id = "svc-older-stop-during-restart"; + // A live PID (this test's own), so the restart's own load does not take + // the older stop's marker for a completed stop and clear the key before + // the restart has captured it. Nothing signals it: every termination + // step here is a stand-in. + let mut planted = stop_test_record(&paths, service_id); + planted.supervisor_pid = std::process::id(); + planted.write()?; + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123")?; + let restart_reached_spawn = std::cell::Cell::new(false); + + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + let spawn: &RestartSpawnOverride<'_> = &|_command| { + restart_reached_spawn.set(true); + Err(io::Error::other("spawn intercepted by test")) + }; + let restart = restart_internal_managed_service_inner( + &paths, + service_id, + None, + |_| bare_termination(Vec::new(), true), + Some(spawn), + ); + assert!(restart.is_err(), "the intercepted spawn fails the restart"); + bare_termination(Vec::new(), true) + })?; + + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + assert!(restart_reached_spawn.get(), "premise: the restart ran"); + assert_eq!( + key.as_deref(), + Some("key-123"), + "the key the restart restored for its child is not the older stop's to delete" + ); + assert_eq!(report["stopped"], false, "{report}"); + assert_eq!(on_disk.restart_count, 0, "nothing was published"); + Ok(()) + } + #[cfg(not(windows))] #[test] fn restart_is_not_blocked_by_a_stopped_or_failed_record_on_the_same_port() -> Result<()> { diff --git a/crates/rocm-core/src/service_record.rs b/crates/rocm-core/src/service_record.rs index 1943af3c1f..7d1ec24571 100644 --- a/crates/rocm-core/src/service_record.rs +++ b/crates/rocm-core/src/service_record.rs @@ -123,6 +123,19 @@ pub struct ManagedServiceRecord { /// ([`Self::reset_for_restart`]); absent from records that never had one. #[serde(default, skip_serializing_if = "Option::is_none")] pub settled_stop_unix_ms: Option, + /// The generation of a relaunch that has begun but not yet published its + /// processes: stamped by `rocm services restart` once its own stop has + /// confirmed and before it restores the endpoint key or spawns, and cleared + /// when the restart publishes ([`Self::reset_for_restart`]). + /// + /// Until then the record is an empty `stopped` manifest, which is also what a + /// stop's own completion looks like, so a stop that read the record earlier + /// would claim it and delete the key the restart restored for its child. A + /// stop covers the record only while the record carries the generation the + /// stop read ([`Self::is_covered_by_stop_of`]). Absent from records that are + /// not between a restart's stop and its publication. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub relaunch_started_unix_ms: Option, /// When the last inference probe was attempted. Throttles re-probing of a /// service that is listed but still loading — see /// [`crate::INFERENCE_PROBE_RETRY_INTERVAL`]. Absent on records written before @@ -269,6 +282,7 @@ impl ManagedServiceRecord { last_restart_unix_ms: None, stop_requested_unix_ms: None, settled_stop_unix_ms: None, + relaunch_started_unix_ms: None, startup_phase: None, inference_verified_at_unix_ms: None, inference_probe_attempted_at_unix_ms: None, @@ -313,8 +327,10 @@ impl ManagedServiceRecord { // died with the previous run. self.anthropic_endpoint_url = None; self.anthropic_endpoint_mode = None; - // A relaunch is no longer the stop that was settled under it. + // A relaunch is no longer the stop that was settled under it, and it is + // published now, so it is no longer waiting to be. self.settled_stop_unix_ms = None; + self.relaunch_started_unix_ms = None; self.restart_count = self.restart_count.saturating_add(1); self.last_restart_unix_ms = Some(unix_time_millis()); } @@ -368,8 +384,16 @@ impl ManagedServiceRecord { /// once it reads `stopped`, or once a liveness refresh consumed that very /// marker ([`Self::settled_stop_unix_ms`]), which leaves a `starting` or /// `failed` record at its status with the marker gone. + /// + /// Never true for a record in another relaunch generation than the stop read + /// ([`Self::relaunch_started_unix_ms`]): a restart that has stopped the + /// service and not yet published its child leaves an empty `stopped` record + /// that an older stop would otherwise take for its own completion. #[must_use] pub fn is_covered_by_stop_of(&self, stopping: &Self) -> bool { + if self.relaunch_started_unix_ms != stopping.relaunch_started_unix_ms { + return false; + } if self.status == "stopped" && self.names_no_process() { return true; } @@ -386,6 +410,19 @@ impl ManagedServiceRecord { && self.settled_stop_unix_ms == stopping.stop_requested_unix_ms)) } + /// Stamp the record as a relaunch that has begun and not yet published its + /// processes, as a generation no earlier stop or relaunch carried. See + /// [`Self::relaunch_started_unix_ms`]. + pub fn begin_relaunch(&mut self) { + let now = unix_time_millis(); + self.relaunch_started_unix_ms = Some( + now.max( + self.relaunch_started_unix_ms + .map_or(0, |earlier| earlier + 1), + ), + ); + } + /// Whether the record names neither a supervisor nor an engine process. const fn names_no_process(&self) -> bool { self.supervisor_pid == 0 @@ -1062,6 +1099,38 @@ mod tests { ); } + /// A restart between its own confirmed stop and the publication of its child + /// leaves an empty `stopped` record. A stop that read the record before that + /// (another generation) must not cover it, however the rest of the record + /// reads, while a stop that read the same generation still does. + #[test] + fn a_stop_does_not_cover_a_record_in_another_relaunch_generation() { + let mut stopping = probe_test_record(11440); + stopping.supervisor_pid = 4242; + stopping.status = "ready".to_owned(); + let mut relaunching = stopping.clone(); + relaunching.clear_recorded_processes(); + "stopped".clone_into(&mut relaunching.status); + assert!( + relaunching.is_covered_by_stop_of(&stopping), + "premise: without a relaunch the completed stop covers it" + ); + relaunching.begin_relaunch(); + assert!( + !relaunching.is_covered_by_stop_of(&stopping), + "an older stop does not cover a record whose relaunch has begun" + ); + // A stop that read the record during the relaunch, and an empty snapshot + // of the same generation, are the same stop as before. + let mut during = relaunching.clone(); + during.stop_requested_unix_ms = Some(9); + assert!(relaunching.is_covered_by_stop_of(&during)); + // The generation is never reused, even within one millisecond. + let first = relaunching.relaunch_started_unix_ms; + relaunching.begin_relaunch(); + assert!(relaunching.relaunch_started_unix_ms > first); + } + /// A restart supersedes the stop that was settled under it: the stamp a /// listing left behind does not outlive the relaunch, or a stop that began /// before it would read the relaunch's empty record as its own completion. @@ -1069,8 +1138,13 @@ mod tests { fn restarting_forgets_a_settled_stop() { let mut record = probe_test_record(11439); record.settled_stop_unix_ms = Some(7); + record.begin_relaunch(); record.reset_for_restart(); assert_eq!(record.settled_stop_unix_ms, None); + assert_eq!( + record.relaunch_started_unix_ms, None, + "publishing ends the relaunch generation" + ); } /// Writing a service manifest never writes through a file another writer diff --git a/docs/testing.md b/docs/testing.md index c1b852d590..099e834a28 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2553,7 +2553,16 @@ covers is decided by `ManagedServiceRecord::is_covered_by_stop_of`: the same processes, or none recorded because a liveness refresh that saw every process gone completed this same stop first (the record reads `stopped` and names nothing). A restart, whether it has recorded new processes or is mid-launch with -none yet (status `starting`), is not covered. The unit test +none yet (status `starting`), is not covered, and neither is the empty `stopped` +record a `rocm services restart` leaves between its own confirmed stop and the +publication of its child: the restart stamps a relaunch generation +(`relaunch_started_unix_ms`) before it restores the endpoint key or spawns, a +stop covers only the generation it read, and publishing the child clears it +(`an_older_stop_finalizing_before_a_restart_publishes_does_not_delete_its_key` +finalizes an older stop in that window and asserts the restored key survives; +`a_stop_does_not_cover_a_record_in_another_relaunch_generation` is the unit +table; `a_restart_publishing_its_child_ends_the_relaunch_generation_and_respects_a_moved_one` +covers the publication guard). The unit test `a_stop_is_covered_by_the_same_processes_or_by_its_own_completion_and_not_by_a_relaunch` is the table, and it runs again from a stopping snapshot that names no process (an empty record is that stop only while it carries the stop's marker, reads