diff --git a/README.md b/README.md index 9473fba23b..74242ff0b7 100644 --- a/README.md +++ b/README.md @@ -1132,6 +1132,14 @@ rocm services remove --yes rocm services prune [--older-than-hours | --any-age] [--dry-run] [--yes] ``` +On Linux, managed supervisor and engine PIDs are recorded with their process +start times. A live numeric PID counts as the recorded service only when a +recorded start time still matches; a mismatch means the PID was recycled, so +`services list` persists the record as stopped without touching the unrelated +process now using that PID. If the start time cannot currently be read, the +service remains conservatively live. Records created before start times were +stored retain their legacy PID-only behavior. + `stop` reports a server stopped only once every recorded process is confirmed gone. If it cannot confirm that, it prints `Local server stop not confirmed` on stderr and exits with an error. It keeps the record, which still names the diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index 222a5ef726..a62efcb4a0 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -1048,6 +1048,8 @@ pub(crate) enum ComfyuiCommand { pub(crate) enum ServicesCommand { /// Show currently running local model servers, and count the records that /// are no longer running. + /// + /// Recycled PIDs with mismatched recorded start times count as no longer running. List { /// Include failed, stopped, and old service records. #[arg(short, long)] diff --git a/apps/rocm/src/managed_service_status.rs b/apps/rocm/src/managed_service_status.rs index 586adccf94..efa02eb989 100644 --- a/apps/rocm/src/managed_service_status.rs +++ b/apps/rocm/src/managed_service_status.rs @@ -43,7 +43,7 @@ use std::time::Duration; use anyhow::{Context, Result}; use rocm_core::{ AppPaths, EndpointReadiness, EndpointReadinessOutcome, ManagedServiceRecord, - managed_service_endpoint_readiness, process_is_running, + managed_service_endpoint_readiness, }; use crate::endpoint_keys; @@ -178,16 +178,40 @@ pub(crate) fn live_managed_services_for_model( const SERVICE_LIVENESS_CHECK_TIMEOUT: Duration = Duration::from_millis(750); -/// The real process ids a record tracks. +/// The real processes a record tracks, each paired with its own start-ticks. /// /// The launcher pid is always recorded; `engine_pid` is adopted once the engine /// state file reports one. A `0` pid is a placeholder, never a real process. -pub(crate) fn recorded_service_pids(record: &ManagedServiceRecord) -> Vec { - [record.engine_pid, Some(record.supervisor_pid)] - .into_iter() - .flatten() - .filter(|pid| *pid != 0) - .collect() +/// When both name the same pid, the supervisor's known start-ticks win a +/// conflict, and either known token wins over `None`. +pub(crate) fn recorded_service_pids( + record: &ManagedServiceRecord, +) -> Vec { + let mut identities: Vec = Vec::new(); + for (pid, ticks) in [ + (Some(record.supervisor_pid), record.supervisor_start_ticks), + (record.engine_pid, record.engine_start_ticks), + ] { + let Some(pid) = pid.filter(|pid| *pid != 0) else { + continue; + }; + match identities.iter_mut().find(|identity| identity.pid == pid) { + Some(existing) if existing.start_ticks.is_none() => existing.start_ticks = ticks, + Some(_) => {} + None => identities.push(rocm_core::ProcessIdentity::new(pid, ticks)), + } + } + identities +} + +/// Whether a recorded process may still be the one the record started. +/// Unverifiable identities are conservatively alive; legacy records without +/// start-ticks retain plain PID liveness. +fn recorded_service_process_is_alive(identity: &rocm_core::ProcessIdentity) -> bool { + matches!( + rocm_core::identity_state(identity), + rocm_core::IdentityState::Matches | rocm_core::IdentityState::Indeterminate + ) } /// Complete a stop that was requested but could not confirm termination, once @@ -212,7 +236,7 @@ fn settle_pending_stop_key_cleanup(paths: &AppPaths, record: &mut ManagedService } if recorded_service_pids(record) .iter() - .any(|pid| process_is_running(*pid)) + .any(recorded_service_process_is_alive) { return false; } @@ -271,7 +295,7 @@ pub(crate) fn refresh_managed_service_runtime_liveness( let tracked_pids = recorded_service_pids(record); let has_tracked_pid = !tracked_pids.is_empty(); - let has_live_pid = tracked_pids.iter().any(|pid| process_is_running(*pid)); + let has_live_pid = tracked_pids.iter().any(recorded_service_process_is_alive); if has_tracked_pid && !has_live_pid { // Reconcile the status only. The endpoint key is *not* dropped here: // dead pids alone do not distinguish a stopped service from a crashed @@ -570,6 +594,148 @@ mod tests { Ok(()) } + /// Run the liveness refresh on a `starting` record whose supervisor and + /// engine pid are both a live `sleep` child, recorded with `ticks(real)`. + #[cfg(target_os = "linux")] + fn refresh_starting_record_for_live_child( + name: &str, + ticks: impl Fn(u64) -> Option, + ) -> (ManagedServiceRecord, bool) { + let mut child = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn child"); + let pid = child.id(); + let (root, paths) = test_paths(name); + let real = rocm_core::process_start_ticks(pid).expect("start-ticks"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-liveness", + "vllm", + "m", + "m", + "127.0.0.1", + 9, + "managed", + pid, + None, + None, + None, + ); + record.engine_pid = Some(pid); + record.supervisor_start_ticks = ticks(real); + record.engine_start_ticks = ticks(real); + + let changed = refresh_managed_service_runtime_liveness(&paths, &mut record); + + let _ = child.kill(); + let _ = child.wait(); + let _ = fs::remove_dir_all(root); + (record, changed) + } + + #[cfg(target_os = "linux")] + #[test] + fn liveness_refresh_demotes_record_whose_pid_was_recycled() { + let (record, changed) = + refresh_starting_record_for_live_child("liveness-recycled", |real| { + Some(real.wrapping_add(1)) + }); + assert_eq!(record.status, "stopped"); + assert!(changed); + } + + #[cfg(target_os = "linux")] + #[test] + fn liveness_refresh_keeps_record_whose_pid_identity_matches_not_recycled() { + let (record, changed) = + refresh_starting_record_for_live_child("liveness-not-recycled", Some); + assert_eq!(record.status, "starting"); + assert!(!changed); + } + + #[cfg(target_os = "linux")] + #[test] + fn liveness_refresh_keeps_legacy_record_without_start_ticks_not_recycled() { + let (record, changed) = + refresh_starting_record_for_live_child("liveness-legacy-not-recycled", |_| None); + assert_eq!(record.status, "starting"); + assert!(!changed); + } + + #[cfg(target_os = "linux")] + #[test] + fn liveness_refresh_demotes_recycled_supervisor_when_engine_ticks_unknown() { + let mut child = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn child"); + let pid = child.id(); + let (root, paths) = test_paths("liveness-recycled-engine-unknown"); + let real = rocm_core::process_start_ticks(pid).expect("start-ticks"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-liveness", + "vllm", + "m", + "m", + "127.0.0.1", + 9, + "managed", + pid, + None, + None, + None, + ); + record.engine_pid = Some(pid); + record.supervisor_start_ticks = Some(real.wrapping_add(1)); + + let changed = refresh_managed_service_runtime_liveness(&paths, &mut record); + + let _ = child.kill(); + let _ = child.wait(); + let _ = fs::remove_dir_all(root); + assert_eq!(record.status, "stopped"); + assert!(changed); + } + + #[cfg(target_os = "linux")] + #[test] + fn liveness_refresh_prefers_supervisor_token_when_known_tokens_conflict() { + let mut child = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn child"); + let pid = child.id(); + let (root, paths) = test_paths("liveness-conflicting-known-tokens"); + let real = rocm_core::process_start_ticks(pid).expect("start-ticks"); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-liveness", + "vllm", + "m", + "m", + "127.0.0.1", + 9, + "managed", + pid, + None, + None, + None, + ); + record.engine_pid = Some(pid); + record.supervisor_start_ticks = Some(real.wrapping_add(1)); + record.engine_start_ticks = Some(real); + + let changed = refresh_managed_service_runtime_liveness(&paths, &mut record); + + let _ = child.kill(); + let _ = child.wait(); + let _ = fs::remove_dir_all(root); + assert_eq!(record.status, "stopped"); + assert!(changed); + } + /// A public record with a dead pid and a stored endpoint key, for the /// liveness-refresh cases below. The port is one nothing listens on, so the /// refresh's endpoint probe fails and it falls through to the pid check. @@ -625,16 +791,20 @@ mod tests { let _ = fs::remove_dir_all(root); } + #[cfg(target_os = "linux")] #[test] - fn unconfirmed_stop_clears_the_endpoint_key_once_the_processes_are_gone() { - // The other half: a stop that could not confirm termination leaves the - // key in place (the engine may still be alive and enforcing it) and - // records the intent. Once the processes are observed gone, the deferred - // cleanup runs, so no plaintext secret is stranded for a service the - // operator did ask to stop. + fn unconfirmed_stop_clears_the_endpoint_key_once_the_recorded_identity_is_recycled() { + let mut child = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn child"); + let pid = child.id(); let (root, paths) = test_paths("liveness-pending-stop-clears-key"); let service_id = "svc-pending-stop"; let mut record = dead_public_service_with_key(&paths, service_id, 11983); + let real = rocm_core::process_start_ticks(pid).expect("start-ticks"); + record.supervisor_pid = pid; + record.supervisor_start_ticks = Some(real.wrapping_add(1)); record.stop_requested_unix_ms = Some(1); let changed = refresh_managed_service_runtime_liveness(&paths, &mut record); @@ -644,12 +814,18 @@ mod tests { assert_eq!( endpoint_keys::endpoint_api_key(&paths, service_id), None, - "a requested stop must still drop the key" + "a requested stop must drop the key once its identity is recycled" ); assert_eq!( record.stop_requested_unix_ms, None, "the marker is consumed, so the cleanup does not run again" ); + assert!( + rocm_core::process_is_running(pid), + "the unrelated process holding the recycled pid remains alive" + ); + let _ = child.kill(); + let _ = child.wait(); let _ = fs::remove_dir_all(root); } diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 53f9b5c41d..b1ac8e545b 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -1078,8 +1078,11 @@ fn record_still_live(record: &ManagedServiceRecord, pid_alive: impl Fn(u32) -> b if !crate::managed_service_status::managed_service_is_live(record) { return false; } - let pids = crate::managed_service_status::recorded_service_pids(record); - pids.is_empty() || pids.into_iter().any(pid_alive) + let identities = crate::managed_service_status::recorded_service_pids(record); + identities.is_empty() + || identities + .into_iter() + .any(|identity| pid_alive(identity.pid)) } /// [`record_still_live`] over the manifest on disk, so a `rocm services stop` diff --git a/apps/rocm/src/service_records.rs b/apps/rocm/src/service_records.rs index 7be31d9579..256f8fc1b9 100644 --- a/apps/rocm/src/service_records.rs +++ b/apps/rocm/src/service_records.rs @@ -169,7 +169,7 @@ fn managed_service_record_is_in_use(record: &ManagedServiceRecord) -> bool { managed_service_is_live(record) || recorded_service_pids(record) .iter() - .any(|pid| process_is_running(*pid)) + .any(|identity| process_is_running(identity.pid)) } /// Remove one non-running local server record and everything it owns. diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 2171fdd3a5..665ff68f23 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -471,6 +471,12 @@ Expected result: block at all. - `rocm services list --all` shows saved history, including the failed or stopped attempts the default view hides. +- On Linux, listing validates a recorded supervisor or engine PID against its + saved process start time. If the numeric PID now belongs to a newer process, + the record is persisted as `stopped`, disappears from the default live-only + list, and the newer process is left alone. If the start time cannot be read, + the service remains conservatively live; older records with no saved start + time keep their PID-only behavior. - `rocm services logs --follow` prints the recent output, then new output as it arrives (a download bar redraws on one terminal row), and returns by itself with `Service is ; stopped following.` once the server diff --git a/docs/testing.md b/docs/testing.md index 1cbf599274..b4ac018f3b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1812,6 +1812,24 @@ a hooks build, as `cargo xtask e2e` makes: `cargo build -p rocm -p rocmd --features rocm/e2e-test-hooks`, then the command above with `-n "service-stop"`. +`features/service_record_cleanup.feature` (`service-cleanup-08` through +`service-cleanup-10`, Linux, mock lane) covers service liveness when a recorded +PID has been recycled. The scenarios record a real live decoy as the supervisor, +as the engine, and as both through a known supervisor token plus a legacy +tokenless engine alias. +Each stores a deliberately wrong start time, runs the real `rocm services +list`, and checks that the live-only output hides the record, the manifest is +persisted as `stopped`, and the decoy remains alive. Run only these scenarios +with: + +```bash +ROCM_CLI_BINARY=target/debug/rocm cargo test -p e2e-cucumber --test e2e -- -n "service-cleanup-(08|09|10)" +``` + +These scenarios exercise liveness reconciliation, not record pruning or the +existing bare-PID prune behavior. Unit tests cover matching tokens remaining +live and tokenless legacy records retaining PID-only behavior. + Manual restricted-tool smoke: ```bash diff --git a/tests/e2e-cucumber/features/service_record_cleanup.feature b/tests/e2e-cucumber/features/service_record_cleanup.feature index 73650eea28..0263d24da9 100644 --- a/tests/e2e-cucumber/features/service_record_cleanup.feature +++ b/tests/e2e-cucumber/features/service_record_cleanup.feature @@ -1,8 +1,10 @@ Feature: Local server record cleanup # `rocm services remove` / `rocm services prune` delete the records a managed - # serve leaves behind. Every scenario plants the records rather than failing a - # real serve, so none needs a GPU and all run on the mock lane every PR. + # serve leaves behind. The final Linux scenario covers the liveness refresh + # that decides whether a record is still running; it does not exercise prune. + # Every scenario plants the records rather than failing a real serve, so none + # needs a GPU and all run on the mock lane every PR. # # A record owns FOUR files, and only two of them sit in the services folder: # the details file and the log live there, the engine state file lives under @@ -118,3 +120,38 @@ Feature: Local server record cleanup Then the prune did not return before the staged hold elapsed And every file belonging to that record is gone And the endpoint key file of the starting server is still there + + # Listing refreshes a starting record against the process identities it + # captured at launch. A live process with the same numeric PID but a different + # start time is a decoy, not evidence that the service is still running. The + # alias example covers the transition shape where the engine and supervisor + # names are the same PID but only the supervisor carries a token. + # + # Linux-only: start-time identity comes from `/proc//stat`. The process + # is an ordinary `sleep`, so no model, engine, GPU, or network is involved. + @id:service-cleanup-recycled-pid-does-not-keep-a-record-running + @requires-os:linux + Scenario: service-cleanup-08 - A recycled supervisor PID does not keep a starting record running + Given a starting local server record whose "supervisor" has the wrong start time for a live decoy + When the user lists running local servers + Then the CLI no longer lists that record as running + And that record is persisted as stopped + And the decoy process is still alive + + @id:service-cleanup-recycled-engine-pid-does-not-keep-a-record-running + @requires-os:linux + Scenario: service-cleanup-09 - A recycled engine PID does not keep a starting record running + Given a starting local server record whose "engine" has the wrong start time for a live decoy + When the user lists running local servers + Then the CLI no longer lists that record as running + And that record is persisted as stopped + And the decoy process is still alive + + @id:service-cleanup-recycled-supervisor-with-legacy-engine-alias + @requires-os:linux + Scenario: service-cleanup-10 - An unknown engine token cannot mask a recycled supervisor PID + Given a starting local server record whose "supervisor with a legacy engine alias" has the wrong start time for a live decoy + When the user lists running local servers + Then the CLI no longer lists that record as running + And that record is persisted as stopped + And the decoy process is still alive diff --git a/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs b/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs index e1bdddbfb3..c2f339e226 100644 --- a/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs @@ -2,16 +2,16 @@ // // SPDX-License-Identifier: MIT -//! Steps for `rocm services remove` / `rocm services prune`. +//! Steps for managed-service record cleanup and liveness reconciliation. //! //! Black-box: each scenario plants a managed-service record as plain JSON in its //! own isolated data dir — the same on-disk shape `rocm serve --managed` writes — -//! then runs the real binary and asserts on the files that survive. Asserting on -//! the filesystem rather than the command's summary line is deliberate: the bug -//! this feature exists to fix is a file being *left behind*, which a summary -//! claiming success cannot reveal. No GPU, no network — mock lane. +//! then runs the real binary. Cleanup scenarios assert on the files that survive; +//! the liveness scenario asserts on CLI output, the persisted record, and the +//! decoy process. No GPU or network — mock lane. use std::path::PathBuf; +use std::process::{Command, Stdio}; use cucumber::{given, then, when}; @@ -193,6 +193,58 @@ async fn record_still_running(world: &mut E2eWorld) { ); } +/// A live process standing in for a PID that the service used to own. +/// +/// The record deliberately carries a different start-time token, so listing it +/// must reconcile the service to stopped without touching the decoy. The alias +/// case is a real transition shape: both roles name one PID, the supervisor has +/// a refutable token, and the legacy engine entry has none. +#[given( + expr = "a starting local server record whose {string} has the wrong start time for a live decoy" +)] +async fn starting_record_with_recycled_pid(world: &mut E2eWorld, recorded_process: String) { + let decoy = Command::new("sleep") + .arg("300") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("failed to start the decoy process"); + let pid = decoy.id(); + world.recorded_process = Some(decoy); + let actual_ticks = rocm_core::process_start_ticks(pid) + .unwrap_or_else(|| panic!("failed to read the decoy process {pid}'s start time")); + let wrong_ticks = actual_ticks.wrapping_add(1); + + plant_record(world, "starting"); + let manifest = services_dir(world).join(format!("{SERVICE_ID}.json")); + let mut record: serde_json::Value = serde_json::from_slice( + &std::fs::read(&manifest).expect("failed to read planted service record"), + ) + .expect("planted service record is not JSON"); + match recorded_process.as_str() { + "supervisor" => { + record["supervisor_pid"] = pid.into(); + record["supervisor_start_ticks"] = wrong_ticks.into(); + } + "engine" => { + record["engine_pid"] = pid.into(); + record["engine_start_ticks"] = wrong_ticks.into(); + } + "supervisor with a legacy engine alias" => { + record["supervisor_pid"] = pid.into(); + record["supervisor_start_ticks"] = wrong_ticks.into(); + record["engine_pid"] = pid.into(); + } + other => panic!("unknown recorded process shape: {other}"), + } + std::fs::write( + &manifest, + serde_json::to_vec_pretty(&record).expect("failed to serialize service record"), + ) + .expect("failed to write service record with recycled PID"); +} + /// A record still claiming `ready` whose server is long gone, and whose files /// have not been touched since. /// @@ -338,6 +390,12 @@ async fn launch_between_its_two_writes(world: &mut E2eWorld) { // ── When ─────────────────────────────────────────────────────────── +#[when("the user lists running local servers")] +async fn list_running_servers(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["services", "list"]); + record(world, stdout, stderr, rc); +} + #[when("the user removes that local server record")] async fn remove_record(world: &mut E2eWorld) { let (stdout, stderr, rc) = crate::run_rocm(world, &["services", "remove", SERVICE_ID, "--yes"]); @@ -405,6 +463,48 @@ async fn run_prune_any_age(world: &mut E2eWorld) { // ── Then ─────────────────────────────────────────────────────────── +#[then("the CLI no longer lists that record as running")] +async fn recycled_record_not_listed_as_running(world: &mut E2eWorld) { + assert_succeeded(world); + let output = combined_output(world); + assert!( + !lists_live_server_row(&output), + "a recycled PID must not keep the record in the live list:\n{output}" + ); + assert!( + output.contains("1 not running"), + "the list must count the reconciled record as not running:\n{output}" + ); +} + +#[then("that record is persisted as stopped")] +async fn recycled_record_persisted_stopped(world: &mut E2eWorld) { + let manifest = services_dir(world).join(format!("{SERVICE_ID}.json")); + let record: serde_json::Value = serde_json::from_slice( + &std::fs::read(&manifest).expect("failed to read reconciled service record"), + ) + .expect("reconciled service record is not JSON"); + assert_eq!( + record["status"], "stopped", + "the liveness correction must be persisted: {record}" + ); +} + +#[then("the decoy process is still alive")] +async fn decoy_still_alive(world: &mut E2eWorld) { + let decoy = world + .recorded_process + .as_mut() + .expect("scenario has no decoy process"); + assert!( + decoy + .try_wait() + .expect("failed to check the decoy process") + .is_none(), + "listing a recycled PID must not terminate the process now using it" + ); +} + #[then("the CLI names the log file before deleting it")] async fn names_the_log(world: &mut E2eWorld) { assert_succeeded(world);