diff --git a/README.md b/README.md index 028f199bd6..a46d140315 100644 --- a/README.md +++ b/README.md @@ -1347,7 +1347,7 @@ rocm services list [--all] [--json] rocm services logs [--follow] rocm services stop --yes rocm services restart --yes -rocm services remove --yes +rocm services remove --yes [--force] rocm services prune [--older-than-hours | --any-age] [--dry-run] [--yes] ``` @@ -1356,10 +1356,15 @@ gone. If it cannot confirm that, it prints `Local server stop not confirmed` on stderr and exits with an error. It keeps the record, which still names the processes, and any endpoint key, which the server may still be enforcing. It names each process that, or a process under it, did not exit after it was -signaled; for a process it could not verify as the server's, it signals nothing -and names an operating-system command to check on it. It does not suggest -running the stop again: when only a process under the recorded one survived, a -second stop can report the server stopped while that process still runs. +signaled. On Windows it also names a process that did exit (or was already gone) +when taking the processes under it down did not complete, or found nothing and +no other recorded process is confirmed gone, because one of them may still be +running. For a process it could not verify as the server's (its recorded +start-time can no longer be read back, or, on Windows, `rocm` cannot query it at +all), it signals nothing and names an operating-system command to check on it. +It does not suggest running the stop again: when only a process under the +recorded one survived, a second stop can report the server stopped while that +process still runs. `restart` starts a server again only once its stop has confirmed that every recorded process is gone; if the stop cannot confirm that, it says so, starts @@ -1368,15 +1373,25 @@ nothing, and leaves the stop request recorded. After the stop confirms, `restart` takes the same launch lock `rocm serve` uses and re-checks the port before it respawns: if another live managed service is now recorded on that port, or (on the default `127.0.0.1` host only) the port is -bound by another process, it refuses and the service stays stopped (`rocm services list` shows it). The lock wait is -bounded (5 seconds), unlike `rocm serve`'s: if a launch still holds the lock, +bound by another process, it refuses and the service stays stopped +(`rocm services list` shows it). The lock wait is bounded (5 seconds), unlike +`rocm serve`'s: if a launch still holds the lock, `restart` fails saying the server was stopped but not restarted and names `rocm services restart --yes` to run once that launch finishes. `remove` deletes one record that is no longer running, together with its log, its engine state file, and its endpoint key file; a running server is refused, so stop it first. `prune` does the same in bulk, always leaves running servers -alone, and additionally clears leftover files whose record is already gone. +alone, and additionally clears leftover files whose record is already gone. A +server counts as running while any process it recorded may be running, which on +Windows includes one `rocm` cannot query (a live process the account may not +open): `remove` and `prune` refuse such a record until that process is gone. +`remove --force --yes` is the way out for exactly that case: it +removes the record when a stop was requested, no recorded process is confirmed +running, and at least one could not be queried, says which one, and never +removes a record whose process `rocm` can see running. `prune` has no +`--force`, but when it leaves such a record alone it lists it with the exact +`remove --force --yes` command. `docs/manual-testing.md` walks through it. Removal destroys both the log and the `restart` option for the records it takes, so `prune` only considers records untouched for 24 hours. Age is diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index 61d8b2668c..310474b111 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -12,13 +12,14 @@ //! the completion builder, and re-imports these types so every other module //! keeps reaching them as `crate::Command`, `crate::Provider` and so on. +pub(crate) use crate::cli_args::{ + DevicePolicyArg, FrameworkArg, InstallFormat, PermissionsModeArg, Provider, SandboxToolArg, + TelemetryModeArg, WatcherModeArg, +}; use crate::service_records::DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS; use crate::{bootstrap, chat, comfyui, remote, storage}; -use clap::{Parser, Subcommand, ValueEnum}; -use rocm_core::{ - DEFAULT_LOCAL_HOST, PERMISSIONS_MODE_ASK, PERMISSIONS_MODE_FULL_ACCESS, TELEMETRY_MODE_LOCAL, - TELEMETRY_MODE_OFF, WatcherMode, -}; +use clap::{Parser, Subcommand}; +use rocm_core::DEFAULT_LOCAL_HOST; use std::path::PathBuf; /// User-selectable serving engines, in the order shown in `--help` and shell @@ -1157,9 +1158,10 @@ pub(crate) enum ServicesCommand { /// confirmed gone. Otherwise it reports on stderr that the stop was not /// confirmed and exits with an error, keeping the record, which still names /// the processes, and any endpoint key, which the server may still be - /// enforcing. It names each process that did not exit, and for one it could - /// not verify it signals nothing and names an operating-system command to - /// check on it. It does not suggest running the stop again. + /// enforcing. It names each process that did not exit (on Windows, also one + /// whose child processes may still be running), and for one it could not + /// verify it signals nothing and names an operating-system command to check + /// on it. It does not suggest running the stop again. Stop { /// Service id from `rocm services list`. service_id: String, @@ -1181,9 +1183,10 @@ pub(crate) enum ServicesCommand { }, /// Delete one local server record and its files. /// - /// Only works on a record that is not running: stop the server first. The - /// record's details file, log, engine state file, and endpoint key file are - /// all deleted, so its log can no longer be read and it can no longer be + /// Only works on a record that is not running: stop the server first (on + /// Windows a process `rocm` cannot query counts as running). The record's + /// details file, log, engine state file, and endpoint key file are all + /// deleted, so its log can no longer be read and it can no longer be /// restarted. Remove { /// Service id from `rocm services list --all`. @@ -1191,11 +1194,21 @@ pub(crate) enum ServicesCommand { /// Required; remove refuses without it. #[arg(long)] yes: bool, + /// Remove a record whose recorded process `rocm` cannot query. + /// + /// Only when a stop was requested, no recorded process is confirmed + /// running, and at least one could not be queried (on Windows, a live + /// process the account may not open). Deletes the record, log, engine + /// state and endpoint key, and says which process; if it is still + /// running it keeps running. Refused for a process confirmed running. + #[arg(long)] + force: bool, }, /// Delete local server records that are no longer running. /// - /// Running servers are always left alone. Leftover files whose record is - /// already gone are cleaned up too. + /// Running servers are always left alone; a record held only by a process + /// `rocm` cannot query (Windows) is listed with the `remove --force` command. + /// Leftover files whose record is already gone are cleaned up too. /// /// Waits for a managed launch already under way to publish its record /// before reading the directory, and waits as long as that launch takes. @@ -1333,164 +1346,6 @@ pub(crate) enum SetupCommand { }, } -/// Which framework `rocm examine` should probe. -/// -/// The CLI-facing mirror of [`rocm_core::FrameworkProbe`], which cannot derive -/// `ValueEnum` itself without pulling clap into `rocm-core`. Every variant of -/// that enum existed and was reachable in the library, but no caller ever passed -/// anything but the default, so the choice could not be made from the command -/// line at all. -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -pub(crate) enum FrameworkArg { - /// Try PyTorch first, then llama.cpp (the default). - Auto, - /// Probe PyTorch only. - Pytorch, - /// Probe llama.cpp only. - #[value(name = "llama-cpp", alias = "llamacpp")] - LlamaCpp, - /// Probe no framework at all - fastest, and enough for GPU and driver questions. - Skip, -} - -impl From for rocm_core::FrameworkProbe { - fn from(value: FrameworkArg) -> Self { - match value { - FrameworkArg::Auto => Self::Auto, - FrameworkArg::Pytorch => Self::PyTorch, - FrameworkArg::LlamaCpp => Self::LlamaCpp, - FrameworkArg::Skip => Self::Skip, - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -pub(crate) enum InstallFormat { - /// Python wheel packages (recommended; smaller, pip-installable). - Wheel, - /// Self-contained tarball archive of the ROCm SDK. - Tarball, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -pub(crate) enum Provider { - /// A local OpenAI-compatible server started via `rocm serve` (no API key). - Local, - /// Anthropic Claude API (requires an API key). - Anthropic, - /// OpenAI API or OpenAI-compatible endpoint (requires an API key). - Openai, -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -pub(crate) enum WatcherModeArg { - /// Only report findings; never take action. - Observe, - /// Report findings and propose changes for you to approve. - Propose, - /// Apply changes automatically within a contained, reversible scope. - Contained, -} - -impl From for WatcherMode { - fn from(value: WatcherModeArg) -> Self { - match value { - WatcherModeArg::Observe => Self::Observe, - WatcherModeArg::Propose => Self::Propose, - WatcherModeArg::Contained => Self::Contained, - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -pub(crate) enum TelemetryModeArg { - /// Collect GPU telemetry on this machine only; nothing leaves the device. - Local, - /// Disable telemetry collection entirely. - Off, -} - -/// User-facing `rocm serve --device` policy. -/// -/// Modeled as a clap `ValueEnum` so invalid input is rejected by clap's usage -/// validation (exit code 2, with the valid choices listed) rather than a generic -/// application error (exit code 1) — matching how every other enum-style CLI -/// argument behaves. The historical aliases (`auto`/`gpu` for `gpu_required`, -/// `cpu` for `cpu_only`) stay accepted for backward compatibility but are hidden -/// from the advertised list. -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -#[value(rename_all = "snake_case")] -pub(crate) enum DevicePolicyArg { - /// Require a ROCm GPU; fail if none is usable (the default). - #[value(alias = "auto", alias = "gpu")] - GpuRequired, - /// Accepted for compatibility; behaves identically to `gpu_required` - /// (rocm serve has no CPU fallback path). - GpuPreferred, - /// Accepted but always rejected: rocm serve has no CPU fallback path, so - /// this exits with an error. Hidden from `--help`, shell completions and the - /// invalid-value suggestion list so it is never advertised as usable, while - /// still parsing so the deliberate rejection message is preserved. - #[value(alias = "cpu", hide = true)] - CpuOnly, -} - -impl DevicePolicyArg { - /// Canonical policy string understood by [`crate::gpu_policy::parse_device_policy`]. Routing the - /// parsed variant back through the string keeps `parse_device_policy` the - /// single choke point that enforces the `cpu_only` rejection across every - /// caller, rather than adding a second conversion path that could bypass it. - pub(crate) const fn as_policy_str(self) -> &'static str { - match self { - Self::GpuRequired => "gpu_required", - Self::GpuPreferred => "gpu_preferred", - Self::CpuOnly => "cpu_only", - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -#[value(rename_all = "snake_case")] -pub(crate) enum SandboxToolArg { - ListServers, - RestartServer, - StopServer, -} - -impl SandboxToolArg { - pub(crate) const fn as_cli_value(self) -> &'static str { - match self { - Self::ListServers => "list_servers", - Self::RestartServer => "restart_server", - Self::StopServer => "stop_server", - } - } -} - -impl TelemetryModeArg { - pub(crate) const fn as_str(self) -> &'static str { - match self { - Self::Local => TELEMETRY_MODE_LOCAL, - Self::Off => TELEMETRY_MODE_OFF, - } - } -} - -#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] -pub(crate) enum PermissionsModeArg { - FullAccess, - Ask, -} - -impl PermissionsModeArg { - pub(crate) const fn as_str(self) -> &'static str { - match self { - Self::FullAccess => PERMISSIONS_MODE_FULL_ACCESS, - Self::Ask => PERMISSIONS_MODE_ASK, - } - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/apps/rocm/src/cli_args.rs b/apps/rocm/src/cli_args.rs new file mode 100644 index 0000000000..a11fd1b520 --- /dev/null +++ b/apps/rocm/src/cli_args.rs @@ -0,0 +1,176 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The clap value enums behind the `--framework`, `--format`, `--provider`, +//! `--mode`, `--device`, `--telemetry` and `--permissions` style arguments, and +//! the conversions from them to the library types they stand for. +//! +//! Mechanically relocated from `cli.rs` with no behavior change: attributes, +//! help text, aliases and declaration order are unchanged. `cli.rs` re-exports +//! each type, so every module keeps reaching them as `crate::cli::Provider` and +//! so on. + +use clap::ValueEnum; +use rocm_core::{ + PERMISSIONS_MODE_ASK, PERMISSIONS_MODE_FULL_ACCESS, TELEMETRY_MODE_LOCAL, TELEMETRY_MODE_OFF, + WatcherMode, +}; + +/// Which framework `rocm examine` should probe. +/// +/// The CLI-facing mirror of [`rocm_core::FrameworkProbe`], which cannot derive +/// `ValueEnum` itself without pulling clap into `rocm-core`. Every variant of +/// that enum existed and was reachable in the library, but no caller ever passed +/// anything but the default, so the choice could not be made from the command +/// line at all. +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +pub(crate) enum FrameworkArg { + /// Try PyTorch first, then llama.cpp (the default). + Auto, + /// Probe PyTorch only. + Pytorch, + /// Probe llama.cpp only. + #[value(name = "llama-cpp", alias = "llamacpp")] + LlamaCpp, + /// Probe no framework at all - fastest, and enough for GPU and driver questions. + Skip, +} + +impl From for rocm_core::FrameworkProbe { + fn from(value: FrameworkArg) -> Self { + match value { + FrameworkArg::Auto => Self::Auto, + FrameworkArg::Pytorch => Self::PyTorch, + FrameworkArg::LlamaCpp => Self::LlamaCpp, + FrameworkArg::Skip => Self::Skip, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +pub(crate) enum InstallFormat { + /// Python wheel packages (recommended; smaller, pip-installable). + Wheel, + /// Self-contained tarball archive of the ROCm SDK. + Tarball, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +pub(crate) enum Provider { + /// A local OpenAI-compatible server started via `rocm serve` (no API key). + Local, + /// Anthropic Claude API (requires an API key). + Anthropic, + /// OpenAI API or OpenAI-compatible endpoint (requires an API key). + Openai, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +pub(crate) enum WatcherModeArg { + /// Only report findings; never take action. + Observe, + /// Report findings and propose changes for you to approve. + Propose, + /// Apply changes automatically within a contained, reversible scope. + Contained, +} + +impl From for WatcherMode { + fn from(value: WatcherModeArg) -> Self { + match value { + WatcherModeArg::Observe => Self::Observe, + WatcherModeArg::Propose => Self::Propose, + WatcherModeArg::Contained => Self::Contained, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +pub(crate) enum TelemetryModeArg { + /// Collect GPU telemetry on this machine only; nothing leaves the device. + Local, + /// Disable telemetry collection entirely. + Off, +} + +/// User-facing `rocm serve --device` policy. +/// +/// Modeled as a clap `ValueEnum` so invalid input is rejected by clap's usage +/// validation (exit code 2, with the valid choices listed) rather than a generic +/// application error (exit code 1) — matching how every other enum-style CLI +/// argument behaves. The historical aliases (`auto`/`gpu` for `gpu_required`, +/// `cpu` for `cpu_only`) stay accepted for backward compatibility but are hidden +/// from the advertised list. +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +#[value(rename_all = "snake_case")] +pub(crate) enum DevicePolicyArg { + /// Require a ROCm GPU; fail if none is usable (the default). + #[value(alias = "auto", alias = "gpu")] + GpuRequired, + /// Accepted for compatibility; behaves identically to `gpu_required` + /// (rocm serve has no CPU fallback path). + GpuPreferred, + /// Accepted but always rejected: rocm serve has no CPU fallback path, so + /// this exits with an error. Hidden from `--help`, shell completions and the + /// invalid-value suggestion list so it is never advertised as usable, while + /// still parsing so the deliberate rejection message is preserved. + #[value(alias = "cpu", hide = true)] + CpuOnly, +} + +impl DevicePolicyArg { + /// Canonical policy string understood by [`crate::gpu_policy::parse_device_policy`]. Routing the + /// parsed variant back through the string keeps `parse_device_policy` the + /// single choke point that enforces the `cpu_only` rejection across every + /// caller, rather than adding a second conversion path that could bypass it. + pub(crate) const fn as_policy_str(self) -> &'static str { + match self { + Self::GpuRequired => "gpu_required", + Self::GpuPreferred => "gpu_preferred", + Self::CpuOnly => "cpu_only", + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +#[value(rename_all = "snake_case")] +pub(crate) enum SandboxToolArg { + ListServers, + RestartServer, + StopServer, +} + +impl SandboxToolArg { + pub(crate) const fn as_cli_value(self) -> &'static str { + match self { + Self::ListServers => "list_servers", + Self::RestartServer => "restart_server", + Self::StopServer => "stop_server", + } + } +} + +impl TelemetryModeArg { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::Local => TELEMETRY_MODE_LOCAL, + Self::Off => TELEMETRY_MODE_OFF, + } + } +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq, ValueEnum)] +pub(crate) enum PermissionsModeArg { + FullAccess, + Ask, +} + +impl PermissionsModeArg { + pub(crate) const fn as_str(self) -> &'static str { + match self { + Self::FullAccess => PERMISSIONS_MODE_FULL_ACCESS, + Self::Ask => PERMISSIONS_MODE_ASK, + } + } +} diff --git a/apps/rocm/src/comfyui/process.rs b/apps/rocm/src/comfyui/process.rs index ec324098d5..5b17ef58c8 100644 --- a/apps/rocm/src/comfyui/process.rs +++ b/apps/rocm/src/comfyui/process.rs @@ -419,7 +419,7 @@ fn process_is_running(pid: u32) -> bool { fn windows_process_is_running(pid: u32) -> bool { let filter = format!("PID eq {pid}"); - let mut command = Command::new("tasklist"); + let mut command = Command::new(rocm_core::windows_system_tool("tasklist.exe")); command.args(["/FI", &filter, "/FO", "CSV", "/NH"]); let Ok(output) = capture_configured_command(command, "check ComfyUI process") else { return false; @@ -468,7 +468,7 @@ fn terminate_process_tree(pid: u32) -> Result<()> { } fn terminate_windows_process_tree(pid: u32) -> Result<()> { - let status = Command::new("taskkill") + let status = Command::new(rocm_core::windows_system_tool("taskkill.exe")) .args(["/PID", &pid.to_string(), "/T", "/F"]) .stdin(Stdio::null()) .stdout(Stdio::null()) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index fcb31d21e7..d713b0c105 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -17,6 +17,7 @@ mod chat; mod chat_cmd; mod chat_host_facts; mod cli; +mod cli_args; mod cli_audit; mod cli_progress; mod cli_report; diff --git a/apps/rocm/src/managed_service_status.rs b/apps/rocm/src/managed_service_status.rs index 586adccf94..15deaef215 100644 --- a/apps/rocm/src/managed_service_status.rs +++ b/apps/rocm/src/managed_service_status.rs @@ -42,8 +42,8 @@ use std::time::Duration; use anyhow::{Context, Result}; use rocm_core::{ - AppPaths, EndpointReadiness, EndpointReadinessOutcome, ManagedServiceRecord, - managed_service_endpoint_readiness, process_is_running, + AppPaths, EndpointReadiness, EndpointReadinessOutcome, ManagedServiceRecord, ProcessProbe, + managed_service_endpoint_readiness, }; use crate::endpoint_keys; @@ -190,9 +190,40 @@ pub(crate) fn recorded_service_pids(record: &ManagedServiceRecord) -> Vec { .collect() } +/// Whether any process the record names may still be running: everything but a +/// confirmed exit counts (see [`ProcessProbe::may_be_running`]), and a `0` +/// placeholder PID is never asked about. +/// +/// The one reading behind every destructive decision made from a recorded +/// process's liveness: the deferred key cleanup, the reconciliation to +/// `stopped`, and the in-use check behind `rocm services remove` and `prune`. +/// Kept in one place so those cannot drift apart on what counts as gone. +pub(crate) fn recorded_pids_may_be_running( + record: &ManagedServiceRecord, + probe: impl Fn(u32) -> ProcessProbe, +) -> bool { + recorded_service_pids(record) + .iter() + .any(|pid| probe(*pid).may_be_running()) +} + +/// Whether the record is a container-backed engine's stop that was requested and +/// has not been confirmed. +/// +/// Such a stop is decided by the engine `Stop` (`engine_stop_decides_confirmation`), +/// not by the recorded supervisor, which can be gone while the container still +/// runs. Until a `services stop` confirms it, nothing here may read the record as +/// stopped, complete its cleanup, or let `remove`/`prune` delete it, on the +/// evidence of the supervisor's PID alone. +pub(crate) fn container_stop_unresolved(record: &ManagedServiceRecord) -> bool { + record.stop_requested_unix_ms.is_some() + && rocm_core::engine_stop_decides_confirmation(&record.engine) +} + /// Complete a stop that was requested but could not confirm termination, once -/// the processes are actually observed gone: drop the endpoint key and clear the -/// marker. Returns whether the record changed. +/// the processes are actually observed gone: drop the endpoint key, clear the +/// marker, forget the recorded processes and read the record as stopped. Returns +/// whether the record changed. /// /// This is deliberately keyed on `stop_requested_unix_ms` rather than on "the /// tracked pids are dead". A *crashed* service also has dead pids, but its @@ -206,18 +237,66 @@ pub(crate) fn recorded_service_pids(record: &ManagedServiceRecord) -> Vec { /// [`refresh_managed_service_runtime_liveness`], so a record that reached /// "stopped" by another route still gets its deferred cleanup rather than /// stranding the key. -fn settle_pending_stop_key_cleanup(paths: &AppPaths, record: &mut ManagedServiceRecord) -> bool { +/// +/// A container-backed engine never completes this way: its supervisor can exit +/// while the container runs, so only a stop whose engine `Stop` confirms may end +/// it. +/// +/// "Observed gone" means every recorded process is *confirmed* gone: a process +/// that could not be queried (on Windows, a live one this process may not open) +/// is not gone, and the stop that reported it unverified must keep its key and +/// its marker until it is. +fn settle_pending_stop_key_cleanup( + paths: &AppPaths, + record: &mut ManagedServiceRecord, + probe: impl Fn(u32) -> ProcessProbe, +) -> bool { if record.stop_requested_unix_ms.is_none() { return false; } - if recorded_service_pids(record) - .iter() - .any(|pid| process_is_running(*pid)) - { + // A container-backed engine (`engine_stop_decides_confirmation`) is stopped by + // its engine `Stop`, not by the recorded supervisor: that process can be gone + // while the container still runs, so PID evidence alone cannot say the stop + // completed. The stop that could not confirm keeps its key, marker and + // recorded processes, and the next `services stop` asks the engine again. + if container_stop_unresolved(record) { + return false; + } + if recorded_pids_may_be_running(record, probe) { return false; } endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); - record.stop_requested_unix_ms = None; + let consumed_marker = record.stop_requested_unix_ms.take(); + // A record that named no process has no stop to complete: the key and the + // stale marker go, as they always did, but it is not read as stopped. It may + // be a launch that has not recorded its supervisor yet. The marker's value + // stays on the record (`settled_stop_unix_ms`), because a stop that began on + // this empty record and finds it with no marker would otherwise read this + // cleanup as a relaunch and report itself unconfirmed. + if recorded_service_pids(record).is_empty() { + record.settled_stop_unix_ms = consumed_marker; + return true; + } + // Every recorded process is confirmed gone, so none of them is the + // service's any more, and their PIDs are stale from here on. Left in the + // record, a later `services stop` would signal whatever the OS has handed + // those numbers to, and a PID recorded without a start-time token cannot be + // refuted, so on Windows the tree kill would take that process and its + // subtree. The same forgetting as a stop that confirmed the exit itself + // (`clear_recorded_processes`), and it is done only here, behind the + // confirmed-gone check above: a process that may still be running stays named. + record.clear_recorded_processes(); + // And the status a confirmed stop leaves, whatever the record read before: + // `stopped`, as `rocm services stop` and `rocmd`'s stop set on a confirmed + // stop for a `failed` record too. Set here rather than left to the + // reconciliation below, which has no PID to read once they are cleared, so a + // `ready` or `running` record would stay live; and not only for a live status, + // because a `failed` record with its processes cleared is not the `stopped` + // record `is_covered_by_stop_of` accepts as this same stop completing, so a + // stop still finalizing would report itself unconfirmed. Setting it also makes + // the caller's liveness gate return at once, so the one write the caller makes + // covers the key cleanup, the cleared processes and the status together. + "stopped".clone_into(&mut record.status); true } @@ -225,7 +304,20 @@ pub(crate) fn refresh_managed_service_runtime_liveness( paths: &AppPaths, record: &mut ManagedServiceRecord, ) -> bool { - let settled_pending_stop = settle_pending_stop_key_cleanup(paths, record); + refresh_managed_service_runtime_liveness_with(paths, record, rocm_core::probe_process) +} + +/// [`refresh_managed_service_runtime_liveness`] with the read of each recorded +/// PID supplied by the caller, so a test drives the answer a host can give for a +/// process it cannot query. Every decision here that a process is gone (the +/// deferred key cleanup, the status reconciled to `stopped`) takes a confirmed +/// exit; a process that could not be queried counts as possibly running. +pub(crate) fn refresh_managed_service_runtime_liveness_with( + paths: &AppPaths, + record: &mut ManagedServiceRecord, + probe: impl Fn(u32) -> ProcessProbe, +) -> bool { + let settled_pending_stop = settle_pending_stop_key_cleanup(paths, record, &probe); if !managed_service_is_live(record) { return settled_pending_stop; } @@ -269,9 +361,14 @@ pub(crate) fn refresh_managed_service_runtime_liveness( return settled_pending_stop || outcome.record_changed; } + // An unresolved container stop is not read as stopped from the supervisor's + // PID, whether or not the endpoint answers: only the engine `Stop` decides it. + if container_stop_unresolved(record) { + return settled_pending_stop || outcome.record_changed; + } let tracked_pids = recorded_service_pids(record); let has_tracked_pid = !tracked_pids.is_empty(); - let has_live_pid = tracked_pids.iter().any(|pid| process_is_running(*pid)); + let has_live_pid = recorded_pids_may_be_running(record, &probe); if has_tracked_pid && !has_live_pid { // Reconcile the status only. The endpoint key is *not* dropped here: // dead pids alone do not distinguish a stopped service from a crashed @@ -923,4 +1020,260 @@ mod tests { let _ = fs::remove_dir_all(root); Ok(()) } + + /// A record whose stop was requested but not confirmed, with its endpoint + /// key on disk: the state a stop leaves when a recorded process cannot be + /// verified. + fn pending_stop_record(paths: &AppPaths, service_id: &str) -> Result { + let port = std::net::TcpListener::bind(("127.0.0.1", 0))? + .local_addr()? + .port(); + let mut record = ManagedServiceRecord::new( + paths, + service_id, + "lemonade", + "model-ref", + "canonical/model", + "127.0.0.1", + port, + "managed", + 999_999_999, + None, + None, + None, + ); + record.status = "ready".to_owned(); + record.stop_requested_unix_ms = Some(1); + record.write()?; + endpoint_keys::store_endpoint_api_key(paths, service_id, "key-123")?; + Ok(record) + } + + /// A process that cannot be queried (on Windows, a live one this process may + /// not open) is not one that is gone. Whatever is reading liveness must keep + /// the key and the stop marker the unverified stop left, and must not + /// reconcile the record to `stopped`; only a confirmed exit may. + #[test] + fn a_pending_stop_keeps_its_key_and_marker_while_a_process_cannot_be_queried() -> Result<()> { + let (root, paths) = test_paths("liveness-unqueryable-pending-stop"); + paths.ensure()?; + let service_id = "svc-unqueryable"; + let mut record = pending_stop_record(&paths, service_id)?; + // The dangerous shape: a supervisor recorded without a start-time token, + // and an engine with one. + record.supervisor_pid = 4242; + record.supervisor_start_ticks = None; + record.engine_pid = Some(4243); + record.engine_start_ticks = Some(77); + + let _ = refresh_managed_service_runtime_liveness_with(&paths, &mut record, |_| { + ProcessProbe::Unqueryable + }); + assert_eq!( + ( + record.supervisor_pid, + record.engine_pid, + record.engine_start_ticks + ), + (4242, Some(4243), Some(77)), + "a process that may still be running stays named: the record is the only thing that names it" + ); + + assert!( + endpoint_keys::endpoint_key_file_if_present(&paths, service_id).is_some(), + "the key of a service that may still be running is kept" + ); + assert_eq!( + record.stop_requested_unix_ms, + Some(1), + "so is the stop marker that hands the cleanup to a later confirmed exit" + ); + assert_ne!( + record.status, "stopped", + "a process that could not be looked at does not make the service stopped" + ); + + // The same record once the process is confirmed gone: now the deferred + // cleanup and the reconciliation both happen. + let _ = refresh_managed_service_runtime_liveness_with(&paths, &mut record, |_| { + ProcessProbe::Gone + }); + assert!( + endpoint_keys::endpoint_key_file_if_present(&paths, service_id).is_none(), + "a confirmed exit drops the key" + ); + assert_eq!(record.stop_requested_unix_ms, None); + assert_eq!(record.status, "stopped"); + assert_eq!( + ( + record.supervisor_pid, + record.supervisor_start_ticks, + record.engine_pid, + record.engine_start_ticks + ), + (0, None, None, None), + "the processes confirmed gone are forgotten, so a later stop cannot reach a recycled PID" + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// The deferred cleanup forgets the PIDs, so the reconciliation after it has + /// none to read: every live status must still end as `stopped`, in the one + /// refresh, rather than a record such as `starting` staying live with no + /// process behind it, and a `failed` one must too, as a confirmed stop makes it. + /// A container-backed engine's supervisor can be gone while the container + /// still runs, and its stop is decided by the engine `Stop`, which the + /// unconfirmed stop did not get. A listing that sees the supervisor gone must + /// not complete that stop: key, marker and recorded processes stay for the + /// next `services stop`. A process engine with the same PID evidence does + /// complete (the sibling tests above). + #[test] + fn a_pending_stop_of_a_container_engine_is_not_completed_by_a_gone_supervisor() -> Result<()> { + assert!(rocm_core::engine_stop_decides_confirmation("aim")); + let (root, paths) = test_paths("liveness-container-pending-stop"); + paths.ensure()?; + let service_id = "svc-container"; + let mut record = pending_stop_record(&paths, service_id)?; + "aim".clone_into(&mut record.engine); + record.supervisor_pid = 4242; + record.supervisor_start_ticks = Some(5); + + let _ = refresh_managed_service_runtime_liveness_with(&paths, &mut record, |_| { + ProcessProbe::Gone + }); + + assert!( + endpoint_keys::endpoint_key_file_if_present(&paths, service_id).is_some(), + "the key stays: the container may still be serving with it" + ); + assert_eq!(record.stop_requested_unix_ms, Some(1)); + assert_eq!( + (record.supervisor_pid, record.supervisor_start_ticks), + (4242, Some(5)), + "the record keeps naming what the stop could not confirm" + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// A record that names no process has no stop to complete: a `starting` + /// record with a stale stop marker gets the marker and key cleaned, as before, + /// but is not read as stopped, since it may be a launch that has not recorded + /// its supervisor yet. + #[test] + fn a_stale_stop_marker_on_a_record_naming_no_process_is_cleaned_without_reading_it_stopped() + -> Result<()> { + let (root, paths) = test_paths("liveness-pending-stop-no-process"); + paths.ensure()?; + let service_id = "svc-no-process"; + let mut record = pending_stop_record(&paths, service_id)?; + "starting".clone_into(&mut record.status); + record.clear_recorded_processes(); + + let changed = refresh_managed_service_runtime_liveness_with(&paths, &mut record, |_| { + ProcessProbe::Gone + }); + + assert!(changed, "the stale marker is consumed and persisted"); + assert_eq!(record.stop_requested_unix_ms, None); + assert!(endpoint_keys::endpoint_key_file_if_present(&paths, service_id).is_none()); + assert_eq!(record.status, "starting", "it is not read as stopped"); + assert_eq!( + record.settled_stop_unix_ms, + Some(1), + "the consumed marker's stamp stays, so a stop that began on this record can tell its own marker was consumed from a relaunch having cleared it" + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn a_completed_pending_stop_reads_stopped_whatever_live_status_it_had() -> Result<()> { + for status in ["starting", "running", "ready", "recovering", "failed"] { + let (root, paths) = test_paths(&format!("liveness-pending-stop-{status}")); + paths.ensure()?; + let mut record = pending_stop_record(&paths, "svc-status")?; + record.status = status.to_owned(); + + let changed = + refresh_managed_service_runtime_liveness_with(&paths, &mut record, |_| { + ProcessProbe::Gone + }); + + assert!(changed, "{status}"); + assert_eq!(record.status, "stopped", "{status}"); + assert_eq!(record.supervisor_pid, 0, "{status}"); + assert_eq!( + record.settled_stop_unix_ms, None, + "{status}: a record that names processes reads stopped and needs no stamp" + ); + let _ = fs::remove_dir_all(root); + } + Ok(()) + } + + #[test] + fn a_pending_stop_keeps_its_key_and_marker_while_a_process_runs() -> Result<()> { + let (root, paths) = test_paths("liveness-running-pending-stop"); + paths.ensure()?; + let service_id = "svc-running"; + let mut record = pending_stop_record(&paths, service_id)?; + + let _ = refresh_managed_service_runtime_liveness_with(&paths, &mut record, |_| { + ProcessProbe::Running + }); + + assert!(endpoint_keys::endpoint_key_file_if_present(&paths, service_id).is_some()); + assert_eq!(record.stop_requested_unix_ms, Some(1)); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// The one reading behind the three destructive decisions: any recorded + /// process that is not confirmed gone counts, both roles are asked about, and + /// a placeholder `0` PID never is. + #[test] + fn recorded_pids_may_be_running_unless_every_recorded_process_is_confirmed_gone() { + use std::cell::RefCell; + + let (root, paths) = test_paths("recorded-pids-may-be-running"); + let mut record = pending_stop_record(&paths, "svc-pids").expect("record"); + let _ = fs::remove_dir_all(root); + record.supervisor_pid = 111; + record.engine_pid = Some(222); + + let asked_cell = RefCell::new(Vec::new()); + let asked = &asked_cell; + let by_pid = |running: &'static [(u32, ProcessProbe)]| { + move |pid: u32| { + asked.borrow_mut().push(pid); + running + .iter() + .find(|(candidate, _)| *candidate == pid) + .map_or(ProcessProbe::Gone, |(_, probe)| *probe) + } + }; + + assert!(recorded_pids_may_be_running( + &record, + by_pid(&[(222, ProcessProbe::Unqueryable)]) + )); + assert!(recorded_pids_may_be_running( + &record, + by_pid(&[(111, ProcessProbe::Running)]) + )); + assert!(!recorded_pids_may_be_running(&record, by_pid(&[]))); + let engine_only = asked.borrow().iter().filter(|pid| **pid == 222).count(); + assert!(engine_only >= 1, "the engine PID is asked about too"); + + record.supervisor_pid = 0; + record.engine_pid = None; + asked.borrow_mut().clear(); + assert!(!recorded_pids_may_be_running( + &record, + by_pid(&[(0, ProcessProbe::Running)]) + )); + assert!(asked.borrow().is_empty(), "a 0 placeholder is never probed"); + } } diff --git a/apps/rocm/src/service_control.rs b/apps/rocm/src/service_control.rs index d2b9128d2b..d2e8410696 100644 --- a/apps/rocm/src/service_control.rs +++ b/apps/rocm/src/service_control.rs @@ -8,9 +8,9 @@ //! Mechanically relocated from `services_text.rs` with no behavior change. //! //! This module owns: the sandbox-tool runner (`run_internal_sandbox_tool`), the -//! stop path (`terminate_recorded_service_pids`, the `MANAGED_STOP_GRACE` -//! constant, the private `RecordedPidOutcome`/`RecordedStopTermination` types -//! and the `scripted_stop_outcome` end-to-end hook, `stop_internal_managed_service`, +//! stop path (`stop_recorded_processes`, which hands the recorded PIDs to the +//! routine `rocm-core` shares with `rocmd`, the `scripted_stop_outcome` +//! end-to-end hook, `stop_internal_managed_service`, //! `stop_internal_managed_service_with`, //! `stop_internal_managed_service_confirmed_with`, //! `unload_lemonade_service_model`), and the restart path @@ -124,119 +124,14 @@ pub(crate) fn run_internal_sandbox_tool( })) } -/// How long a managed stop waits for each recorded process to actually exit -/// after `SIGTERM` before escalating to `SIGKILL` and then reporting a timeout. -const MANAGED_STOP_GRACE: Duration = Duration::from_secs(10); - -/// One recorded PID's fate in a managed stop: the role the record gave it, and -/// what [`rocm_core::terminate_verified`] did about it. Kept per PID because the -/// outcomes that leave a stop unconfirmed are reported differently: a -/// `TimedOut` PID was signalled and it or a process under it outlived the -/// deadline, while an `Unverified` one is not signalled by this stop, and the -/// report names a command to check on it. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct RecordedPidOutcome { - pid: u32, - /// `supervisor`, `engine`, or `supervisor_and_engine` when the record names - /// one PID for both — the same labels `rocmd`'s `pid_outcomes` uses. - role: &'static str, - outcome: rocm_core::TerminationOutcome, -} - -/// What a managed stop did to the processes its record names. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -pub(crate) struct RecordedStopTermination { - /// Every PID that received a signal, including one that then outlived the - /// deadline. - signaled_pids: Vec, - /// Whether every recorded process is confirmed no longer running. - all_stopped: bool, - pid_outcomes: Vec, -} - -impl RecordedStopTermination { - /// A stop stand-in that confirms every recorded process gone without - /// signalling any, for tests in other modules that drive the real stop path. - #[cfg(test)] - pub(crate) fn confirmed_stopped() -> Self { - Self { - all_stopped: true, - ..Self::default() - } - } -} - -/// Terminate the processes recorded for a managed service, verifying each PID's -/// identity first so a recycled PID never causes an unrelated process tree to be -/// killed. -/// -/// The supervisor (launcher) and the engine server are distinct processes, so -/// each PID is paired with **its own** start-time token (`supervisor_start_ticks` -/// vs `engine_start_ticks`). When they resolve to the same PID (e.g. before the -/// engine state has been refreshed) the entry is de-duplicated, preferring a -/// known token. Returns the PIDs actually signalled, whether every recorded -/// process is confirmed no longer running, and each PID's own outcome. -fn terminate_recorded_service_pids(record: &ManagedServiceRecord) -> RecordedStopTermination { - // Build the (pid, own-token, role) work list, de-duplicating on PID and - // preferring an entry that carries a verifiable start-time. - let mut entries: Vec<(u32, Option, &'static str)> = Vec::new(); - for (pid, ticks, role) in [ - ( - Some(record.supervisor_pid), - record.supervisor_start_ticks, - "supervisor", - ), - (record.engine_pid, record.engine_start_ticks, "engine"), - ] { - let Some(pid) = pid.filter(|pid| *pid != 0 && *pid != std::process::id()) else { - continue; - }; - if let Some(existing) = entries.iter_mut().find(|(seen, _, _)| *seen == pid) { - if existing.1.is_none() { - existing.1 = ticks; - } - existing.2 = "supervisor_and_engine"; - } else { - entries.push((pid, ticks, role)); - } - } - - let mut termination = RecordedStopTermination { - all_stopped: true, - ..RecordedStopTermination::default() - }; - for (pid, ticks, role) in entries { - let identity = rocm_core::ProcessIdentity::new(pid, ticks); - // The managed stop contract is definitive: after the adapter gets its - // graceful opportunity, this cleanup pass forces any verified survivor - // so the command does not report success while a GPU worker remains. - let outcome = scripted_stop_outcome().unwrap_or_else(|| { - rocm_core::terminate_verified( - &identity, - rocm_core::KillScope::Tree, - MANAGED_STOP_GRACE, - true, - ) - }); - if !outcome.stopped() { - termination.all_stopped = false; - } - // Report every PID that received a signal. TimedOut belongs here because - // signalling was attempted even though the process was not confirmed - // stopped; `all_stopped` separately carries the truthful completion state. - if matches!( - outcome, - rocm_core::TerminationOutcome::Graceful - | rocm_core::TerminationOutcome::Forced - | rocm_core::TerminationOutcome::TimedOut - ) { - termination.signaled_pids.push(pid); - } - termination - .pid_outcomes - .push(RecordedPidOutcome { pid, role, outcome }); - } - termination +/// Terminate the processes recorded for a managed service through the routine +/// `rocmd` shares, [`rocm_core::terminate_recorded_service_pids`], with +/// [`rocm_core::terminate_recorded_pid`] as the termination of each PID. The +/// one thing this adds is the e2e seam below. +fn stop_recorded_processes(record: &ManagedServiceRecord) -> Vec { + rocm_core::terminate_recorded_service_pids(record, |identity| { + scripted_stop_outcome().unwrap_or_else(|| rocm_core::terminate_recorded_pid(identity)) + }) } /// E2E hook: when `ROCM_E2E_STOP_OUTCOME` is `timed_out` or `unverified`, a @@ -245,7 +140,8 @@ fn terminate_recorded_service_pids(record: &ManagedServiceRecord) -> RecordedSto /// /// Neither outcome can be staged from outside the binary: `TimedOut` needs a /// process that survives `SIGKILL`, and `Unverified` one whose recorded -/// start-time cannot be read back. Without this seam the "stop not confirmed" +/// start-time cannot be read back, or (on Windows) that cannot be queried at +/// all. Without this seam the "stop not confirmed" /// report, its exit status and what it keeps on disk have no live coverage. /// The recorded process is left untouched. Compiled only into the /// `e2e-test-hooks` binary; production builds never see it. @@ -264,12 +160,12 @@ const fn scripted_stop_outcome() -> Option { } fn stop_internal_managed_service(paths: &AppPaths, service_id: &str) -> Result { - stop_internal_managed_service_with(paths, service_id, terminate_recorded_service_pids) + stop_internal_managed_service_with(paths, service_id, stop_recorded_processes) } /// [`stop_internal_managed_service`], with the termination of the recorded /// processes supplied by the caller. Production passes -/// [`terminate_recorded_service_pids`]; a test passes a stand-in that observes +/// [`stop_recorded_processes`]; a test passes a stand-in that observes /// what is on disk at the moment the processes would be signalled. Everything /// else — the engine stop, the manifest writes, the key cleanup — runs for real. /// @@ -278,7 +174,7 @@ fn stop_internal_managed_service(paths: &AppPaths, service_id: &str) -> Result RecordedStopTermination, + terminate: impl FnOnce(&ManagedServiceRecord) -> Vec, ) -> Result { stop_internal_managed_service_confirmed_with(paths, service_id, terminate) .map(|(report, _)| report) @@ -292,7 +188,7 @@ pub(crate) fn stop_internal_managed_service_with( fn stop_internal_managed_service_confirmed_with( paths: &AppPaths, service_id: &str, - terminate: impl FnOnce(&ManagedServiceRecord) -> RecordedStopTermination, + terminate: impl FnOnce(&ManagedServiceRecord) -> Vec, ) -> Result<(serde_json::Value, bool)> { let mut stopping = load_managed_service(paths, service_id)?; // Record the request on disk before anything is stopped, not only after. @@ -320,11 +216,17 @@ fn stop_internal_managed_service_confirmed_with( }, ) }; - let RecordedStopTermination { - signaled_pids, - all_stopped: terminated_all, - pid_outcomes, - } = terminate(&stopping); + let stops = terminate(&stopping); + // Every PID that received a signal, including one that then outlived the + // deadline: signalling was attempted even though the process was not + // confirmed stopped, and `terminated_all` separately carries the truthful + // completion state. + let signaled_pids: Vec = stops + .iter() + .filter(|stop| stop.signaled()) + .map(|stop| stop.pid) + .collect(); + let terminated_all = stops.iter().all(|stop| stop.stopped()); // Re-read the record rather than write back the copy loaded before the // stop, which can be some forty seconds old by now: `write` replaces the // whole file, and a supervisor, the daemon's restart or another stop may @@ -348,11 +250,20 @@ fn stop_internal_managed_service_confirmed_with( // supervisor that follows it, so a failed `Stop` leaves the stop unconfirmed. let engine_stop_confirms = !rocm_core::engine_stop_decides_confirmation(&stopping.engine) || engine_stop.is_ok(); - let record_changed = !stopping.names_same_processes(&record); + let record_changed = !record.is_covered_by_stop_of(&stopping); let all_stopped = terminated_all && !record_changed && engine_stop_confirms; if all_stopped { record.status = "stopped".to_owned(); record.stop_requested_unix_ms = None; + // The processes this stop confirmed gone are not the service's any more, + // and their PIDs are stale the instant they exited. Left in the record, + // a later stop would signal them again: a PID recorded without a + // start-time token (every one a Windows record had before it recorded + // a creation time) cannot be refuted, so if the + // OS had given it to another process the Windows tree kill would take + // that process and its subtree. The same call, on the same confirmed + // path, as `rocmd`'s stop. + record.clear_recorded_processes(); } else { // The marker records that a stop was *asked for* even though it could // not be confirmed. `refresh_managed_service_runtime_liveness` needs it @@ -399,15 +310,9 @@ fn stop_internal_managed_service_confirmed_with( "status": record.status, "engine_stop": engine_stop, "signaled_pids": signaled_pids, - "pid_outcomes": pid_outcomes + "pid_outcomes": stops .iter() - .map(|entry| { - serde_json::json!({ - "pid": entry.pid, - "role": entry.role, - "outcome": entry.outcome.as_str(), - }) - }) + .map(|stop| stop.report_entry()) .collect::>(), "record_changed": record_changed, // A container engine is confirmed stopped by its own `Stop`; when that @@ -515,12 +420,7 @@ pub(crate) fn restart_internal_managed_service( service_id: &str, runtime_pin: Option<&str>, ) -> Result { - restart_internal_managed_service_with( - paths, - service_id, - runtime_pin, - terminate_recorded_service_pids, - ) + restart_internal_managed_service_with(paths, service_id, runtime_pin, stop_recorded_processes) } /// [`restart_internal_managed_service`], with the termination of the recorded @@ -536,7 +436,7 @@ fn restart_internal_managed_service_with( paths: &AppPaths, service_id: &str, runtime_pin: Option<&str>, - terminate: impl FnOnce(&ManagedServiceRecord) -> RecordedStopTermination, + terminate: impl FnOnce(&ManagedServiceRecord) -> Vec, ) -> Result { restart_internal_managed_service_inner(paths, service_id, runtime_pin, terminate, None) } @@ -571,7 +471,7 @@ fn restart_internal_managed_service_inner( paths: &AppPaths, service_id: &str, runtime_pin: Option<&str>, - terminate: impl FnOnce(&ManagedServiceRecord) -> RecordedStopTermination, + terminate: impl FnOnce(&ManagedServiceRecord) -> Vec, spawn_override: Option<&RestartSpawnOverride<'_>>, ) -> Result { #[cfg(windows)] @@ -606,6 +506,26 @@ fn restart_internal_managed_service_inner( stop_report["signaled_pids"] ); } + // Publish that a relaunch has begun before anything is restored or spawned. + // From here until `record_restarted_child` the record is an empty `stopped` + // manifest, which an older stop still finalizing would take for its own + // completion and delete the restored key under; the generation stamp is what + // keeps `is_covered_by_stop_of` from covering it. Applied to the record as it + // is on disk now, guarded on it still being what the confirmed stop left. + let (confirmed_record, _) = read_managed_service_manifest(paths, service_id)?; + if update_restarting_record( + paths, + &confirmed_record, + ManagedServiceRecord::begin_relaunch, + )? + .is_none() + { + bail!( + "`{service_id}` was stopped but NOT restarted: it changed on disk while the \ + restart was beginning; run `{}` to try again", + service_hints::service_action_hint("restart", service_id) + ); + } if let Some(key) = preserved_endpoint_key.as_deref() { endpoint_keys::store_endpoint_api_key(paths, service_id, key)?; } @@ -770,8 +690,8 @@ fn restart_internal_managed_service_inner( /// against for the supervisor's writes. /// /// Nothing is written, and `None` returned, when the record no longer reads -/// as `expected` left it — it is gone, names other processes, or has another -/// status: whoever wrote since holds a truer view than this restart. `update` +/// as `expected` left it — it is gone, names other processes, has another +/// status, or is in another relaunch generation: whoever wrote since holds a truer view than this restart. `update` /// never touches the stop marker, so a stop asked for meanwhile survives. fn update_restarting_record( paths: &AppPaths, @@ -782,7 +702,10 @@ fn update_restarting_record( return Ok(None); } let (mut current, _) = read_managed_service_manifest(paths, &expected.service_id)?; - if !current.names_same_processes(expected) || current.status != expected.status { + if !current.names_same_processes(expected) + || current.status != expected.status + || current.relaunch_started_unix_ms != expected.relaunch_started_unix_ms + { return Ok(None); } update(&mut current); @@ -1073,7 +996,7 @@ mod tests { // Proves the *ordering*, not just the refusal: had the guard run after // `stop_internal_managed_service`, the stop would have written // "stopped". It reaches that state here because the record's only pid is - // the test's own (`engine_pid` is None and `terminate_recorded_service_pids` + // the test's own (`engine_pid` is None and the shared stop routine // skips the caller's pid), so the stop confirms termination trivially. let after = load_managed_service(&paths, service_id).unwrap(); assert_ne!( @@ -1216,6 +1139,275 @@ mod tests { assert_eq!(key.as_deref(), Some("key-123"), "the key is kept"); } + /// A liveness refresh that lands after the stop's processes have exited but + /// before the stop reads the record again completes the same stop (key, + /// marker, recorded processes cleared, `stopped`) and writes it. That record + /// names no process, which must not read as a restart: the stop is confirmed, + /// the marker is not put back, and a restart is not refused for it. A record + /// that read `failed` is completed to `stopped` too, as a confirmed stop makes + /// it, so it is covered as well and not left reading `failed` with no process. + #[test] + fn a_listing_that_completes_the_stop_before_finalization_does_not_unconfirm_it() { + use rocm_core::TerminationOutcome::Graceful; + for status in ["ready", "failed"] { + let (root, paths) = test_paths(&format!("stop-listing-before-finalization-{status}")); + paths.ensure().unwrap(); + let service_id = "svc-stop-listing-before-finalization"; + let mut planted = stop_test_record(&paths, service_id); + planted.status = status.to_owned(); + planted.supervisor_pid = 4242; + planted.supervisor_start_ticks = None; + planted.engine_pid = Some(4243); + planted.engine_start_ticks = Some(77); + planted.write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |stopping| { + assert!(stopping.stop_requested_unix_ms.is_some()); + // The processes have exited; a concurrent `services` listing sees + // it, completes the pending stop and persists that, before this + // stop reads the record again. + let (mut listed, _) = read_managed_service_manifest(&paths, service_id).unwrap(); + // The fake PIDs read as gone to the stop's own initial refresh, + // which may already have demoted the status: put back the status + // the listing under test starts from. + status.clone_into(&mut listed.status); + let changed = + crate::managed_service_status::refresh_managed_service_runtime_liveness_with( + &paths, + &mut listed, + |_| rocm_core::ProcessProbe::Gone, + ); + assert!( + changed, + "premise: the listing completed the stop ({status})" + ); + listed.write().unwrap(); + assert_eq!(raw_manifest(&paths, service_id).supervisor_pid, 0); + vec![stop_of(4242, Graceful), stop_of(4243, Graceful)] + }) + .unwrap(); + + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + assert_eq!(report["stopped"], true, "{status}: {report}"); + assert_eq!(on_disk.status, "stopped", "{status}"); + assert_eq!( + on_disk.stop_requested_unix_ms, None, + "{status}: a confirmed stop does not put the marker back" + ); + assert_eq!(on_disk.supervisor_pid, 0, "{status}"); + assert_eq!(key, None, "{status}: the key goes with the confirmed stop"); + } + } + + /// The same race through `restart`: the restart's own stop is completed by a + /// listing before it finalizes, and the restart must carry on past the stop + /// (here to the launch lock, which a holder keeps, so it stops there) + /// instead of refusing with "could not confirm". + #[test] + fn a_restart_is_not_refused_because_a_listing_completed_its_stop() -> Result<()> { + use rocm_core::TerminationOutcome::Graceful; + let (root, paths) = test_paths("restart-listing-before-finalization"); + paths.ensure()?; + let service_id = "svc-restart-listing-before-finalization"; + stop_test_record(&paths, service_id).write()?; + let holder = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + + LAUNCH_LOCK_TIMEOUT_OVERRIDE.with(|cell| cell.set(Some(Duration::from_millis(100)))); + let result = restart_internal_managed_service_with(&paths, service_id, None, |_| { + let (mut listed, _) = read_managed_service_manifest(&paths, service_id).unwrap(); + let _ = crate::managed_service_status::refresh_managed_service_runtime_liveness_with( + &paths, + &mut listed, + |_| rocm_core::ProcessProbe::Gone, + ); + listed.write().unwrap(); + vec![stop_of(999_999_999, Graceful)] + }); + LAUNCH_LOCK_TIMEOUT_OVERRIDE.with(|cell| cell.set(None)); + drop(holder); + let _ = fs::remove_dir_all(root); + + let message = format!( + "{:#}", + result.expect_err("the held launch lock fails the restart") + ); + assert!( + !message.contains("could not confirm"), + "the stop was completed, the restart must not be refused for it: {message}" + ); + assert!(message.contains("stopped but NOT restarted"), "{message}"); + Ok(()) + } + + /// The other half of the same check: a restart that has begun launching but + /// has recorded no process yet (status `starting`, nothing named) is not the + /// stop completing, and the stop must not claim it or drop the key it will + /// serve with. + #[test] + fn stop_does_not_confirm_over_a_restart_that_has_recorded_no_process_yet() { + let (root, paths) = test_paths("stop-restart-in-flight"); + paths.ensure().unwrap(); + let service_id = "svc-stop-restart-in-flight"; + stop_test_record(&paths, service_id).write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + let mut restarting = raw_manifest(&paths, service_id); + restarting.status = "starting".to_owned(); + restarting.clear_recorded_processes(); + restarting.stop_requested_unix_ms = None; + restarting.write().unwrap(); + bare_termination(Vec::new(), true) + }) + .unwrap(); + + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + assert_eq!(report["status"], "starting", "{report}"); + assert_eq!(on_disk.status, "starting", "the restart's status stays"); + assert_eq!(key.as_deref(), Some("key-123"), "the key is kept"); + } + + /// The same as above when the record being stopped names no process either: + /// a relaunch's empty `starting` record then equals the stopping snapshot's + /// processes, which must not make it the stop's own. Its marker is cleared and + /// it reads `starting`, so the stop does not claim it, overwrite its status or + /// delete the key it will serve with. A stop of an empty record that nobody + /// relaunched is still confirmed. + #[test] + fn stop_does_not_confirm_over_an_empty_relaunch_of_an_empty_record() { + let (root, paths) = test_paths("stop-empty-relaunch"); + paths.ensure().unwrap(); + let service_id = "svc-stop-empty-relaunch"; + let mut empty = stop_test_record(&paths, service_id); + empty.clear_recorded_processes(); + empty.write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + let mut relaunch = raw_manifest(&paths, service_id); + relaunch.status = "starting".to_owned(); + relaunch.stop_requested_unix_ms = None; + relaunch.write().unwrap(); + bare_termination(Vec::new(), true) + }) + .unwrap(); + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + + // And with nobody relaunching, the stop of the same empty record is a stop. + let untouched = stop_internal_managed_service_with(&paths, service_id, |_| { + bare_termination(Vec::new(), true) + }) + .unwrap(); + let _ = fs::remove_dir_all(root); + + assert_eq!(report["stopped"], false, "{report}"); + assert_eq!(on_disk.status, "starting", "the relaunch's status stays"); + assert_eq!(key.as_deref(), Some("key-123"), "its key is kept"); + assert_eq!(untouched["stopped"], true, "{untouched}"); + } + + /// The interleaving the empty-relaunch guard above must not mistake for a + /// relaunch: the record being stopped names no process and reads `starting` or + /// `failed` (a liveness refresh leaves both as they are), and a concurrent + /// listing finds the stop marker with nothing recorded to wait for and clears + /// it before the stop reads the record again. Nobody relaunched, so the stop is + /// confirmed as it is when no listing intervenes (the last block), and the + /// record reads `stopped` afterwards with its key gone. + #[test] + fn a_listing_clearing_the_marker_of_an_empty_stop_does_not_make_it_unconfirmed() { + for status in ["starting", "failed"] { + let (root, paths) = test_paths(&format!("stop-empty-listing-{status}")); + paths.ensure().unwrap(); + let service_id = "svc-stop-empty-listing"; + let mut empty = stop_test_record(&paths, service_id); + status.clone_into(&mut empty.status); + empty.clear_recorded_processes(); + empty.write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |stopping| { + assert!(stopping.stop_requested_unix_ms.is_some()); + // The listing under test: it reads the record with the stop's + // marker on it, finds no process to wait for, and persists the + // cleanup before this stop reads the record again. + let (mut listed, _) = read_managed_service_manifest(&paths, service_id).unwrap(); + let changed = + crate::managed_service_status::refresh_managed_service_runtime_liveness_with( + &paths, + &mut listed, + |_| rocm_core::ProcessProbe::Gone, + ); + assert!( + changed, + "premise: the listing cleaned the marker ({status})" + ); + listed.write().unwrap(); + let after_listing = raw_manifest(&paths, service_id); + assert_eq!(after_listing.stop_requested_unix_ms, None, "{status}"); + assert_eq!(after_listing.status, status, "premise: status unchanged"); + bare_termination(Vec::new(), true) + }) + .unwrap(); + + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + assert_eq!(report["stopped"], true, "{status}: {report}"); + assert_eq!(on_disk.status, "stopped", "{status}"); + assert_eq!(on_disk.stop_requested_unix_ms, None, "{status}"); + assert_eq!(key, None, "{status}: the key goes with the confirmed stop"); + } + } + + /// The same interleaving followed by a real relaunch: the listing consumed the + /// marker, then the daemon restarted the empty record (a restart counts itself, + /// clears the marker and reads `recovering` or `starting`). The stamp the + /// listing left is the stop's own, but the relaunch wipes it, so the stop still + /// does not claim the relaunch's record. + #[test] + fn a_relaunch_after_a_listing_consumed_the_marker_is_still_not_the_stop() { + let (root, paths) = test_paths("stop-empty-listing-then-relaunch"); + paths.ensure().unwrap(); + let service_id = "svc-stop-empty-listing-then-relaunch"; + let mut empty = stop_test_record(&paths, service_id); + "starting".clone_into(&mut empty.status); + empty.clear_recorded_processes(); + empty.write().unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + let (mut listed, _) = read_managed_service_manifest(&paths, service_id).unwrap(); + let _ = crate::managed_service_status::refresh_managed_service_runtime_liveness_with( + &paths, + &mut listed, + |_| rocm_core::ProcessProbe::Gone, + ); + listed.write().unwrap(); + let mut relaunch = raw_manifest(&paths, service_id); + assert!( + relaunch.settled_stop_unix_ms.is_some(), + "premise: the listing left the stop's stamp" + ); + "recovering".clone_into(&mut relaunch.status); + relaunch.reset_for_restart(); + relaunch.stop_requested_unix_ms = None; + relaunch.write().unwrap(); + bare_termination(Vec::new(), true) + }) + .unwrap(); + + let on_disk = raw_manifest(&paths, service_id); + let _ = fs::remove_dir_all(root); + assert_eq!(report["stopped"], false, "{report}"); + assert_eq!(on_disk.status, "recovering", "the relaunch's status stays"); + assert_eq!(on_disk.restart_count, 1); + } + #[test] fn stop_of_a_container_engine_is_unconfirmed_when_the_engine_stop_fails() { // For `aim` the recorded PID is only the supervisor; killing it says @@ -1370,27 +1562,41 @@ mod tests { assert_eq!(key.as_deref(), Some("key-123"), "the endpoint key is kept"); } - /// A stop stand-in's bare verdict, without per-PID detail. - fn bare_termination(signaled_pids: Vec, all_stopped: bool) -> RecordedStopTermination { - RecordedStopTermination { - signaled_pids, - all_stopped, - pid_outcomes: Vec::new(), + /// One recorded process's stop, for a stand-in to return. + fn stop_of(pid: u32, outcome: rocm_core::TerminationOutcome) -> rocm_core::RecordedPidStop { + rocm_core::RecordedPidStop { + pid, + role: "supervisor_and_engine", + outcome: rocm_core::RecordedPidOutcome::Terminated(outcome), } } + /// A stop stand-in's bare verdict, without per-PID detail: `signaled_pids` + /// were signalled and `all_stopped` says whether they exited. A stop that + /// signalled nothing and is not confirmed has one unverifiable process. + fn bare_termination( + signaled_pids: Vec, + all_stopped: bool, + ) -> Vec { + use rocm_core::TerminationOutcome::{Graceful, TimedOut, Unverified}; + if signaled_pids.is_empty() { + return if all_stopped { + Vec::new() + } else { + vec![stop_of(999_999_999, Unverified)] + }; + } + let outcome = if all_stopped { Graceful } else { TimedOut }; + signaled_pids + .into_iter() + .map(|pid| stop_of(pid, outcome)) + .collect() + } + /// A stop stand-in whose one recorded process (the record's shared /// supervisor/engine PID) was signalled and outlived the deadline. - fn timed_out_termination(pid: u32) -> RecordedStopTermination { - RecordedStopTermination { - signaled_pids: vec![pid], - all_stopped: false, - pid_outcomes: vec![RecordedPidOutcome { - pid, - role: "supervisor_and_engine", - outcome: rocm_core::TerminationOutcome::TimedOut, - }], - } + fn timed_out_termination(pid: u32) -> Vec { + vec![stop_of(pid, rocm_core::TerminationOutcome::TimedOut)] } /// The most recent audit event written under `paths`. @@ -1410,17 +1616,13 @@ mod tests { stop_test_record(&paths, service_id).write().unwrap(); endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); - let report = - stop_internal_managed_service_with(&paths, service_id, |_| RecordedStopTermination { - signaled_pids: Vec::new(), - all_stopped: false, - pid_outcomes: vec![RecordedPidOutcome { - pid: 999_999_999, - role: "supervisor_and_engine", - outcome: rocm_core::TerminationOutcome::Unverified, - }], - }) - .unwrap(); + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + vec![stop_of( + 999_999_999, + rocm_core::TerminationOutcome::Unverified, + )] + }) + .unwrap(); let answer = internal_stop_server_value(report); let outcome = finish_service_action(&paths, "stop_server", service_id, &answer); @@ -1483,31 +1685,26 @@ mod tests { record.engine_pid = Some(888_888_888); record.write().unwrap(); - let report = - stop_internal_managed_service_with(&paths, service_id, |_| RecordedStopTermination { - signaled_pids: vec![999_999_999], - all_stopped: false, - pid_outcomes: vec![ - RecordedPidOutcome { - pid: 999_999_999, - role: "supervisor", - outcome: rocm_core::TerminationOutcome::TimedOut, - }, - RecordedPidOutcome { - pid: 888_888_888, - role: "engine", - outcome: rocm_core::TerminationOutcome::Unverified, - }, - ], - }) - .unwrap(); + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + vec![ + rocm_core::RecordedPidStop { + role: "supervisor", + ..stop_of(999_999_999, rocm_core::TerminationOutcome::TimedOut) + }, + rocm_core::RecordedPidStop { + role: "engine", + ..stop_of(888_888_888, rocm_core::TerminationOutcome::Unverified) + }, + ] + }) + .unwrap(); let answer = internal_stop_server_value(report); let outcome = finish_service_action(&paths, "stop_server", service_id, &answer); let _ = fs::remove_dir_all(root); let rendered = format!("{:#}", outcome.expect_err("unconfirmed")); assert_no_retry_line(&rendered); assert!( - rendered.contains("check: `ps -p 888888888 -o pid,args`"), + rendered.contains(&format!("check: `{}`", process_check_command(888_888_888))), "{rendered}" ); assert!(!rendered.contains("the record changed"), "{rendered}"); @@ -1608,6 +1805,128 @@ mod tests { assert_eq!(key, None, "a confirmed stop drops the key"); } + /// A confirmed stop must not leave the processes it stopped named in the + /// record. Their PIDs are stale the moment they exit, and a recorded PID + /// with no start-time token is signalled on trust alone once the OS has + /// handed it to another process; on Windows the tree kill then takes that + /// process and everything under it. `rocmd`'s stop clears them for the same + /// reason (`stop_managed_service_clears_recorded_pids_once_the_processes_are_gone`). + #[test] + fn a_confirmed_stop_clears_the_recorded_processes_so_a_second_stop_signals_nothing() { + use rocm_core::TerminationOutcome::Graceful; + let (root, paths) = test_paths("services-stop-clears-pids"); + paths.ensure().unwrap(); + let service_id = "svc-services-stop-clears-pids"; + let mut record = stop_test_record(&paths, service_id); + // The dangerous shape: a supervisor recorded without a start-time token, + // and an engine with one. + record.supervisor_pid = 4242; + record.supervisor_start_ticks = None; + record.engine_pid = Some(4243); + record.engine_start_ticks = Some(77); + record.write().unwrap(); + + // The termination of each recorded PID, observed: the real routine runs + // (which PIDs it visits, in which order), only the signalling is replaced. + let signalled = std::cell::RefCell::new(Vec::new()); + let stop = |record: &ManagedServiceRecord| { + rocm_core::terminate_recorded_service_pids(record, |identity| { + signalled.borrow_mut().push(*identity); + Graceful + }) + }; + + let first = stop_internal_managed_service_with(&paths, service_id, stop).unwrap(); + let after_first = raw_manifest(&paths, service_id); + let signalled_by_first = signalled.borrow().clone(); + + let second = stop_internal_managed_service_with(&paths, service_id, stop).unwrap(); + let after_second = raw_manifest(&paths, service_id); + let signalled_in_all = signalled.borrow().clone(); + let _ = fs::remove_dir_all(root); + + assert_eq!(first["stopped"], true, "{first}"); + assert_eq!( + signalled_by_first, + vec![ + rocm_core::ProcessIdentity::new(4242, None), + rocm_core::ProcessIdentity::new(4243, Some(77)), + ] + ); + // The persisted state after the confirmed stop names no process. + assert_eq!(after_first.status, "stopped"); + assert_eq!(after_first.stop_requested_unix_ms, None); + assert_eq!(after_first.supervisor_pid, 0); + assert_eq!(after_first.supervisor_start_ticks, None); + assert_eq!(after_first.engine_pid, None); + assert_eq!(after_first.engine_start_ticks, None); + // So a stop of the same record signals nothing, and still reads stopped. + assert_eq!( + signalled_in_all, signalled_by_first, + "the second stop reached a PID the first one had already stopped" + ); + assert_eq!(second["stopped"], true, "{second}"); + assert_eq!(second["signaled_pids"], serde_json::json!([]), "{second}"); + assert_eq!(second["pid_outcomes"], serde_json::json!([]), "{second}"); + assert_eq!(after_second.status, "stopped"); + assert_eq!(after_second.supervisor_pid, 0); + assert_eq!(after_second.engine_pid, None); + } + + // Unix only: the processes must really be running. The second stop reads the + // record through the liveness refresh, which forgets the PIDs of processes it + // observes gone (`a_stop_completed_by_a_later_listing_leaves_no_pid_for_the_next_stop`), + // so made-up PIDs would be forgotten and not kept. + #[cfg(unix)] + #[test] + fn an_unconfirmed_stop_keeps_the_recorded_processes_for_the_next_stop() { + use rocm_core::TerminationOutcome::TimedOut; + let mut supervisor = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn supervisor"); + let mut engine = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn engine"); + let (supervisor_pid, engine_pid) = (supervisor.id(), engine.id()); + let (root, paths) = test_paths("services-stop-keeps-pids"); + paths.ensure().unwrap(); + let service_id = "svc-services-stop-keeps-pids"; + let mut record = stop_test_record(&paths, service_id); + record.supervisor_pid = supervisor_pid; + record.supervisor_start_ticks = Some(5); + record.engine_pid = Some(engine_pid); + record.engine_start_ticks = Some(77); + record.write().unwrap(); + + let signalled = std::cell::Cell::new(0_usize); + let stop = |record: &ManagedServiceRecord| { + rocm_core::terminate_recorded_service_pids(record, |_| { + signalled.set(signalled.get() + 1); + TimedOut + }) + }; + let first = stop_internal_managed_service_with(&paths, service_id, stop).unwrap(); + let after_first = raw_manifest(&paths, service_id); + let _ = stop_internal_managed_service_with(&paths, service_id, stop).unwrap(); + let signalled_in_all = signalled.get(); + let _ = supervisor.kill(); + let _ = supervisor.wait(); + let _ = engine.kill(); + let _ = engine.wait(); + let _ = fs::remove_dir_all(root); + + // Nothing was confirmed gone, so the record still names what survived: + // that is the only thing a later stop can find them by. + assert_eq!(first["stopped"], false, "{first}"); + assert_eq!(after_first.supervisor_pid, supervisor_pid); + assert_eq!(after_first.supervisor_start_ticks, Some(5)); + assert_eq!(after_first.engine_pid, Some(engine_pid)); + assert_eq!(after_first.engine_start_ticks, Some(77)); + assert_eq!(signalled_in_all, 4, "each stop tried both processes"); + } + #[test] fn restart_does_not_start_an_engine_over_an_unconfirmed_stop() { // A restart used to discard its stop's result and start the new engine @@ -1751,6 +2070,49 @@ mod tests { assert_eq!(on_disk.supervisor_pid, std::process::id()); } + /// The relaunch generation a restart stamps lives only until the restart + /// publishes: recording the child clears it, and a record whose generation + /// moved since the restart read it (another restart began, or this one's + /// stamp is gone) is not written over, as one with other processes is not. + #[test] + fn a_restart_publishing_its_child_ends_the_relaunch_generation_and_respects_a_moved_one() { + let (root, paths) = test_paths("restart-relaunch-generation"); + paths.ensure().unwrap(); + let service_id = "svc-restart-relaunch-generation"; + let mut stopped = stop_test_record(&paths, service_id); + stopped.status = "stopped".to_owned(); + stopped.clear_recorded_processes(); + stopped.begin_relaunch(); + stopped.write().unwrap(); + let stopped = raw_manifest(&paths, service_id); + assert!(stopped.relaunch_started_unix_ms.is_some(), "premise"); + + // Another restart began meanwhile: the generation on disk is not the one + // this restart read. + let mut moved = stopped.clone(); + moved.begin_relaunch(); + moved.write().unwrap(); + assert!( + moved.relaunch_started_unix_ms > stopped.relaunch_started_unix_ms, + "a new generation is always distinct from the one before it" + ); + let refused = record_restarted_child(&paths, &stopped, None, std::process::id()); + let untouched = raw_manifest(&paths, service_id); + + // With the generation as the restart read it, publication goes through. + stopped.write().unwrap(); + let started = record_restarted_child(&paths, &stopped, None, std::process::id()).unwrap(); + let on_disk = raw_manifest(&paths, service_id); + let _ = fs::remove_dir_all(root); + + assert!(refused.is_err(), "a moved generation is not written over"); + assert_eq!(untouched.status, "stopped"); + assert_eq!(untouched.supervisor_pid, 0, "no child was recorded"); + assert_eq!(started.relaunch_started_unix_ms, None); + assert_eq!(on_disk.relaunch_started_unix_ms, None); + assert_eq!(on_disk.supervisor_pid, std::process::id()); + } + #[test] fn restart_reports_a_child_it_could_not_record_and_writes_nothing() { // Between the restart's stop and its spawn, another restart recorded @@ -1907,8 +2269,11 @@ mod tests { // The claim the message makes: the service really is stopped, and no // replacement child was started behind the failed wait. assert_eq!(record.status, "stopped"); + // The confirmed stop cleared the recorded processes (the record names + // none), so a supervisor recorded behind the failed wait would show as a + // nonzero PID. assert_eq!( - record.supervisor_pid, 999_999_999, + record.supervisor_pid, 0, "no new supervisor may be recorded behind the failed wait" ); Ok(()) @@ -2002,6 +2367,60 @@ mod tests { Ok(()) } + /// An older stop finalizing while a restart is between its own confirmed stop + /// and the publication of its child. The record is then an empty `stopped` + /// manifest, which is also what an older stop's own completion looks like, but + /// the restart has restored the endpoint key and is about to serve with it. + /// The older stop (here the outer one, whose termination step runs the whole + /// restart, so its snapshot predates it) must neither claim that record nor + /// delete the key. The restart's spawn is failed on purpose, which leaves the + /// record exactly as it is at the spawn: published nothing, key restored. + #[cfg(not(windows))] + #[test] + fn an_older_stop_finalizing_before_a_restart_publishes_does_not_delete_its_key() -> Result<()> { + let (root, paths) = test_paths("older-stop-during-restart"); + paths.ensure()?; + let service_id = "svc-older-stop-during-restart"; + // A live PID (this test's own), so the restart's own load does not take + // the older stop's marker for a completed stop and clear the key before + // the restart has captured it. Nothing signals it: every termination + // step here is a stand-in. + let mut planted = stop_test_record(&paths, service_id); + planted.supervisor_pid = std::process::id(); + planted.write()?; + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123")?; + let restart_reached_spawn = std::cell::Cell::new(false); + + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + let spawn: &RestartSpawnOverride<'_> = &|_command| { + restart_reached_spawn.set(true); + Err(io::Error::other("spawn intercepted by test")) + }; + let restart = restart_internal_managed_service_inner( + &paths, + service_id, + None, + |_| bare_termination(Vec::new(), true), + Some(spawn), + ); + assert!(restart.is_err(), "the intercepted spawn fails the restart"); + bare_termination(Vec::new(), true) + })?; + + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + assert!(restart_reached_spawn.get(), "premise: the restart ran"); + assert_eq!( + key.as_deref(), + Some("key-123"), + "the key the restart restored for its child is not the older stop's to delete" + ); + assert_eq!(report["stopped"], false, "{report}"); + assert_eq!(on_disk.restart_count, 0, "nothing was published"); + Ok(()) + } + #[cfg(not(windows))] #[test] fn restart_is_not_blocked_by_a_stopped_or_failed_record_on_the_same_port() -> Result<()> { @@ -2383,7 +2802,7 @@ mod tests { "svc-cpu-policy", "vllm", "starting", - // Skipped by `terminate_recorded_service_pids`, which never + // Skipped by the shared stop routine, which never // signals this process — so the stop runs for real without the // test killing itself. std::process::id(), @@ -2469,15 +2888,14 @@ mod tests { let real = rocm_core::process_start_ticks(pid).expect("start-ticks"); let record = managed_record_for_pid(&paths, pid, Some(real.wrapping_add(1))); - let RecordedStopTermination { - signaled_pids: signaled, - all_stopped, - .. - } = terminate_recorded_service_pids(&record); + let stops = stop_recorded_processes(&record); - assert!(signaled.is_empty(), "must not signal a mismatched identity"); + assert!( + stops.iter().all(|stop| !stop.signaled()), + "must not signal a mismatched identity" + ); // A recycled PID means the recorded service process itself is gone. - assert!(all_stopped); + assert!(stops.iter().all(|stop| stop.stopped())); assert!( rocm_core::process_is_running(pid), "the unrelated live process must survive" @@ -2507,11 +2925,13 @@ mod tests { record.engine_pid = Some(eng_pid); record.engine_start_ticks = rocm_core::process_start_ticks(eng_pid); - let RecordedStopTermination { - signaled_pids: mut signaled, - all_stopped, - .. - } = terminate_recorded_service_pids(&record); + let stops = stop_recorded_processes(&record); + let all_stopped = stops.iter().all(|stop| stop.stopped()); + let mut signaled: Vec = stops + .iter() + .filter(|stop| stop.signaled()) + .map(|stop| stop.pid) + .collect(); signaled.sort_unstable(); let mut expected = vec![sup_pid, eng_pid]; expected.sort_unstable(); @@ -2539,14 +2959,15 @@ mod tests { let real = rocm_core::process_start_ticks(pid).expect("start-ticks"); let record = managed_record_for_pid(&paths, pid, Some(real)); - let RecordedStopTermination { - signaled_pids: signaled, - all_stopped, - .. - } = terminate_recorded_service_pids(&record); + let stops = stop_recorded_processes(&record); + let signaled: Vec = stops + .iter() + .filter(|stop| stop.signaled()) + .map(|stop| stop.pid) + .collect(); assert_eq!(signaled, vec![pid]); - assert!(all_stopped); + assert!(stops.iter().all(|stop| stop.stopped())); // Reap our own child (a detached managed process would be reaped by init) // so the liveness check does not observe a not-yet-reaped zombie. let mut child = child; @@ -2557,4 +2978,468 @@ mod tests { ); let _ = fs::remove_dir_all(root); } + + // ---- This module's stop, through the routine it shares with `rocmd`. ---- + // + // The routine itself (visiting order, start-time pairing, the `self` row, + // the per-outcome classification, the Windows tree-kill step) is tested in + // `rocm-core`'s `proc_lifecycle.rs`. These check that this caller reaches it + // and reports what it returns. Every PID below is one that cannot be + // running, so nothing is signalled and the tests behave the same on every + // host. + + /// A PID no host has in use, so the stop reports it as already gone. + const ABSENT_SUPERVISOR_PID: u32 = 999_999_998; + const ABSENT_ENGINE_PID: u32 = 999_999_997; + + /// Run the production stop of a record naming `supervisor` and `engine`, + /// and return its report. + fn stop_report_for(name: &str, supervisor: u32, engine: Option) -> serde_json::Value { + let (root, paths) = test_paths(name); + paths.ensure().unwrap(); + let service_id = "svc-stop-through-the-shared-routine"; + let mut record = stop_test_record(&paths, service_id); + record.supervisor_pid = supervisor; + record.engine_pid = engine; + record.write().unwrap(); + + let report = + stop_internal_managed_service_with(&paths, service_id, stop_recorded_processes) + .unwrap(); + let _ = fs::remove_dir_all(root); + report + } + + #[test] + fn a_stop_reports_the_supervisor_then_the_engine_in_that_order() { + let report = stop_report_for( + "managed-stop-order", + ABSENT_SUPERVISOR_PID, + Some(ABSENT_ENGINE_PID), + ); + + assert_eq!( + report["pid_outcomes"], + serde_json::json!([ + {"pid": ABSENT_SUPERVISOR_PID, "role": "supervisor", "outcome": "already_gone"}, + {"pid": ABSENT_ENGINE_PID, "role": "engine", "outcome": "already_gone"}, + ]) + ); + assert_eq!(report["signaled_pids"], serde_json::json!([])); + assert_eq!(report["stopped"], true); + } + + #[test] + fn a_stop_reports_a_pid_recorded_twice_once_under_both_roles() { + let report = stop_report_for( + "managed-stop-aliased", + ABSENT_SUPERVISOR_PID, + Some(ABSENT_SUPERVISOR_PID), + ); + + assert_eq!( + report["pid_outcomes"], + serde_json::json!([ + {"pid": ABSENT_SUPERVISOR_PID, "role": "supervisor_and_engine", "outcome": "already_gone"}, + ]) + ); + } + + /// Both binaries report the stopping process's own PID as `self`, never + /// signal it, and do not let it stand in the way of a confirmed stop. `rocm` + /// used to drop it from the report, so `pid_outcomes` showed one entry fewer + /// than `rocmd`'s for the same record. + #[test] + fn a_stop_reports_its_own_pid_as_self() { + let own = std::process::id(); + + let report = stop_report_for("managed-stop-own-pid", own, Some(ABSENT_ENGINE_PID)); + + assert_eq!( + report["pid_outcomes"], + serde_json::json!([ + {"pid": own, "role": "supervisor", "outcome": "self"}, + {"pid": ABSENT_ENGINE_PID, "role": "engine", "outcome": "already_gone"}, + ]) + ); + assert_eq!( + report["signaled_pids"], + serde_json::json!([]), + "the stopping process is never signalled" + ); + assert_eq!(report["stopped"], true); + } + + #[test] + fn a_stop_of_a_record_naming_only_its_own_pid_confirms() { + let own = std::process::id(); + + let report = stop_report_for("managed-stop-only-own-pid", own, None); + + assert_eq!( + report["pid_outcomes"], + serde_json::json!([{"pid": own, "role": "supervisor", "outcome": "self"}]) + ); + assert_eq!(report["stopped"], true); + } + + /// The `self` row is not one of the outcomes that leave a stop unconfirmed, + /// so it never reaches the report's `still running` or `not signaled` lines. + #[test] + fn a_self_row_adds_no_line_to_an_unconfirmed_report() { + let (root, paths) = test_paths("managed-stop-self-row-unconfirmed"); + paths.ensure().unwrap(); + let service_id = "svc-self-row-unconfirmed"; + stop_test_record(&paths, service_id).write().unwrap(); + let own = std::process::id(); + + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + vec![ + rocm_core::RecordedPidStop { + pid: own, + role: "supervisor", + outcome: rocm_core::RecordedPidOutcome::OwnProcess, + }, + rocm_core::RecordedPidStop { + role: "engine", + ..stop_of(999_999_999, rocm_core::TerminationOutcome::TimedOut) + }, + ] + }) + .unwrap(); + let answer = internal_stop_server_value(report); + let outcome = finish_service_action(&paths, "stop_server", service_id, &answer); + let _ = fs::remove_dir_all(root); + let rendered = format!("{:#}", outcome.expect_err("unconfirmed")); + + assert!( + rendered.contains("still running: process 999999999 (engine)"), + "{rendered}" + ); + assert!( + !rendered.contains(&format!("process {own}")), + "the stopping process is not a survivor: {rendered}" + ); + } + + #[cfg(unix)] + #[test] + fn managed_stop_signals_the_supervisor_before_the_engine() { + let sup = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn supervisor"); + let eng = std::process::Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn engine"); + let (sup_pid, eng_pid) = (sup.id(), eng.id()); + let (root, paths) = test_paths("managed-stop-signal-order"); + let mut record = + managed_record_for_pid(&paths, sup_pid, rocm_core::process_start_ticks(sup_pid)); + record.engine_pid = Some(eng_pid); + record.engine_start_ticks = rocm_core::process_start_ticks(eng_pid); + + let stops = stop_recorded_processes(&record); + let (mut sup, mut eng) = (sup, eng); + let _ = sup.wait(); + let _ = eng.wait(); + let _ = fs::remove_dir_all(root); + + // Not sorted: the order is the routine's, and a copy that visited the + // engine first would fail here. + assert_eq!( + stops + .iter() + .filter(|stop| stop.signaled()) + .map(|stop| stop.pid) + .collect::>(), + vec![sup_pid, eng_pid] + ); + assert!(stops.iter().all(|stop| stop.stopped())); + } + + /// A second copy of the shared routine, or of its grace constant, is how the + /// two binaries diverged in the first place; the shared one lives in + /// `rocm-core`'s `proc_lifecycle.rs`. The needles are assembled so that this + /// test does not match itself. + #[test] + fn this_module_carries_no_private_copy_of_the_shared_stop_routine() { + let source = include_str!("service_control.rs"); + for needle in [ + ["const ", "MANAGED_STOP_GRACE"].concat(), + ["fn ", "terminate_recorded_service_pids"].concat(), + ["fn ", "terminate_recorded_pid"].concat(), + ["Command::new(\"task", "kill\")"].concat(), + ] { + assert!( + !source.contains(&needle), + "`{needle}` is defined in `rocm-core`, not here" + ); + } + } + + /// The stop and the liveness read after it, end to end: a stop that cannot + /// verify a recorded process leaves the key and the stop request, and a later + /// refresh that still cannot query the process must not take them away or + /// reconcile the record to `stopped`. The refresh reads the record the stop + /// wrote, as a service listing does. + #[test] + fn an_unverified_stop_keeps_its_key_and_marker_through_a_later_liveness_refresh() { + let (root, paths) = test_paths("stop-unverified-then-refresh"); + paths.ensure().unwrap(); + let service_id = "svc-unverified-then-refresh"; + stop_test_record(&paths, service_id).write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + vec![stop_of( + 999_999_999, + rocm_core::TerminationOutcome::Unverified, + )] + }) + .unwrap(); + assert_eq!(report["stopped"], false, "{report}"); + let (mut record, _) = read_managed_service_manifest(&paths, service_id).unwrap(); + assert!( + record.stop_requested_unix_ms.is_some(), + "the stop is pending" + ); + + let changed = crate::managed_service_status::refresh_managed_service_runtime_liveness_with( + &paths, + &mut record, + |_| rocm_core::ProcessProbe::Unqueryable, + ); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + + assert_eq!(key.as_deref(), Some("key-123"), "the key is kept"); + assert!( + record.stop_requested_unix_ms.is_some(), + "the stop request is kept" + ); + assert!( + !changed, + "nothing about the record may change while the process is unqueryable" + ); + } + + /// An unconfirmed stop that a later listing completes (every recorded process + /// is observed gone) must leave a record that names no process, as a confirmed + /// stop does: otherwise the next `services stop` signals whatever the OS has + /// since given those PIDs to, and a PID with no start-time token cannot be + /// refuted. The same record, written back and stopped again, visits no PID. + #[test] + fn a_stop_completed_by_a_later_listing_leaves_no_pid_for_the_next_stop() { + let (root, paths) = test_paths("stop-completed-by-listing-clears-pids"); + paths.ensure().unwrap(); + let service_id = "svc-completed-by-listing"; + let mut planted = stop_test_record(&paths, service_id); + planted.supervisor_pid = 4242; + planted.supervisor_start_ticks = None; + planted.engine_pid = Some(4243); + planted.engine_start_ticks = Some(77); + planted.write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + vec![stop_of(4242, rocm_core::TerminationOutcome::Unverified)] + }) + .unwrap(); + assert_eq!(report["stopped"], false, "{report}"); + let (mut record, _) = read_managed_service_manifest(&paths, service_id).unwrap(); + assert_eq!( + record.supervisor_pid, 4242, + "an unconfirmed stop keeps the PIDs" + ); + + // The listing that finds both processes gone completes the stop. + let changed = crate::managed_service_status::refresh_managed_service_runtime_liveness_with( + &paths, + &mut record, + |_| rocm_core::ProcessProbe::Gone, + ); + assert!(changed); + record.write().unwrap(); + let after_listing = raw_manifest(&paths, service_id); + + let visited = std::cell::RefCell::new(Vec::new()); + let second = stop_internal_managed_service_with(&paths, service_id, |record| { + rocm_core::terminate_recorded_service_pids(record, |identity| { + visited.borrow_mut().push(*identity); + rocm_core::TerminationOutcome::Graceful + }) + }) + .unwrap(); + let visited = visited.into_inner(); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + + assert_eq!(after_listing.status, "stopped"); + assert_eq!(after_listing.stop_requested_unix_ms, None); + assert_eq!(key, None, "the deferred cleanup drops the key"); + assert_eq!( + ( + after_listing.supervisor_pid, + after_listing.supervisor_start_ticks, + after_listing.engine_pid, + after_listing.engine_start_ticks + ), + (0, None, None, None) + ); + assert!( + visited.is_empty(), + "a later stop reached a PID the listing had observed gone: {visited:?}" + ); + assert_eq!(second["stopped"], true, "{second}"); + assert_eq!(second["pid_outcomes"], serde_json::json!([]), "{second}"); + } + + /// A stop whose recorded process was seen to exit but whose tree kill did not + /// complete says so in those words, not as a timeout, and keeps what an + /// unconfirmed stop keeps: the claim and the state are checked together. + #[test] + fn a_stop_with_an_unconfirmed_tree_kill_says_what_happened_and_keeps_the_service() { + let (root, paths) = test_paths("services-stop-descendants-unconfirmed"); + paths.ensure().unwrap(); + let service_id = "svc-descendants-unconfirmed"; + stop_test_record(&paths, service_id).write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |_| { + vec![stop_of( + 999_999_999, + rocm_core::TerminationOutcome::DescendantsUnconfirmed, + )] + }) + .unwrap(); + let answer = internal_stop_server_value(report); + let outcome = finish_service_action(&paths, "stop_server", service_id, &answer); + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + // What the report says about the warning, checked against what happens + // next: a later stop finds the root gone and, with no new evidence about + // what was under it, reports the stop confirmed. + let later = stop_internal_managed_service_with(&paths, service_id, |_| { + vec![stop_of( + 999_999_999, + rocm_core::TerminationOutcome::AlreadyGone, + )] + }) + .unwrap(); + let _ = fs::remove_dir_all(root); + + assert_eq!(answer["status"], "stop_unconfirmed", "{answer}"); + let rendered = format!("{:#}", outcome.expect_err("not a confirmed stop")); + assert!( + rendered.contains( + "not confirmed: process 999999999 (supervisor_and_engine) exited, but a process under it may still be running" + ), + "{rendered}" + ); + assert!( + rendered.contains("rocm does not keep this warning, so a later stop or listing will find the process gone and report the stop confirmed"), + "the report must not imply the warning outlives this run: {rendered}" + ); + assert_eq!( + later["stopped"], true, + "the claim the report makes is true: {later}" + ); + assert!( + !rendered.contains("still running: process") + && !rendered.contains("did not exit after it was signaled"), + "the root was seen to exit, so nothing outlived a deadline: {rendered}" + ); + assert!(!rendered.contains("Local server stopped"), "{rendered}"); + assert_no_retry_line(&rendered); + // The claim of what was kept, against what was kept. + assert!(rendered.contains("kept: the server record"), "{rendered}"); + assert_eq!(on_disk.supervisor_pid, 999_999_999); + assert!(on_disk.stop_requested_unix_ms.is_some()); + assert_eq!(key.as_deref(), Some("key-123")); + } + + /// A root that exited by itself before its tree could be taken down confirms + /// the stop only through the record's other roles, through the real routine + /// and this caller's own folding: the root's row alone is not enough, and the + /// record, its PIDs, its stop request and its key are kept unless the engine + /// role is confirmed gone too. + #[test] + fn a_root_that_exited_alone_confirms_the_stop_only_when_the_engine_role_does() { + use rocm_core::TerminationOutcome::{ExitedBeforeTreeKill, Forced, TimedOut}; + + for (engine, confirmed) in [(Forced, true), (TimedOut, false)] { + let (root, paths) = test_paths("services-stop-root-exited-alone"); + paths.ensure().unwrap(); + let service_id = "svc-root-exited-alone"; + let mut record = stop_test_record(&paths, service_id); + record.supervisor_pid = 999_999_998; + record.engine_pid = Some(999_999_997); + record.write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |record| { + rocm_core::terminate_recorded_service_pids(record, |identity| { + if identity.pid == 999_999_998 { + ExitedBeforeTreeKill + } else { + engine + } + }) + }) + .unwrap(); + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + + assert_eq!(report["stopped"], confirmed, "{engine:?}: {report}"); + if confirmed { + assert_eq!(on_disk.status, "stopped"); + assert_eq!( + on_disk.supervisor_pid, 0, + "the confirmed stop clears the PIDs" + ); + assert_eq!(key, None, "and the key"); + } else { + assert_eq!(on_disk.supervisor_pid, 999_999_998, "the PIDs are kept"); + assert!(on_disk.stop_requested_unix_ms.is_some()); + assert_eq!(key.as_deref(), Some("key-123"), "the key is kept"); + } + } + } + + /// A lone root that exited by itself is not vouched for by anything else the + /// record names, so this caller does not clear the record on its say-so. + #[test] + fn a_lone_root_that_exited_alone_leaves_the_stop_unconfirmed() { + let (root, paths) = test_paths("services-stop-lone-root-exited-alone"); + paths.ensure().unwrap(); + let service_id = "svc-lone-root-exited-alone"; + stop_test_record(&paths, service_id).write().unwrap(); + endpoint_keys::store_endpoint_api_key(&paths, service_id, "key-123").unwrap(); + + let report = stop_internal_managed_service_with(&paths, service_id, |record| { + rocm_core::terminate_recorded_service_pids(record, |_| { + rocm_core::TerminationOutcome::ExitedBeforeTreeKill + }) + }) + .unwrap(); + let answer = internal_stop_server_value(report); + let outcome = finish_service_action(&paths, "stop_server", service_id, &answer); + let on_disk = raw_manifest(&paths, service_id); + let key = endpoint_keys::endpoint_api_key(&paths, service_id); + let _ = fs::remove_dir_all(root); + + let rendered = format!("{:#}", outcome.expect_err("not a confirmed stop")); + assert!( + rendered.contains("was already gone when the processes under it were to be taken down"), + "{rendered}" + ); + assert!(!rendered.contains("Local server stopped"), "{rendered}"); + assert_no_retry_line(&rendered); + assert_eq!(on_disk.supervisor_pid, 999_999_999); + assert!(on_disk.stop_requested_unix_ms.is_some()); + assert_eq!(key.as_deref(), Some("key-123")); + } } diff --git a/apps/rocm/src/service_hints.rs b/apps/rocm/src/service_hints.rs index 49d659dc10..6556c2e70a 100644 --- a/apps/rocm/src/service_hints.rs +++ b/apps/rocm/src/service_hints.rs @@ -38,6 +38,16 @@ pub(crate) fn service_action_hint(action: &str, service_id: &str) -> String { format!("rocm services {action} {} --yes", shell_quote(service_id)) } +/// The directly-runnable `rocm services remove --force --yes` hint: the +/// removal that goes ahead for a record whose recorded processes cannot be +/// queried (see `remove_managed_service_record`). +pub(crate) fn service_forced_remove_hint(service_id: &str) -> String { + format!( + "rocm services remove {} --force --yes", + shell_quote(service_id) + ) +} + /// The directly-runnable `rocm services stop --yes` hint. pub(crate) fn service_stop_hint(service_id: &str) -> String { service_action_hint("stop", service_id) diff --git a/apps/rocm/src/service_launch/attach.rs b/apps/rocm/src/service_launch/attach.rs index cd1bfe7700..d76a90e8d4 100644 --- a/apps/rocm/src/service_launch/attach.rs +++ b/apps/rocm/src/service_launch/attach.rs @@ -561,7 +561,13 @@ mod tests { finish("attached-unverified", Ok(stop_answer(false, "unverified"))); let error = format!("{:#}", result.expect_err("an unverified stop fails")); assert!(error.contains("not signaled: process 4242"), "{error}"); - assert!(error.contains("check: `ps -p 4242 -o pid,args`"), "{error}"); + assert!( + error.contains(&format!( + "check: `{}`", + crate::services_text::process_check_command(4242) + )), + "{error}" + ); crate::services_text::tests::assert_no_retry_line(&error); assert!(out.is_empty(), "{out}"); assert_eq!(signalled, None); diff --git a/apps/rocm/src/service_launch/launch_transaction.rs b/apps/rocm/src/service_launch/launch_transaction.rs index 13fe90ed1b..4fb530a5b8 100644 --- a/apps/rocm/src/service_launch/launch_transaction.rs +++ b/apps/rocm/src/service_launch/launch_transaction.rs @@ -104,6 +104,30 @@ pub(super) fn resolve_managed_port_in_transaction( /// be published. Short: the child is milliseconds old and has nothing to flush. const MANAGED_LAUNCH_CLEANUP_GRACE: Duration = Duration::from_secs(2); +/// Whether this host records a start token for a process: Linux (the kernel +/// start-time) and Windows (the creation time). Anywhere else +/// `rocm_core::process_start_ticks` has nothing to read. +const fn host_records_start_tokens() -> bool { + cfg!(any(target_os = "linux", windows)) +} + +/// The verdict that ends a failed launch's cleanup before anything is signalled: +/// `Some(Unverified)` for a child whose token was not captured on a host that +/// records tokens, because that child was gone before it could be captured and +/// its pid may now belong to another process. `None` otherwise: a captured token +/// is verified by the termination, and on a host with no tokens there is nothing +/// to verify, so the kill proceeds as it always did there. +const fn launch_cleanup_refusal( + engine_start_ticks: Option, + host_records_tokens: bool, +) -> Option { + if engine_start_ticks.is_none() && host_records_tokens { + Some(rocm_core::TerminationOutcome::Unverified) + } else { + None + } +} + /// Undo a failed managed launch attempt before the launch lock is released, /// so no failed transaction leaves a live reservation or a stray endpoint key. /// @@ -120,23 +144,54 @@ const MANAGED_LAUNCH_CLEANUP_GRACE: Duration = Duration::from_secs(2); /// Every step is best-effort and idempotent; cleanup must never mask the /// original failure with a second one. pub(super) fn abandon_failed_managed_launch(paths: &AppPaths, record: &ManagedServiceRecord) { + // The shared routine `rocm services stop` and `rocmd` use, not + // `terminate_verified` alone: on Windows only the routine takes the + // processes under the child down, so a failed launch cannot leave the engine + // one level below the supervisor running. + abandon_failed_managed_launch_with(paths, record, |identity| { + rocm_core::terminate_recorded_pid_within(identity, MANAGED_LAUNCH_CLEANUP_GRACE) + }); +} + +/// [`abandon_failed_managed_launch`] with the termination of the child supplied +/// by the caller, so a test sees what the cleanup asks it to terminate, and +/// whether it asks at all. +fn abandon_failed_managed_launch_with( + paths: &AppPaths, + record: &ManagedServiceRecord, + terminate: impl Fn(&rocm_core::ProcessIdentity) -> rocm_core::TerminationOutcome, +) { if let Some(pid) = record.engine_pid.filter(|pid| *pid != 0) { // The token captured at spawn, not a fresh `capture`: when the child // already exited, a fresh one would describe a recycled pid's new owner // and the forced tree kill would hit it. Where start tokens exist - // (Linux), a missing one means the child was gone before it could be - // captured, so the pid is unverifiable and nothing is signalled - // (`terminate_verified` alone would "best-effort proceed"). Windows - // records no token at all, so there the kill proceeds as before. - let outcome = if record.engine_start_ticks.is_none() && cfg!(target_os = "linux") { - rocm_core::TerminationOutcome::Unverified + // (Linux, and Windows, which records the process creation time), a + // missing one means the child was gone before it could be captured, so + // the pid is unverifiable and nothing is signalled (`terminate_verified` + // alone would "best-effort proceed"). A token that was captured is + // verified by the termination on both. + // + // The tradeoff on Windows, chosen deliberately: a child whose creation + // time could not be read at capture is not killed, so if `OpenProcess` + // failed for a child that is in fact alive (it is this account's own + // fresh child, so that is not expected), a failed launch leaves its + // engine running and the audit event says `unverified`. The alternative + // is a forced tree kill of a pid that may by now belong to another + // process, which cannot be undone. The spawn primitive returns only the + // pid and closes its handle, so a handle-based identity would mean + // changing it; that is not done here. The refusal decision + // (`launch_cleanup_refusal`) and what the cleanup asks the termination to + // do are under tests that run on Windows; the cleanup of a real Windows + // child is run by no test. + let outcome = if let Some(refusal) = + launch_cleanup_refusal(record.engine_start_ticks, host_records_start_tokens()) + { + refusal } else { - rocm_core::terminate_verified( - &rocm_core::ProcessIdentity::new(pid, record.engine_start_ticks), - rocm_core::KillScope::Tree, - MANAGED_LAUNCH_CLEANUP_GRACE, - true, - ) + terminate(&rocm_core::ProcessIdentity::new( + pid, + record.engine_start_ticks, + )) }; record_cli_audit_event( paths, @@ -703,7 +758,7 @@ mod tests { // confirms every recorded process gone without touching the test // process itself. crate::service_control::stop_internal_managed_service_with(&paths, "lemonade-a-1", |_| { - crate::service_control::RecordedStopTermination::confirmed_stopped() + Vec::new() })?; let decision = resolve_managed_port_in_transaction( @@ -898,6 +953,55 @@ mod tests { Ok(()) } + /// What the cleanup asks of the termination, through the seam and so on every + /// host: the child's PID with the token captured at spawn, and nothing at all + /// for a child with no token where tokens are recorded. The record is marked + /// stopped either way and the audit event carries the verdict the termination + /// gave. + #[test] + fn abandoned_launch_asks_the_termination_for_the_spawned_child_with_its_token() -> Result<()> { + use std::cell::RefCell; + let cases = [ + // (token captured, whether the cleanup is expected to ask) + (Some(77_u64), true), + (None, !host_records_start_tokens()), + ]; + for (token, expect_call) in cases { + let (root, paths) = test_paths("abandon-launch-asks-termination"); + paths.ensure()?; + let mut record = write_live_service_record( + &paths, + "lemonade-abandon-seam", + "abandon-seam-model", + 11_494, + "running", + )?; + record.engine_pid = Some(4242); + record.engine_start_ticks = token; + let asked = RefCell::new(Vec::new()); + + abandon_failed_managed_launch_with(&paths, &record, |identity| { + asked.borrow_mut().push(*identity); + rocm_core::TerminationOutcome::Forced + }); + + let asked = asked.into_inner(); + let persisted = load_managed_services(&paths)? + .into_iter() + .find(|candidate| candidate.service_id == record.service_id) + .expect("a launch that spawned a child keeps its record"); + let _ = fs::remove_dir_all(root); + let expected = if expect_call { + vec![rocm_core::ProcessIdentity::new(4242, token)] + } else { + Vec::new() + }; + assert_eq!(asked, expected, "token {token:?}"); + assert_eq!(persisted.status, "stopped", "token {token:?}"); + } + Ok(()) + } + #[test] #[cfg(target_os = "linux")] fn abandoned_launch_never_signals_a_pid_that_no_longer_matches_the_spawn_token() -> Result<()> { @@ -999,4 +1103,35 @@ mod tests { ); Ok(()) } + + /// A failed launch's cleanup refuses to signal a child whose token was not + /// captured on any host that records tokens. Windows is one of them, since + /// it records the creation time: a missing token there means the child was + /// gone before it could be captured, and its pid may now be another + /// process's, so the kill must not proceed as it did when Windows had none. + #[test] + fn launch_cleanup_refuses_a_child_without_a_token_only_where_tokens_exist() { + use rocm_core::TerminationOutcome::Unverified; + + // A host that records tokens: a missing one is unverifiable, a captured + // one goes on to be verified by the termination. + assert_eq!(launch_cleanup_refusal(None, true), Some(Unverified)); + assert_eq!(launch_cleanup_refusal(Some(7), true), None); + // A host with no tokens has nothing to verify and proceeds as before. + assert_eq!(launch_cleanup_refusal(None, false), None); + assert_eq!(launch_cleanup_refusal(Some(7), false), None); + } + + /// Ties the cleanup's idea of "this host records tokens" to what + /// `process_start_ticks` really does, on the host the test runs on: Linux and + /// Windows read a token for a live process (this one), anywhere else there + /// is none. If a platform gains or loses tokens without the cleanup being + /// told, this fails there. + #[test] + fn the_cleanup_guard_agrees_with_whether_this_host_reads_tokens() { + assert_eq!( + host_records_start_tokens(), + rocm_core::process_start_ticks(std::process::id()).is_some() + ); + } } diff --git a/apps/rocm/src/service_records.rs b/apps/rocm/src/service_records.rs index 5fe013a611..2220426374 100644 --- a/apps/rocm/src/service_records.rs +++ b/apps/rocm/src/service_records.rs @@ -36,10 +36,11 @@ use std::path::{Path, PathBuf}; use std::time::{Duration, SystemTime}; use anyhow::{Context, Result, bail}; -use rocm_core::{AppPaths, ManagedServiceRecord, process_is_running}; +use rocm_core::{AppPaths, ManagedServiceRecord, ProcessProbe}; use crate::managed_service_status::{ - load_managed_services, managed_service_is_live, recorded_service_pids, + container_stop_unresolved, load_managed_services, managed_service_is_live, + recorded_pids_may_be_running, recorded_service_pids, }; use crate::services_text::{load_managed_service, validate_service_id}; use crate::{cli_progress, cli_report, endpoint_keys, service_hints}; @@ -165,11 +166,72 @@ fn remove_service_record_artifacts(artifacts: &ServiceRecordArtifacts) -> Result /// server whose engine reported failure while its process is still up reads as /// removable, and deleting it takes the log and the 0600 endpoint key out from /// under a process that may still be serving. -fn managed_service_record_is_in_use(record: &ManagedServiceRecord) -> bool { +/// +/// A recorded process that could not be queried (on Windows, a live one this +/// process may not open) counts as running: deleting the record and its key on +/// the strength of a failed look is the mistake this guard exists to prevent. +/// +/// The read of each recorded PID is supplied by the caller, so a test drives an +/// unqueryable process on every host; production passes +/// [`rocm_core::probe_process`]. +fn managed_service_record_is_in_use_with( + record: &ManagedServiceRecord, + probe: impl Fn(u32) -> ProcessProbe, +) -> bool { managed_service_is_live(record) - || recorded_service_pids(record) - .iter() - .any(|pid| process_is_running(*pid)) + || container_stop_unresolved(record) + || recorded_pids_may_be_running(record, probe) +} + +/// The recorded processes `--force` may remove a record over: `Some` only when +/// none of them is confirmed running, at least one could not be queried (so +/// `rocm` cannot tell whether it is still running), and a stop was requested on +/// the record, so the user has already tried to stop it. A process confirmed +/// running is never forced past. +fn forceable_unqueryable_pids( + record: &ManagedServiceRecord, + probe: impl Fn(u32) -> ProcessProbe, +) -> Option> { + let probed: Vec<(u32, ProcessProbe)> = recorded_service_pids(record) + .into_iter() + .map(|pid| (pid, probe(pid))) + .collect(); + if probed + .iter() + .any(|(_, probed)| *probed == ProcessProbe::Running) + { + return None; + } + let unqueryable: Vec = probed + .into_iter() + .filter(|(_, probed)| *probed == ProcessProbe::Unqueryable) + .map(|(pid, _)| pid) + .collect(); + (!unqueryable.is_empty() && record.stop_requested_unix_ms.is_some()).then_some(unqueryable) +} + +/// The recorded PIDs as the message names them: `process 4242`, or +/// `processes 4242 and 4243`. +fn named_processes(pids: &[u32]) -> String { + match pids { + [one] => format!("process {one}"), + many => format!( + "processes {}", + many.iter() + .map(u32::to_string) + .collect::>() + .join(" and ") + ), + } +} + +/// What [`remove_managed_service_record`] did. +pub(crate) struct RemovedRecord { + /// The text to print. + pub(crate) text: String, + /// The removal went ahead over a recorded process that could not be queried + /// (`--force`), rather than for a record that is not running. + pub(crate) forced: bool, } /// Remove one non-running local server record and everything it owns. @@ -180,28 +242,70 @@ fn managed_service_record_is_in_use(record: &ManagedServiceRecord) -> bool { /// which has already been refreshed against the engine state file and the real /// processes — a manifest still saying `ready` for a dead PID has demoted to /// `stopped` by then, and must stay removable. +/// +/// A recorded process that could not be queried (on Windows, a live one this +/// process may not open) makes the record count as running, and nothing else +/// `rocm` has can tell a dead server whose PID was reused from a live one. For +/// that case only, `--force` removes the record anyway: when no recorded process +/// is confirmed running, at least one could not be queried, and a stop was +/// requested. The removal then says exactly that, since the process may still +/// be serving and its endpoint key is deleted with the record. +#[cfg(test)] pub(crate) fn remove_managed_service_record( paths: &AppPaths, service_id: &str, yes: bool, ) -> Result { + remove_managed_service_record_with(paths, service_id, yes, false, rocm_core::probe_process) + .map(|removed| removed.text) +} + +/// [`remove_managed_service_record`] with `--force` and the read of each +/// recorded PID supplied by the caller. +pub(crate) fn remove_managed_service_record_with( + paths: &AppPaths, + service_id: &str, + yes: bool, + force: bool, + probe: impl Fn(u32) -> ProcessProbe, +) -> Result { validate_service_id(service_id)?; let record = load_managed_service(paths, service_id)?; + let mut forced_over = None; // Checked before `--yes` on purpose: for a running server "stop it first" is // the actionable error, and repeating the command with --yes must not be the // advice a user takes away from it. - if managed_service_record_is_in_use(&record) { - bail!( - "local server `{service_id}` is {} and cannot be removed while it is running.\n\nTry: {}", - record.status, - service_hints::service_stop_hint(service_id) - ); + if managed_service_record_is_in_use_with(&record, &probe) { + let forceable = forceable_unqueryable_pids(&record, &probe); + match (force, forceable) { + (true, Some(pids)) => forced_over = Some(pids), + (false, Some(pids)) => bail!( + "local server `{service_id}` cannot be removed: {} it recorded could not be queried, so `rocm` cannot tell whether it is still running.\n\nTry: {}", + named_processes(&pids), + service_hints::service_forced_remove_hint(service_id) + ), + (true, None) => bail!( + "--force removes a record only when none of its recorded processes is confirmed running, at least one could not be queried, and a stop was requested. Local server `{service_id}` is {} and does not meet that.\n\nTry: {}", + record.status, + service_hints::service_stop_hint(service_id) + ), + (false, None) => bail!( + "local server `{service_id}` is {} and cannot be removed while it is running.\n\nTry: {}", + record.status, + service_hints::service_stop_hint(service_id) + ), + } } if !yes { - bail!( - "Removing local server record `{service_id}` requires --yes.\n\nTry: {}", + // The command that completes this removal, not a neighbour of it: a + // forced removal that is missing `--yes` is told to add it, not to drop + // the `--force` it asked for. + let hint = if forced_over.is_some() { + service_hints::service_forced_remove_hint(service_id) + } else { service_hints::service_action_hint("remove", service_id) - ); + }; + bail!("Removing local server record `{service_id}` requires --yes.\n\nTry: {hint}"); } let artifacts = service_record_artifacts(paths, &record.service_id, &record.engine)?; @@ -213,6 +317,15 @@ pub(crate) fn remove_managed_service_record( "Removing local server record `{service_id}` (status: {}).", record.status ); + if let Some(pids) = &forced_over { + // Said before the delete and in full: what is being risked is the user's + // to know, and after this there is no record left to ask about. + let _ = writeln!( + output, + "--force: {} could not be queried, so `rocm` cannot tell whether it is still running. If it is, it keeps running; this removes the record and the endpoint key it may still be serving with.", + named_processes(pids) + ); + } let _ = writeln!(output, " log: {}", artifacts.log.display()); let _ = writeln!( output, @@ -231,7 +344,10 @@ pub(crate) fn remove_managed_service_record( report = report.detail("removed", path.display()); } let _ = write!(output, "{}", report.render()); - Ok(output) + Ok(RemovedRecord { + text: output, + forced: forced_over.is_some(), + }) } /// One record `rocm services prune` would delete. @@ -416,6 +532,7 @@ fn build_service_prune_plan( min_age: Duration, hours: u64, now: SystemTime, + probe: &impl Fn(u32) -> ProcessProbe, ) -> Result { let mut plan = ServicePrunePlan::default(); // Taken first, because `load_managed_services` below rewrites the manifest of @@ -427,14 +544,28 @@ fn build_service_prune_plan( // the real processes before returning it, so liveness below is read from the // refreshed view, never from the manifest as it was on disk. for record in load_managed_services(paths)? { - if managed_service_record_is_in_use(&record) { + if managed_service_record_is_in_use_with(&record, probe) { plan.skipped_live += 1; - plan.skipped.push(format!( - "{} is {} - stop it first with `{}`", - record.service_id, - record.status, - service_hints::service_stop_hint(&record.service_id) - )); + // A record held only by a process `rocm` cannot query, after a stop + // was requested, is the one `remove --force` exists for: say so and + // name it, instead of advising a stop that can only report the same + // thing again. `prune` itself never forces. + let skipped = match forceable_unqueryable_pids(&record, probe) { + Some(pids) => format!( + "{} is {} and {} it recorded could not be queried - remove it with `{}`", + record.service_id, + record.status, + named_processes(&pids), + service_hints::service_forced_remove_hint(&record.service_id) + ), + None => format!( + "{} is {} - stop it first with `{}`", + record.service_id, + record.status, + service_hints::service_stop_hint(&record.service_id) + ), + }; + plan.skipped.push(skipped); continue; } let artifacts = match service_record_artifacts(paths, &record.service_id, &record.engine) { @@ -579,10 +710,22 @@ pub(crate) struct ServicePruneOutcome { /// was built. That window is the only reason the liveness re-check below exists, /// and no test driving `prune` end to end can open it: one call builds the plan /// and applies it, so a record is either live for both halves or dead for both. +#[cfg(test)] fn apply_service_prune_plan( paths: &AppPaths, plan: &ServicePrunePlan, outcome: &mut ServicePruneOutcome, +) { + apply_service_prune_plan_with(paths, plan, outcome, &rocm_core::probe_process); +} + +/// [`apply_service_prune_plan`] with the read of each recorded PID supplied by +/// the caller. +fn apply_service_prune_plan_with( + paths: &AppPaths, + plan: &ServicePrunePlan, + outcome: &mut ServicePruneOutcome, + probe: &impl Fn(u32) -> ProcessProbe, ) { for entry in &plan.remove { // Liveness was snapshotted while the plan was built. A @@ -592,7 +735,7 @@ fn apply_service_prune_plan( // `remove` narrows the same window by loading the record it deletes. // Neither closes it: the check and the delete are not atomic either way. if load_managed_service(paths, &entry.service_id) - .is_ok_and(|record| managed_service_record_is_in_use(&record)) + .is_ok_and(|record| managed_service_record_is_in_use_with(&record, probe)) { outcome.skipped_live += 1; let _ = writeln!( @@ -718,6 +861,18 @@ pub(crate) fn prune_managed_service_records( hours: u64, dry_run: bool, yes: bool, +) -> Result { + prune_managed_service_records_with(paths, hours, dry_run, yes, &rocm_core::probe_process) +} + +/// [`prune_managed_service_records`] with the read of each recorded PID supplied +/// by the caller. +fn prune_managed_service_records_with( + paths: &AppPaths, + hours: u64, + dry_run: bool, + yes: bool, + probe: &impl Fn(u32) -> ProcessProbe, ) -> Result { if !dry_run && !yes { bail!( @@ -742,7 +897,7 @@ pub(crate) fn prune_managed_service_records( rocm_core::FileLock::acquire(paths.managed_launch_lock_path())? }; let min_age = Duration::from_secs(hours.saturating_mul(3600)); - let plan = build_service_prune_plan(paths, min_age, hours, SystemTime::now())?; + let plan = build_service_prune_plan(paths, min_age, hours, SystemTime::now(), probe)?; let mut outcome = ServicePruneOutcome { text: render_service_prune_plan(&plan, hours, dry_run), skipped_live: plan.skipped_live, @@ -753,7 +908,7 @@ pub(crate) fn prune_managed_service_records( return Ok(outcome); } - apply_service_prune_plan(paths, &plan, &mut outcome); + apply_service_prune_plan_with(paths, &plan, &mut outcome, probe); let _ = writeln!(outcome.text); let _ = write!( @@ -1795,4 +1950,423 @@ mod tests { assert!(manifest_exists, "a refused prune must delete nothing"); Ok(()) } + + /// A record whose status is not live is still in use while a process it + /// recorded may be running, and a process that could not be queried (on + /// Windows, a live one this process may not open) may be. Only a confirmed + /// exit makes it removable. + #[test] + fn a_record_is_in_use_unless_every_recorded_process_is_confirmed_gone() -> Result<()> { + let (root, paths) = test_paths("services-in-use-probe"); + paths.ensure()?; + let record = plant_service_record(&paths, "svc-in-use", "failed", 999_999_999)?; + let _ = fs::remove_dir_all(&root); + + assert!(managed_service_record_is_in_use_with(&record, |_| { + ProcessProbe::Unqueryable + })); + assert!(managed_service_record_is_in_use_with(&record, |_| { + ProcessProbe::Running + })); + assert!(!managed_service_record_is_in_use_with(&record, |_| { + ProcessProbe::Gone + })); + Ok(()) + } + + /// The by-hand way out, which still works alongside `--force`, for a record + /// whose process `rocm` cannot query: the record is refused while that process may be + /// running, and deleting its `.json` by hand and running + /// `prune --any-age` leaves nothing behind (log, key and engine state all + /// swept), with the endpoint key among the files removed. + #[test] + fn a_record_held_by_an_unqueryable_process_can_be_cleared_by_hand_and_prune() -> Result<()> { + let (root, paths) = test_paths("services-unqueryable-way-out"); + paths.ensure()?; + let record = plant_service_record(&paths, "svc-stuck", "stopped", 999_999_999)?; + let refused = managed_service_record_is_in_use_with(&record, |_| ProcessProbe::Unqueryable); + let key = endpoint_keys::endpoint_key_file_path(&paths, "svc-stuck"); + assert!(key.exists(), "the planted record has its key"); + + fs::remove_file(&record.manifest_path)?; + let outcome = prune_managed_service_records(&paths, 0, false, true); + let outcome = match outcome { + Ok(outcome) => outcome, + Err(error) => { + let _ = fs::remove_dir_all(&root); + return Err(error); + } + }; + let leftovers = [&record.log_path, &record.engine_state_path, &key] + .iter() + .filter(|path| path.exists()) + .count(); + let _ = fs::remove_dir_all(&root); + + assert!( + refused, + "a record whose process cannot be queried is in use" + ); + assert_eq!(leftovers, 0, "prune swept every leftover file"); + assert_eq!(outcome.removed_files, 3); + Ok(()) + } + + /// A record the user already tried to stop, held by the test's own live PID, + /// so the real liveness read that loads it leaves it alone and only the + /// injected probe says the process cannot be queried. + fn held_record(paths: &AppPaths, service_id: &str, stop_requested: bool) -> Result<()> { + let mut record = plant_service_record(paths, service_id, "ready", std::process::id())?; + if stop_requested { + record.stop_requested_unix_ms = Some(1); + record.write()?; + } + Ok(()) + } + + fn files_left(paths: &AppPaths, service_id: &str) -> usize { + let manifest = paths.service_manifest_path(service_id).exists(); + let log = paths.service_log_path(service_id).exists(); + let key = endpoint_keys::endpoint_key_file_path(paths, service_id).exists(); + usize::from(manifest) + usize::from(log) + usize::from(key) + } + + /// Without `--force` the refusal names the forced removal, and everything the + /// record owns is still there: the advice and the state, together. + #[test] + fn a_record_held_by_an_unqueryable_process_is_refused_naming_the_forced_removal() -> Result<()> + { + let (root, paths) = test_paths("services-remove-unqueryable-refused"); + paths.ensure()?; + held_record(&paths, "svc-held", true)?; + + let error = remove_managed_service_record_with(&paths, "svc-held", true, false, |_| { + ProcessProbe::Unqueryable + }) + .map(|removed| removed.text) + .expect_err("a record held by an unqueryable process is refused"); + let message = format!("{error:#}"); + let left = files_left(&paths, "svc-held"); + let _ = fs::remove_dir_all(root); + + assert!(message.contains("could not be queried"), "{message}"); + assert!( + message.contains("rocm services remove svc-held --force --yes"), + "{message}" + ); + assert_eq!(left, 3, "nothing was removed"); + // The command the refusal names parses as printed. + let words: Vec = service_hints::service_forced_remove_hint("svc-held") + .split_whitespace() + .map(str::to_owned) + .collect(); + assert_eq!( + crate::advised_commands::rocm_verdict(&words[1..]), + crate::advised_commands::Verdict::Parses, + "{words:?}" + ); + Ok(()) + } + + /// `--force` removes it and says what it risked; the audit-relevant flag is set. + #[test] + fn force_removes_a_record_held_by_an_unqueryable_process_and_says_so() -> Result<()> { + let (root, paths) = test_paths("services-remove-unqueryable-forced"); + paths.ensure()?; + held_record(&paths, "svc-held", true)?; + let pid = std::process::id(); + + let removed = remove_managed_service_record_with(&paths, "svc-held", true, true, |_| { + ProcessProbe::Unqueryable + })?; + let left = files_left(&paths, "svc-held"); + let _ = fs::remove_dir_all(root); + + assert!(removed.forced); + assert!( + removed.text.contains(&format!( + "--force: process {pid} could not be queried, so `rocm` cannot tell whether it is still running" + )), + "{}", + removed.text + ); + assert!(removed.text.contains("endpoint key"), "{}", removed.text); + assert_eq!(left, 0, "every file the record owned is gone"); + Ok(()) + } + + /// An `aim` stop whose engine `Stop` failed leaves the stop pending, and its + /// supervisor can be gone while the container still runs. Neither the liveness + /// refresh nor `remove` and `prune` may then treat the record as stopped and + /// delete it, with the key the container may still be enforcing: the refresh + /// leaves the status alone, `remove` refuses it, and `prune` skips it, all + /// with the supervisor confirmed gone. A process engine in the same state is + /// an ordinary stopped record that is removed. + #[test] + fn a_pending_container_stop_survives_refresh_remove_and_prune() -> Result<()> { + let gone = |_: u32| ProcessProbe::Gone; + let (root, paths) = test_paths("services-container-pending-stop"); + paths.ensure()?; + // A record already reading `stopped` (an earlier reconciliation wrote it) + // is the case only the in-use guard protects. + for (engine, status, id, protected) in [ + ("aim", "ready", "svc-container", true), + ("aim", "stopped", "svc-container-stopped", true), + ("vllm", "ready", "svc-process", false), + ] { + let mut record = plant_service_record(&paths, id, status, 999_999_999)?; + record.engine = engine.to_owned(); + record.stop_requested_unix_ms = Some(1); + record.write()?; + + let mut refreshed = record.clone(); + let _ = crate::managed_service_status::refresh_managed_service_runtime_liveness_with( + &paths, + &mut refreshed, + gone, + ); + let removal = remove_managed_service_record_with(&paths, id, true, false, gone); + let pruned = prune_managed_service_records_with(&paths, 0, false, true, &gone)?; + let left = files_left(&paths, id); + + if protected { + assert_eq!(refreshed.status, status, "{engine}: refresh"); + assert_eq!(refreshed.supervisor_pid, 999_999_999, "{engine}: refresh"); + assert!(removal.is_err(), "{engine}: remove must refuse"); + assert_eq!(pruned.removed_records, 0, "{engine}: prune"); + assert_eq!(left, 3, "{engine}: nothing was deleted"); + } else { + assert!(removal.is_ok() || pruned.removed_records == 1, "{engine}"); + assert_eq!(left, 0, "{engine}: an ordinary stopped record goes"); + } + } + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// A `ready` record whose dead server's PID is now held by a process this + /// account cannot open, and that nobody has tried to stop: `--force` is + /// refused (it never forces a record nobody tried to stop), and the refusal + /// names `services stop`. That stop cannot verify the process, reports it + /// unverified and records the request, after which `--force` removes the + /// record. So the record is not stuck: the way out is those two commands, and + /// this is the intended shape, not a gap. + #[test] + fn a_ready_record_held_by_an_unqueryable_process_is_cleared_by_stop_then_force() -> Result<()> { + let (root, paths) = test_paths("services-remove-ready-unqueryable-chain"); + paths.ensure()?; + held_record(&paths, "svc-ready-held", false)?; + let pid = std::process::id(); + + let first = + remove_managed_service_record_with(&paths, "svc-ready-held", true, true, |_| { + ProcessProbe::Unqueryable + }) + .map(|removed| removed.text) + .expect_err("a record nobody tried to stop is not forced"); + let report = crate::service_control::stop_internal_managed_service_with( + &paths, + "svc-ready-held", + |_| { + vec![rocm_core::RecordedPidStop { + pid, + role: "supervisor_and_engine", + outcome: rocm_core::RecordedPidOutcome::Terminated( + rocm_core::TerminationOutcome::Unverified, + ), + }] + }, + )?; + let second = + remove_managed_service_record_with(&paths, "svc-ready-held", true, true, |_| { + ProcessProbe::Unqueryable + })?; + let left = files_left(&paths, "svc-ready-held"); + let _ = fs::remove_dir_all(root); + + assert!( + format!("{first:#}").contains("rocm services stop svc-ready-held --yes"), + "{first:#}" + ); + assert_eq!(report["stopped"], false, "{report}"); + assert!(second.forced); + assert_eq!(left, 0, "the forced removal took the record and its files"); + Ok(()) + } + + /// `--force` never goes past a process that is confirmed running, and it + /// needs a stop to have been requested: the refusal says which, and names the + /// stop that is the way forward. + #[test] + fn force_is_refused_for_a_running_process_and_without_a_requested_stop() -> Result<()> { + let (root, paths) = test_paths("services-remove-force-refused"); + paths.ensure()?; + held_record(&paths, "svc-running", true)?; + held_record(&paths, "svc-no-stop", false)?; + + let running = remove_managed_service_record_with(&paths, "svc-running", true, true, |_| { + ProcessProbe::Running + }) + .map(|removed| removed.text) + .expect_err("a confirmed running process is never forced past"); + let no_stop = remove_managed_service_record_with(&paths, "svc-no-stop", true, true, |_| { + ProcessProbe::Unqueryable + }) + .map(|removed| removed.text) + .expect_err("a record nobody tried to stop is not forced"); + let left = files_left(&paths, "svc-running") + files_left(&paths, "svc-no-stop"); + let _ = fs::remove_dir_all(root); + + for error in [format!("{running:#}"), format!("{no_stop:#}")] { + assert!( + error.contains("--force removes a record only when"), + "{error}" + ); + } + assert!( + format!("{running:#}").contains("rocm services stop svc-running --yes"), + "{running:#}" + ); + assert_eq!(left, 6, "nothing was removed"); + Ok(()) + } + + /// `--force` on a record that is simply not running changes nothing: it is + /// removed as usual and not reported as forced. + #[test] + fn force_on_a_record_that_is_not_running_is_an_ordinary_removal() -> Result<()> { + let (root, paths) = test_paths("services-remove-force-unneeded"); + paths.ensure()?; + plant_service_record(&paths, "svc-stopped", "stopped", 999_999_999)?; + + let removed = + remove_managed_service_record_with(&paths, "svc-stopped", true, true, |_| { + ProcessProbe::Gone + })?; + let _ = fs::remove_dir_all(root); + + assert!(!removed.forced); + assert!(!removed.text.contains("--force"), "{}", removed.text); + Ok(()) + } + + /// One recorded process that cannot be queried does not make a record + /// forceable while another is confirmed running: the running one is never + /// forced past, whichever role it has. + #[test] + fn force_is_refused_when_one_process_is_unqueryable_and_another_is_running() -> Result<()> { + let (root, paths) = test_paths("services-remove-force-mixed"); + paths.ensure()?; + let own = std::process::id(); + let mut record = plant_service_record(&paths, "svc-mixed", "ready", own)?; + record.engine_pid = Some(999_999_997); + record.stop_requested_unix_ms = Some(1); + record.write()?; + + let error = remove_managed_service_record_with(&paths, "svc-mixed", true, true, |pid| { + if pid == own { + ProcessProbe::Unqueryable + } else { + ProcessProbe::Running + } + }) + .map(|removed| removed.text) + .expect_err("a record with a process confirmed running is never forced"); + let left = files_left(&paths, "svc-mixed"); + let _ = fs::remove_dir_all(root); + + assert!( + format!("{error:#}").contains("--force removes a record only when"), + "{error:#}" + ); + assert_eq!(left, 3, "nothing was removed"); + Ok(()) + } + + /// A forced removal that is missing `--yes` is told to add `--yes`, not to + /// drop the `--force` it asked for: the command it names completes the + /// removal, which is asserted by running it. + #[test] + fn a_forced_removal_missing_yes_names_the_command_that_completes_it() -> Result<()> { + let (root, paths) = test_paths("services-remove-force-needs-yes"); + paths.ensure()?; + held_record(&paths, "svc-held", true)?; + let unqueryable = |_: u32| ProcessProbe::Unqueryable; + + let error = + remove_managed_service_record_with(&paths, "svc-held", false, true, unqueryable) + .map(|removed| removed.text) + .expect_err("a forced removal without --yes is refused"); + let message = format!("{error:#}"); + let left_before = files_left(&paths, "svc-held"); + // The named command, run as named, removes the record. + let named = service_hints::service_forced_remove_hint("svc-held"); + let completed = + remove_managed_service_record_with(&paths, "svc-held", true, true, unqueryable)?; + let left_after = files_left(&paths, "svc-held"); + let _ = fs::remove_dir_all(root); + + assert!(message.contains("requires --yes"), "{message}"); + assert!(message.contains(&format!("Try: {named}")), "{message}"); + assert_eq!(left_before, 3, "the refusal removed nothing"); + assert!(completed.forced); + assert_eq!(left_after, 0, "the command it named cleared the record"); + // An ordinary removal still names the plain command. + let plain = service_hints::service_action_hint("remove", "svc-held"); + assert!( + !named.eq(&plain) && named.contains("--force"), + "{named} / {plain}" + ); + Ok(()) + } + + /// `prune` never forces, and says what does: a record held only by an + /// unqueryable process, after a stop was requested, is reported with the + /// exact `remove --force --yes` command, and that command, run, clears the + /// record. A record with a process confirmed running keeps the stop advice. + #[test] + fn prune_names_the_forced_removal_for_a_record_held_by_an_unqueryable_process() -> Result<()> { + let (root, paths) = test_paths("services-prune-names-force"); + paths.ensure()?; + held_record(&paths, "svc-held", true)?; + let unqueryable = |_: u32| ProcessProbe::Unqueryable; + + let outcome = prune_managed_service_records_with(&paths, 0, false, true, &unqueryable)?; + let left_after_prune = files_left(&paths, "svc-held"); + let named = service_hints::service_forced_remove_hint("svc-held"); + // Run exactly what prune named. + let removed = + remove_managed_service_record_with(&paths, "svc-held", true, true, unqueryable)?; + let left_after_named = files_left(&paths, "svc-held"); + + held_record(&paths, "svc-running", true)?; + let running = + prune_managed_service_records_with(&paths, 0, true, false, &|_| ProcessProbe::Running)?; + let _ = fs::remove_dir_all(root); + + assert!( + outcome.text.contains(&format!( + "svc-held is starting and process {} it recorded could not be queried - remove it with `{named}`", + std::process::id() + )), + "{}", + outcome.text + ); + assert_eq!(outcome.removed_records, 0, "prune itself never forces"); + assert_eq!(left_after_prune, 3, "and left every file alone"); + assert!(removed.forced); + assert_eq!( + left_after_named, 0, + "the command prune named clears the record" + ); + assert!( + running.text.contains( + "svc-running is starting - stop it first with `rocm services stop svc-running --yes`" + ), + "{}", + running.text + ); + assert!(!running.text.contains("--force"), "{}", running.text); + Ok(()) + } } diff --git a/apps/rocm/src/services_cmd.rs b/apps/rocm/src/services_cmd.rs index b559503110..79c21675f4 100644 --- a/apps/rocm/src/services_cmd.rs +++ b/apps/rocm/src/services_cmd.rs @@ -38,7 +38,7 @@ use crate::service_hints::{ service_action_verb, }; use crate::service_records::{ - prune_managed_service_records, remove_managed_service_record, service_prune_min_age_hours, + prune_managed_service_records, remove_managed_service_record_with, service_prune_min_age_hours, }; use crate::services_text::{ follow_service_logs, render_service_action_result, @@ -74,17 +74,32 @@ pub(crate) fn services(command: Option) -> Result<()> { ServicesCommand::Restart { service_id, yes } => { run_approved_service_action(&paths, "restart_server", &service_id, yes) } - ServicesCommand::Remove { service_id, yes } => { - print!( - "{}", - remove_managed_service_record(&paths, &service_id, yes)? - ); + ServicesCommand::Remove { + service_id, + yes, + force, + } => { + let removed = remove_managed_service_record_with( + &paths, + &service_id, + yes, + force, + rocm_core::probe_process, + )?; + print!("{}", removed.text); + let message = if removed.forced { + format!( + "removed local server record {service_id} (--force: a recorded process could not be queried)" + ) + } else { + format!("removed local server record {service_id}") + }; record_cli_audit_event( &paths, "service", "remove_record", "info", - format!("removed local server record {service_id}"), + message, Some(&service_id), ); Ok(()) diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index 67a3adbb68..fb0e97f29c 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -1072,9 +1072,20 @@ pub(crate) fn render_service_action_result(tool: &str, value: &serde_json::Value /// /// - a process that was signalled and that, or a process under it, outlived /// the deadline (`timed_out`); +/// - a process that is gone, with the separate kill of the processes under it +/// not completed (`descendants_unconfirmed`, Windows): this stop saw the +/// recorded process exit, so it does not say anything outlived a deadline, but +/// one of the processes under it may still be running; +/// - a recorded process that exited by itself before the processes under it +/// could be taken down (`exited_before_tree_kill`, Windows), with no other +/// recorded process confirmed gone to vouch for what was under it: nothing +/// was signalled, and one of them may still be running; /// - a process that was not signalled because its identity could not be -/// verified (`unverified`), with how to look at it. It may have exited -/// meanwhile: the identity read can fail because the process is going away; +/// verified (`unverified`), with how to look at it. There are two routes to +/// it: a recorded start-time that can no longer be read back from a running +/// process, and, on Windows, a PID that cannot be queried at all. It may have +/// exited meanwhile: the identity read can fail because the process is going +/// away; /// - the record changed while the stop ran, typically because a restart /// recorded new processes, so the stop's verdict says nothing about what the /// record names now. @@ -1109,6 +1120,10 @@ fn render_unconfirmed_service_stop(output: &serde_json::Value) -> String { .collect() }; let timed_out = with_outcome(rocm_core::TerminationOutcome::TimedOut.as_str()); + let descendants_unconfirmed = + with_outcome(rocm_core::TerminationOutcome::DescendantsUnconfirmed.as_str()); + let exited_before_tree_kill = + with_outcome(rocm_core::TerminationOutcome::ExitedBeforeTreeKill.as_str()); let unverified = with_outcome(rocm_core::TerminationOutcome::Unverified.as_str()); let mut text = String::new(); @@ -1132,6 +1147,18 @@ fn render_unconfirmed_service_stop(output: &serde_json::Value) -> String { " still running: process {pid} ({role}) or a process under it did not exit after it was signaled" ); } + for (pid, role) in &descendants_unconfirmed { + let _ = writeln!( + text, + " not confirmed: process {pid} ({role}) exited, but a process under it may still be running, because taking the processes under it down did not complete; rocm does not keep this warning, so a later stop or listing will find the process gone and report the stop confirmed" + ); + } + for (pid, role) in &exited_before_tree_kill { + let _ = writeln!( + text, + " not confirmed: process {pid} ({role}) was already gone when the processes under it were to be taken down, and nothing else the server recorded vouches for them, so one of them may still be running; rocm does not keep this warning, so a later stop or listing will find the process gone and report the stop confirmed" + ); + } for (pid, role) in &unverified { let _ = writeln!( text, @@ -1194,11 +1221,24 @@ fn render_unconfirmed_service_stop(output: &serde_json::Value) -> String { /// stop left alone because it could not verify it. Not a `rocm` command: the /// CLI deliberately does not act on that process, so the user decides. /// -/// A Linux command because only Linux can produce that outcome: it needs a -/// recorded start-time that can no longer be read back, and -/// `rocm_core::process_start_ticks` records one only on Linux. +/// Linux and Windows can both produce that outcome, by two routes. On both, a +/// recorded start-time that can no longer be read back from a running process +/// (`rocm_core::process_start_ticks` records one on both). On Windows also, a PID +/// that cannot be queried at all, such as a live process this one may not open, +/// which reads as unverifiable with or without a recorded token. So the command +/// is the one the host has: `ps` on Linux, `tasklist` on Windows. pub(crate) fn process_check_command(pid: u64) -> String { - format!("ps -p {pid} -o pid,args") + process_check_command_for(cfg!(windows), pid) +} + +/// [`process_check_command`] for a chosen platform, so a test reads both forms +/// on every host. +fn process_check_command_for(windows: bool, pid: u64) -> String { + if windows { + format!("tasklist /FI \"PID eq {pid}\"") + } else { + format!("ps -p {pid} -o pid,args") + } } /// A managed service's record exactly as it is on disk, without the engine-state @@ -3262,4 +3302,24 @@ pub(crate) mod tests { })); assert_eq!(answer["status"], "stop_unconfirmed", "{answer}"); } + + /// The check a stop that could not verify a process tells the user to run + /// is a command the host has: `ps` on Linux, `tasklist` on Windows, where a + /// Windows record now carries a creation time that can fail to read back. + /// The wording is pinned here; that the Windows form shows whether a PID is + /// running is what the `service-stop-07` steps rely on, which look a PID up + /// with the same `tasklist /FI "PID eq "` filter. + #[test] + fn the_process_check_command_is_the_one_the_host_has() { + assert_eq!(process_check_command_for(false, 42), "ps -p 42 -o pid,args"); + assert_eq!( + process_check_command_for(true, 42), + "tasklist /FI \"PID eq 42\"" + ); + assert_eq!( + process_check_command(42), + process_check_command_for(cfg!(windows), 42), + "production picks by the host it runs on" + ); + } } diff --git a/apps/rocmd/src/service.rs b/apps/rocmd/src/service.rs index 41e7fcf844..c0579a668c 100644 --- a/apps/rocmd/src/service.rs +++ b/apps/rocmd/src/service.rs @@ -7,12 +7,9 @@ use rocm_core::{AppPaths, ManagedServiceRecord}; use rocm_engine_protocol::{EngineMethod, StopRequest, StopResponse}; use serde_json::Value; use serde_json::json; -#[cfg(windows)] -use std::process::{Command as ProcessCommand, Stdio}; -use std::time::Duration; pub(crate) fn stop_managed_service(paths: &AppPaths, service_id: &str) -> Result { - stop_managed_service_with(paths, service_id, terminate_recorded_pid) + stop_managed_service_with(paths, service_id, rocm_core::terminate_recorded_pid) } fn find_managed_service(paths: &AppPaths, service_id: &str) -> Result { @@ -23,8 +20,8 @@ fn find_managed_service(paths: &AppPaths, service_id: &str) -> Result Result { let mut stopping = find_managed_service(paths, service_id)?; // Record the request before the stop starts, not only after it ends. A stop - // can take up to `MANAGED_STOP_GRACE` twice per recorded PID, and for that + // can take up to `rocm_core::MANAGED_STOP_GRACE` twice per recorded PID, and for that // whole window the service still reads `ready`/`running` while its endpoint // stops answering — exactly what the daemon's `server-recover` watcher // restarts. The marker is what that watcher skips (`stop_requested`), so it @@ -73,32 +70,27 @@ pub(crate) fn stop_managed_service_via( // container-backed engine (`aim`) this is the only thing that stops the // container, since its recorded PIDs are only the supervisor that follows it. let engine_stop = (stopping.engine != "lemonade").then(|| engine_stop(paths, &stopping)); - let stops = terminate_recorded_service_pids(&stopping, terminate); - let pids_where = |keep: fn(&RecordedPidStop) -> bool| -> Vec { + let stops = rocm_core::terminate_recorded_service_pids(&stopping, terminate); + let pids_where = |keep: fn(rocm_core::RecordedPidStop) -> bool| -> Vec { stops .iter() - .filter(|stop| keep(stop)) - .map(|s| s.pid) + .copied() + .filter(|stop| keep(*stop)) + .map(|stop| stop.pid) .collect() }; - let signaled_pids = pids_where(|stop| stop.signaled); - let force_signaled_pids = pids_where(|stop| stop.forced); + let signaled_pids = pids_where(rocm_core::RecordedPidStop::signaled); + let force_signaled_pids = pids_where(rocm_core::RecordedPidStop::forced); // Everything that was deliberately not signalled, which is no longer one // kind of thing: the stopping process itself; a PID that was already gone; // a PID that is live but provably belongs to somebody else now; and a live // PID whose identity could not be read at all, which may still be the engine // and may still hold the device. Only `pid_outcomes` separates them, and // only `stopped` reports that the last of those is not a completed stop. - let skipped_pids = pids_where(|stop| !stop.signaled); + let skipped_pids = pids_where(|stop| !stop.signaled()); let pid_outcomes = stops .iter() - .map(|stop| { - json!({ - "pid": stop.pid, - "role": stop.role, - "outcome": stop.outcome, - }) - }) + .map(|stop| stop.report_entry()) .collect::>(); // Re-read the record rather than write back the copy loaded before the // stop: `write` replaces the whole file, and other writers — a supervisor @@ -113,8 +105,8 @@ pub(crate) fn stop_managed_service_via( // would each be false. let engine_stop_confirms = !rocm_core::engine_stop_decides_confirmation(&stopping.engine) || engine_stop.as_ref().is_some_and(Result::is_ok); - let all_stopped = stops.iter().all(|stop| stop.stopped) - && stopping.names_same_processes(&record) + let all_stopped = stops.iter().all(|stop| stop.stopped()) + && record.is_covered_by_stop_of(&stopping) && engine_stop_confirms; // A PID whose identity cannot be read at all — `/proc//stat` // unreadable while `kill(pid, 0)` still answers: another uid, `hidepid`, an @@ -135,11 +127,9 @@ pub(crate) fn stop_managed_service_via( record.stop_requested_unix_ms = None; // Recorded PIDs go stale the instant their processes exit, and a stale // PID is exactly what the identity check exists to catch — so do not - // leave one behind for the next stop to find. - record.supervisor_pid = 0; - record.supervisor_start_ticks = None; - record.engine_pid = None; - record.engine_start_ticks = None; + // leave one behind for the next stop to find. The same call + // `rocm services stop` makes on its confirmed path. + record.clear_recorded_processes(); } else { // The recorded PIDs stay: an unconfirmed survivor is still the // service's, and the record is what names it. @@ -208,179 +198,6 @@ pub(crate) fn stop_report_confirmed(report: &Value) -> bool { report.get("stopped").and_then(Value::as_bool) == Some(true) } -/// How long a stop waits for a recorded process tree to exit on its own before -/// escalating to a forced kill. Matches `rocm services stop`, which terminates -/// the same records. -const MANAGED_STOP_GRACE: Duration = Duration::from_secs(10); - -/// What a stop did about one PID recorded in a service manifest. -struct RecordedPidStop { - pid: u32, - /// Which recorded PID this is: `supervisor`, `engine`, or - /// `supervisor_and_engine` when the record holds the same PID for both. - role: &'static str, - /// Stable label for what happened, from - /// [`rocm_core::TerminationOutcome::as_str`], plus `self` for the stopping - /// process's own PID. - outcome: &'static str, - signaled: bool, - forced: bool, - /// Whether the recorded process is confirmed to be no longer running. - stopped: bool, -} - -/// Terminate the processes a service manifest records, verifying identity first. -/// -/// PIDs are recycled, so a persisted PID is not by itself evidence that the -/// recorded process is still the one holding it. Each PID is therefore paired -/// with **its own** start-time token — `supervisor_start_ticks` with -/// `supervisor_pid`, `engine_start_ticks` with `engine_pid` — and routed through -/// [`rocm_core::terminate_verified`], which signals nothing when the recorded -/// identity is refuted. `rocm services stop` reads the same records through the -/// same primitive, so neither command can decide differently about one. -/// -/// Descendants are reached from the *verified* root rather than expanded out of -/// a live process listing keyed on a PID nobody checked — which, given a stale -/// root, enumerated a stranger's children and signalled those too. Where the -/// kernel start-time is readable, `terminate_verified` additionally binds each -/// child to its own identity before any forced kill; see -/// [`terminate_recorded_pid`] for what Windows can and cannot do here. -/// -/// The stopping process's own PID is reported but never signalled, and does not -/// count towards "the recorded service is gone": `rocmd` records itself as the -/// supervisor of the services it launches, and it is the launcher, not the -/// service. -/// -/// `terminate` performs the verified termination of one recorded PID; -/// production passes [`terminate_recorded_pid`]. -fn terminate_recorded_service_pids( - record: &ManagedServiceRecord, - terminate: impl Fn(&rocm_core::ProcessIdentity) -> rocm_core::TerminationOutcome, -) -> Vec { - // Build the (pid, role, own-token) work list, de-duplicating on PID and - // preferring an entry that carries a verifiable start-time. The two PIDs can - // coincide — a launcher that is also the server — and a token must never be - // read across from the PID it does not belong to. - let mut entries: Vec<(u32, &'static str, Option)> = Vec::new(); - for (pid, role, ticks) in [ - ( - Some(record.supervisor_pid), - "supervisor", - record.supervisor_start_ticks, - ), - (record.engine_pid, "engine", record.engine_start_ticks), - ] { - let Some(pid) = pid.filter(|pid| *pid != 0) else { - continue; - }; - if let Some(existing) = entries.iter_mut().find(|(seen, _, _)| *seen == pid) { - // Only the engine entry can land here, aliasing the supervisor: the - // one PID is both, and is reported as both. - existing.1 = "supervisor_and_engine"; - if existing.2.is_none() { - existing.2 = ticks; - } - } else { - entries.push((pid, role, ticks)); - } - } - - entries - .into_iter() - .map(|(pid, role, ticks)| { - if pid == std::process::id() { - return RecordedPidStop { - pid, - role, - outcome: "self", - signaled: false, - forced: false, - stopped: true, - }; - } - let outcome = terminate(&rocm_core::ProcessIdentity::new(pid, ticks)); - RecordedPidStop { - pid, - role, - outcome: outcome.as_str(), - // TimedOut belongs with the signalled: the signal went out, the - // exit was just never observed. `stopped` carries that truth. - signaled: matches!( - outcome, - rocm_core::TerminationOutcome::Graceful - | rocm_core::TerminationOutcome::Forced - | rocm_core::TerminationOutcome::TimedOut - ), - // "The stop escalated to a forced kill", not "the forced kill - // worked": a forced stop only reaches `TimedOut` by way of that - // escalation, and `stopped` is what says whether the process - // actually went away. - forced: matches!( - outcome, - rocm_core::TerminationOutcome::Forced | rocm_core::TerminationOutcome::TimedOut - ), - stopped: outcome.stopped(), - } - }) - .collect() -} - -/// Terminate one recorded process after verifying its identity. -/// -/// The stop is forced: its contract is definitive, so the command must not -/// report success while an engine worker still holds the GPU. Only a process -/// whose recorded identity is confirmed is ever reached by it. -fn terminate_recorded_pid(identity: &rocm_core::ProcessIdentity) -> rocm_core::TerminationOutcome { - // Windows needs the descendants taken separately. `rocm_core` has no way to - // walk a process tree there — `process_tree_pids` returns just the root — and - // the engine runs one level below the recorded PID, because `rocmd` launches - // it through an `__engine-serve-http` process. A root-only kill would leave - // the engine running and still holding the device. - // - // Identity verification does not apply on Windows. `process_start_ticks` - // has no `/proc` to read there and returns `None`, so no recorded identity - // can be refuted and the gate below only ever turns away a PID that is not - // running at all. A recorded PID that has been recycled by an unrelated - // process still passes it, and `taskkill /T /F` then takes that process and - // its whole subtree: the stale-PID hazard is unchanged on Windows, exactly - // as it was before this gate existed. Closing it needs `rocm_core` to read a - // Windows process creation time. - #[cfg(windows)] - if matches!( - rocm_core::identity_state(identity), - rocm_core::IdentityState::Matches - ) { - // Taken while the root is still alive: `taskkill /T` resolves children - // through the live parent, so it cannot reach them once the root is gone. - let _ = ProcessCommand::new("taskkill") - .args(["/PID", &identity.pid.to_string(), "/T", "/F"]) - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status(); - // `terminate_verified` still owns the bounded wait and the verdict, but - // a root the tree kill already took now reads back as `AlreadyGone` — - // which would deny a stop this command did perform. - let outcome = rocm_core::terminate_verified( - identity, - rocm_core::KillScope::Tree, - MANAGED_STOP_GRACE, - true, - ); - return if matches!(outcome, rocm_core::TerminationOutcome::AlreadyGone) { - rocm_core::TerminationOutcome::Forced - } else { - outcome - }; - } - rocm_core::terminate_verified( - identity, - rocm_core::KillScope::Tree, - MANAGED_STOP_GRACE, - true, - ) -} - #[cfg(test)] mod tests { use super::*; @@ -510,10 +327,11 @@ mod tests { /// a different process: the recorded service already exited and its PID was /// recycled. The stop must refuse it rather than signal a stranger. /// - /// Linux-only: a refutable identity needs a readable start-time, and - /// `rocm_core::process_start_ticks` reads it from `/proc`. On Windows it is - /// always `None`, so no identity can be refuted there and the state this - /// asserts on cannot be constructed. + /// Linux-only: it stages the bystander with `sleep` and offsets its token + /// read from `/proc`. The same refusal on Windows, where the token is the + /// process creation time, is asserted in `rocm-core`'s `proc_lifecycle` + /// tests (`terminate_recorded_pid_refuses_a_pid_whose_recorded_identity_no_longer_matches` + /// and its Windows-only neighbour) and by scenario `service-stop-08`. #[cfg(target_os = "linux")] #[test] fn stop_managed_service_refuses_a_pid_whose_recorded_identity_no_longer_matches() -> Result<()> @@ -1037,6 +855,137 @@ mod tests { Ok(()) } + /// A `rocm` listing that sees every process gone completes the pending stop + /// and persists it (marker and key cleared, recorded processes forgotten, + /// `stopped`), possibly before this stop reads the record again. That record + /// names no process, which is not a restart: the stop is still confirmed and + /// the marker is not put back. The write below is exactly the one the + /// listing's cleanup makes (it lives in the `rocm` binary, which this crate + /// cannot call; `rocm` has the test that runs the real refresh). + #[test] + fn a_stop_completed_by_a_concurrent_listing_is_still_confirmed() -> Result<()> { + let (root, paths) = temp_app_paths("stop-listing-completed-it"); + paths.ensure()?; + let service_id = "svc-stop-listing-completed-it"; + let key_path = seed_keyed_service(&paths, service_id, 11455)?; + + let result = stop_managed_service_with(&paths, service_id, |_| { + let mut on_disk = load_service_record(&paths, service_id).expect("record"); + on_disk.clear_recorded_processes(); + on_disk.stop_requested_unix_ms = None; + on_disk.status = "stopped".to_owned(); + on_disk.write().expect("concurrent listing write"); + rocm_core::TerminationOutcome::Graceful + }); + let key_removed = !key_path.exists(); + let reloaded = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let value = result?; + assert_eq!( + value.get("stopped").and_then(Value::as_bool), + Some(true), + "{value}" + ); + let reloaded = reloaded?; + assert_eq!(reloaded.status, "stopped"); + assert_eq!( + reloaded.stop_requested_unix_ms, None, + "a confirmed stop does not put the marker back" + ); + assert!(key_removed); + Ok(()) + } + + /// The listing's cleanup of an empty record (see above) leaves a `starting` or + /// `failed` record at its status with the marker gone and the marker's stamp + /// kept: that is the stop's own marker consumed, not a relaunch, so the stop + /// is confirmed. A record naming no process has no PID to terminate, so the + /// per-PID callback never runs; the engine `Stop` does (for every engine but + /// lemonade), and that is where the listing's write is made, with an assertion + /// that it happened. `rocm` has the test that runs the real refresh. + #[test] + fn a_stop_of_an_empty_record_whose_marker_a_listing_consumed_is_still_confirmed() -> Result<()> + { + for status in ["starting", "failed"] { + let (root, paths) = temp_app_paths(&format!("stop-empty-listing-{status}")); + paths.ensure()?; + let service_id = "svc-stop-empty-listing"; + let key_path = seed_keyed_service(&paths, service_id, 11457)?; + let mut empty = load_service_record(&paths, service_id)?; + empty.clear_recorded_processes(); + status.clone_into(&mut empty.status); + empty.write()?; + let interleaved = std::cell::Cell::new(false); + + let result = stop_managed_service_via( + &paths, + service_id, + |_| unreachable!("a record naming no process has no PID to terminate"), + |_, _| { + let mut on_disk = load_service_record(&paths, service_id).expect("record"); + assert!(on_disk.stop_requested_unix_ms.is_some(), "{status}"); + on_disk.settled_stop_unix_ms = on_disk.stop_requested_unix_ms.take(); + on_disk.write().expect("concurrent listing write"); + interleaved.set(true); + Ok(StopResponse { + stopped: true, + graceful: true, + }) + }, + ); + let key_removed = !key_path.exists(); + let reloaded = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + assert!(interleaved.get(), "{status}: premise: the listing ran"); + let value = result?; + assert_eq!( + value.get("stopped").and_then(Value::as_bool), + Some(true), + "{status}: {value}" + ); + let reloaded = reloaded?; + assert_eq!(reloaded.status, "stopped", "{status}"); + assert_eq!(reloaded.stop_requested_unix_ms, None, "{status}"); + assert!(key_removed, "{status}"); + } + Ok(()) + } + + /// A restart that has begun launching but recorded no process yet reads + /// `starting` with nothing named: not the stop completing, so the stop does + /// not claim it or drop the key. + #[test] + fn a_stop_does_not_claim_a_restart_that_has_recorded_no_process_yet() -> Result<()> { + let (root, paths) = temp_app_paths("stop-restart-in-flight"); + paths.ensure()?; + let service_id = "svc-stop-restart-in-flight"; + let key_path = seed_keyed_service(&paths, service_id, 11456)?; + + let result = stop_managed_service_with(&paths, service_id, |_| { + let mut on_disk = load_service_record(&paths, service_id).expect("record"); + on_disk.clear_recorded_processes(); + on_disk.stop_requested_unix_ms = None; + on_disk.status = "starting".to_owned(); + on_disk.write().expect("concurrent restart write"); + rocm_core::TerminationOutcome::Graceful + }); + let key_kept = key_path.exists(); + let reloaded = load_service_record(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let value = result?; + assert_eq!( + value.get("stopped").and_then(Value::as_bool), + Some(false), + "{value}" + ); + assert_eq!(reloaded?.status, "starting"); + assert!(key_kept, "the restarting service's key must be kept"); + Ok(()) + } + /// A launcher that is also the server is recorded under both roles. The /// stop handles that PID once, and must report it under both. #[test] @@ -1080,4 +1029,289 @@ mod tests { ); Ok(()) } + + // ---- This module's stop, through the routine it shares with `rocm`. ---- + // + // The routine itself (visiting order, start-time pairing, the `self` row, + // the per-outcome classification, the Windows tree-kill step) is tested in + // `rocm-core`'s `proc_lifecycle.rs`. These check that this caller reaches it + // and reports what it returns. The termination is a stand-in throughout, so + // no process is signalled and the tests behave the same on every host. + + const SUPERVISOR_PID: u32 = 999_999_998; + const ENGINE_PID: u32 = 999_999_997; + + /// Plant a record naming `supervisor` and `engine`, with the given start-time + /// tokens, and return the paths it lives under. + fn plant_record_naming( + name: &str, + supervisor: u32, + supervisor_ticks: Option, + engine: Option, + engine_ticks: Option, + ) -> Result<(std::path::PathBuf, AppPaths, &'static str)> { + let (root, paths) = temp_app_paths(name); + paths.ensure()?; + let service_id = "svc-stop-through-the-shared-routine"; + let mut record = ManagedServiceRecord::new( + &paths, + service_id, + "vllm", + "m", + "m", + "127.0.0.1", + 9, + "managed", + supervisor, + None, + None, + None, + ); + record.supervisor_start_ticks = supervisor_ticks; + record.engine_pid = engine; + record.engine_start_ticks = engine_ticks; + record.status = "ready".to_owned(); + record.write()?; + Ok((root, paths, service_id)) + } + + fn pids_listed(report: &Value, key: &str) -> Vec { + report[key] + .as_array() + .unwrap_or_else(|| panic!("no `{key}` in {report}")) + .iter() + .filter_map(Value::as_u64) + .collect() + } + + #[test] + fn a_stop_terminates_the_supervisor_then_the_engine_each_with_its_own_start_time() -> Result<()> + { + let (root, paths, service_id) = plant_record_naming( + "stop-order", + SUPERVISOR_PID, + Some(11), + Some(ENGINE_PID), + Some(22), + )?; + let seen = std::cell::RefCell::new(Vec::new()); + + let report = stop_managed_service_with(&paths, service_id, |identity| { + seen.borrow_mut().push(*identity); + rocm_core::TerminationOutcome::Graceful + }); + fs::remove_dir_all(root).ok(); + + let report = report?; + assert_eq!( + seen.into_inner(), + vec![ + rocm_core::ProcessIdentity::new(SUPERVISOR_PID, Some(11)), + rocm_core::ProcessIdentity::new(ENGINE_PID, Some(22)), + ] + ); + assert_eq!( + report["pid_outcomes"], + json!([ + {"pid": SUPERVISOR_PID, "role": "supervisor", "outcome": "graceful"}, + {"pid": ENGINE_PID, "role": "engine", "outcome": "graceful"}, + ]) + ); + assert_eq!( + pids_listed(&report, "signaled_pids"), + vec![u64::from(SUPERVISOR_PID), u64::from(ENGINE_PID)] + ); + Ok(()) + } + + /// The stopping process's own PID is reported as `self`, never handed to the + /// termination, and does not stand in the way of a confirmed stop. + #[test] + fn a_stop_reports_its_own_pid_as_self_and_skips_it() -> Result<()> { + let own = std::process::id(); + let (root, paths, service_id) = + plant_record_naming("stop-own-pid", own, None, Some(ENGINE_PID), None)?; + let seen = std::cell::RefCell::new(Vec::new()); + + let report = stop_managed_service_with(&paths, service_id, |identity| { + seen.borrow_mut().push(identity.pid); + rocm_core::TerminationOutcome::Graceful + }); + fs::remove_dir_all(root).ok(); + + let report = report?; + assert_eq!(seen.into_inner(), vec![ENGINE_PID]); + assert_eq!( + report["pid_outcomes"][0], + json!({"pid": own, "role": "supervisor", "outcome": "self"}) + ); + assert_eq!(pids_listed(&report, "skipped_pids"), vec![u64::from(own)]); + assert_eq!( + pids_listed(&report, "signaled_pids"), + vec![u64::from(ENGINE_PID)] + ); + assert_eq!(report["stopped"], true); + Ok(()) + } + + #[test] + fn a_stop_sorts_each_outcome_into_the_signaled_forced_and_skipped_lists() -> Result<()> { + use rocm_core::TerminationOutcome::{ + AlreadyGone, Forced, Graceful, IdentityMismatch, TimedOut, Unverified, + }; + // (outcome, signaled, forced, skipped, stopped) + let expected = [ + (AlreadyGone, false, false, true, true), + (IdentityMismatch, false, false, true, true), + (Unverified, false, false, true, false), + (Graceful, true, false, false, true), + (Forced, true, true, false, true), + (TimedOut, true, true, false, false), + ]; + for (outcome, signaled, forced, skipped, stopped) in expected { + let (root, paths, service_id) = + plant_record_naming("stop-classes", SUPERVISOR_PID, None, None, None)?; + + let report = stop_managed_service_with(&paths, service_id, |_| outcome); + fs::remove_dir_all(root).ok(); + + let report = report?; + let listed = |key| !pids_listed(&report, key).is_empty(); + assert_eq!( + ( + listed("signaled_pids"), + listed("force_signaled_pids"), + listed("skipped_pids"), + report["stopped"].as_bool() + ), + (signaled, forced, skipped, Some(stopped)), + "{outcome:?}: {report}" + ); + } + Ok(()) + } + + #[test] + fn the_production_stop_reports_an_absent_pid_as_already_gone() -> Result<()> { + let (root, paths, service_id) = + plant_record_naming("stop-production-absent", SUPERVISOR_PID, None, None, None)?; + + let report = stop_managed_service(&paths, service_id); + fs::remove_dir_all(root).ok(); + + let report = report?; + assert_eq!(report["pid_outcomes"][0]["outcome"], "already_gone"); + assert_eq!(report["stopped"], true); + Ok(()) + } + + /// A second copy of the shared routine, or of its grace constant, is how the + /// two binaries diverged in the first place; the shared one lives in + /// `rocm-core`'s `proc_lifecycle.rs`. The needles are assembled so that this + /// test does not match itself. + #[test] + fn this_module_carries_no_private_copy_of_the_shared_stop_routine() { + let source = include_str!("service.rs"); + for needle in [ + ["const ", "MANAGED_STOP_GRACE"].concat(), + ["fn ", "terminate_recorded_service_pids"].concat(), + ["fn ", "terminate_recorded_pid"].concat(), + ["Command::new(\"task", "kill\")"].concat(), + ] { + assert!( + !source.contains(&needle), + "`{needle}` is defined in `rocm-core`, not here" + ); + } + } + + /// A root that exited by itself before its tree could be taken down confirms + /// the stop only through the record's other roles, as this caller folds + /// them: the supervisor's own row is not enough, and the record, its PIDs, + /// its stop marker and its key are kept unless the engine role is + /// confirmed gone too. + #[test] + fn a_root_that_exited_alone_confirms_the_stop_only_when_the_engine_role_does() -> Result<()> { + use rocm_core::TerminationOutcome::{ExitedBeforeTreeKill, Forced, TimedOut}; + + for (engine, confirmed) in [(Forced, true), (TimedOut, false)] { + let (root, paths, service_id) = plant_record_naming( + "stop-root-exited-alone", + SUPERVISOR_PID, + None, + Some(ENGINE_PID), + None, + )?; + let key = rocm_engine_protocol::endpoint_key_file_path(&paths, service_id); + fs::create_dir_all(paths.services_dir())?; + fs::write(&key, "secret-key")?; + + let report = stop_managed_service_with(&paths, service_id, |identity| { + if identity.pid == SUPERVISOR_PID { + ExitedBeforeTreeKill + } else { + engine + } + })?; + let record = find_managed_service(&paths, service_id)?; + let key_kept = key.exists(); + fs::remove_dir_all(root).ok(); + + assert_eq!( + stop_report_confirmed(&report), + confirmed, + "{engine:?}: {report}" + ); + assert_eq!( + report["pid_outcomes"][0]["outcome"], + if confirmed { + "already_gone" + } else { + "exited_before_tree_kill" + }, + "{engine:?}: {report}" + ); + if confirmed { + assert_eq!(record.status, "stopped"); + assert_eq!( + record.supervisor_pid, 0, + "the confirmed stop clears the PIDs" + ); + assert!(!key_kept, "and the key"); + } else { + assert_eq!(record.status, "ready", "{engine:?}"); + assert_eq!(record.supervisor_pid, SUPERVISOR_PID, "the PIDs are kept"); + assert!( + record.stop_requested_unix_ms.is_some(), + "the marker is kept" + ); + assert!(key_kept, "the key is kept"); + } + } + Ok(()) + } + + /// A root that exited by itself with no other role recorded cannot be + /// vouched for, so this caller does not clear the record on its say-so. + #[test] + fn a_lone_root_that_exited_alone_leaves_the_stop_unconfirmed() -> Result<()> { + let (root, paths, service_id) = plant_record_naming( + "stop-lone-root-exited-alone", + SUPERVISOR_PID, + None, + None, + None, + )?; + + let report = stop_managed_service_with(&paths, service_id, |_| { + rocm_core::TerminationOutcome::ExitedBeforeTreeKill + })?; + let record = find_managed_service(&paths, service_id)?; + fs::remove_dir_all(root).ok(); + + assert!(!stop_report_confirmed(&report), "{report}"); + assert_eq!(record.supervisor_pid, SUPERVISOR_PID); + assert!(record.stop_requested_unix_ms.is_some()); + Ok(()) + } } diff --git a/apps/rocmd/src/supervise.rs b/apps/rocmd/src/supervise.rs index a19f9d680b..bfb5092ada 100644 --- a/apps/rocmd/src/supervise.rs +++ b/apps/rocmd/src/supervise.rs @@ -1315,8 +1315,9 @@ mod tests { /// calls `std::process::exit`, which inside a test binary would take the /// whole suite with it. /// - /// Linux-only: `rocm_core::process_start_ticks` reads `/proc` and returns - /// `None` everywhere else, so on other platforms there is no token to write. + /// Linux-only here: `rocm_core::process_start_ticks` gives a token on Linux + /// and on Windows (the creation time) and `None` elsewhere, and this test has + /// not been taken to Windows. #[cfg(target_os = "linux")] #[test] fn supervise_service_persists_the_supervisor_identity_token() -> Result<()> { diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index ae46f662f8..a8181620d0 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -184,14 +184,16 @@ pub use paths::{ AppPaths, CACHE_DIR_ENV, CONFIG_DIR_ENV, DATA_DIR_ENV, ENGINE_ENVS_ROOT_ENV, engine_plugin_dirs, }; pub use proc_lifecycle::{ - IdentityState, KillScope, ProcessIdentity, TerminationOutcome, identity_state, - process_start_ticks, terminate_verified, + IdentityState, KillScope, MANAGED_STOP_GRACE, ProcessIdentity, RecordedPidOutcome, + RecordedPidStop, TerminationOutcome, identity_state, process_start_ticks, + terminate_recorded_pid, terminate_recorded_pid_within, terminate_recorded_service_pids, + terminate_verified, }; pub use process::{ - FileLock, LoopbackPortLease, MANAGED_LAUNCH_LOCK_TIMEOUT, detach_command_session, + FileLock, LoopbackPortLease, MANAGED_LAUNCH_LOCK_TIMEOUT, ProcessProbe, detach_command_session, is_canonical_loopback_host, launch_port_argument, lease_loopback_port, - lease_loopback_port_or_bind_error, process_is_running, terminate_process, - terminate_process_tree, + lease_loopback_port_or_bind_error, probe_process, process_is_running, terminate_process, + terminate_process_tree, windows_system_tool, }; #[cfg(windows)] pub use process::{ diff --git a/crates/rocm-core/src/proc_lifecycle.rs b/crates/rocm-core/src/proc_lifecycle.rs index 57da4bb1fe..dbe18a856e 100644 --- a/crates/rocm-core/src/proc_lifecycle.rs +++ b/crates/rocm-core/src/proc_lifecycle.rs @@ -15,6 +15,8 @@ use std::time::{Duration, Instant}; +use crate::process::ProcessProbe; + /// How often the bounded waits poll for the target's exit. const POLL_INTERVAL: Duration = Duration::from_millis(25); @@ -32,8 +34,10 @@ pub enum KillScope { /// A spawned process identified in a way that is robust to PID recycling. /// /// `start_ticks` is the kernel start-time of the process (clock ticks since -/// boot). It is `None` on platforms without `/proc`, where identity cannot be -/// verified and callers fall back to best-effort behaviour. +/// boot) on Linux, and its creation time as a `FILETIME` on Windows. It is +/// `None` on other platforms, and for a record written before Windows recorded +/// one, where identity cannot be verified and callers fall back to best-effort +/// behaviour. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct ProcessIdentity { pub pid: u32, @@ -69,8 +73,10 @@ pub enum IdentityState { /// was recycled. Recycled, /// The PID is live and an identity was recorded, but the current start-time - /// cannot be read, so it can be neither confirmed nor refuted. The process - /// must not be signalled, and it must not be reported as stopped. + /// cannot be read, so it can be neither confirmed nor refuted; or the PID + /// cannot be queried at all (on Windows, a live process this one may not + /// open), so it cannot be told from a process that is gone. The process must + /// not be signalled, and it must not be reported as stopped. Indeterminate, /// The PID is not live (or is a not-yet-reaped zombie). Gone, @@ -85,10 +91,61 @@ pub enum IdentityState { /// [`IdentityState::Matches`]. #[must_use] pub fn identity_state(id: &ProcessIdentity) -> IdentityState { - if !crate::process_is_running(id.pid) || process_has_exited(id.pid) { - return IdentityState::Gone; + identity_state_with( + id, + crate::process::probe_process, + process_has_exited, + process_start_ticks, + ) +} + +/// [`identity_state`] with its three system reads supplied by the caller: whether +/// the PID can be queried and is running, whether it is a zombie, and its +/// current start-time token. The liveness gate in front of every stop, so a test +/// drives each answer a read can give on every host. +/// +/// A PID that cannot be queried is [`IdentityState::Indeterminate`], not +/// [`IdentityState::Gone`]: a failure to look is not an exit, and reading it as +/// one would let a stop report a live process as stopped. That is one of the two +/// routes to [`IdentityState::Indeterminate`] and so to an `unverified` stop; the +/// other is a recorded token that cannot be read back from a running process +/// ([`classify_live_identity`]). +/// +/// A token that was recorded and can still be read, and that differs, outranks +/// the failed query: it proves a different process holds the PID, so the +/// recorded one is gone. On Windows this narrows to the case where the process +/// opens but the exit-code query fails: the token and the probe both open the +/// PID with `PROCESS_QUERY_LIMITED_INFORMATION`, so a process that cannot be +/// opened has no readable token either, and stays [`IdentityState::Indeterminate`]. +fn identity_state_with( + id: &ProcessIdentity, + probe: impl Fn(u32) -> ProcessProbe, + has_exited: impl Fn(u32) -> bool, + start_ticks: impl Fn(u32) -> Option, +) -> IdentityState { + match probe(id.pid) { + ProcessProbe::Gone => IdentityState::Gone, + ProcessProbe::Unqueryable => match (id.start_ticks, start_ticks(id.pid)) { + (Some(recorded), Some(live)) if recorded != live => IdentityState::Recycled, + _ => IdentityState::Indeterminate, + }, + ProcessProbe::Running if has_exited(id.pid) => IdentityState::Gone, + ProcessProbe::Running => classify_live_identity(id.start_ticks, start_ticks(id.pid)), } - match (id.start_ticks, process_start_ticks(id.pid)) { +} + +/// The identity verdict for a PID that is running: what its recorded token +/// (`recorded`) says about the token read from the live process now (`live`). +/// +/// The one decision every stop path turns on, apart from the system reads, so +/// a test covers each case on every host. A process is refused only when a +/// token was recorded and the live one is provably different. A record with no +/// token (an older record, or one written where no token could be read) cannot +/// be refuted and is a match whatever the live process reads, so its stop and +/// its tree kill go ahead as they always did. A recorded token that cannot be +/// confirmed now is neither signalled nor reported stopped. +const fn classify_live_identity(recorded: Option, live: Option) -> IdentityState { + match (recorded, live) { (Some(expected), Some(actual)) => { if expected == actual { IdentityState::Matches @@ -104,6 +161,18 @@ pub fn identity_state(id: &ProcessIdentity) -> IdentityState { } } +/// The verdict that ends a termination before anything is signalled, when the +/// recorded identity says not to: `None` for a process that matches and so goes +/// on to be stopped. +const fn refusal_before_signalling(state: IdentityState) -> Option { + match state { + IdentityState::Gone => Some(TerminationOutcome::AlreadyGone), + IdentityState::Recycled => Some(TerminationOutcome::IdentityMismatch), + IdentityState::Indeterminate => Some(TerminationOutcome::Unverified), + IdentityState::Matches => None, + } +} + /// Whether `state` means the recorded process is definitively no longer running. /// /// [`IdentityState::Indeterminate`] is intentionally excluded: a process we can @@ -120,6 +189,15 @@ pub enum TerminationOutcome { AlreadyGone, /// The PID now belongs to a different process (recycled), so the recorded /// process has already exited; nothing was signalled. + /// + /// Counted as stopped, on every platform, because a token that is recorded + /// and readable and differs is positive evidence of a different process. The + /// engine's own token comes from the same [`process_start_ticks`] that reads + /// it back here, taken for that PID's process when it was spawned or reported + /// (a creation time does not change over a process's life), so a mismatch for + /// the real engine is not an expected outcome. It stays visible as + /// `identity_mismatch` in the report rather than as a signal sent to a + /// process nobody could prove was the server's. IdentityMismatch, /// The recorded PID is live but its identity could not be confirmed, so it /// was deliberately left untouched and its state is unknown. @@ -130,16 +208,42 @@ pub enum TerminationOutcome { Forced, /// At least one signalled process was still alive after the bounded deadline. TimedOut, + /// The recorded process itself is gone (this stop's own wait saw it exit, or + /// it was already gone), but the separate kill of the processes under it did + /// not complete (on Windows, a `taskkill` that failed, stalled, or found no + /// such process while the root was still running), so one of them may still + /// be running and holding the device. Deliberately not + /// [`TimedOut`](Self::TimedOut): the root did not outlive any deadline. + DescendantsUnconfirmed, + /// The recorded process was gone by the time the separate kill of the + /// processes under it ran (on Windows, `taskkill` found no such process and + /// the root's re-read agrees): it exited by itself after the identity check. + /// `taskkill /T` reaches children only through a live parent, and nothing + /// recorded what was under the root, so this stop cannot say whether a + /// process it left running is still there. Not a stop on its own; the + /// aggregate over the record's processes confirms it when another recorded + /// process is confirmed gone ([`terminate_recorded_service_pids`]). + ExitedBeforeTreeKill, } impl TerminationOutcome { /// Is the recorded service confirmed to be no longer running as a result? /// - /// `false` for [`TimedOut`](Self::TimedOut) (still alive) and - /// [`Unverified`](Self::Unverified) (could not be confirmed either way). + /// `false` for [`TimedOut`](Self::TimedOut) (still alive), + /// [`Unverified`](Self::Unverified) (could not be confirmed either way) and + /// [`DescendantsUnconfirmed`](Self::DescendantsUnconfirmed) (the root is gone + /// but a process under it may not be) and + /// [`ExitedBeforeTreeKill`](Self::ExitedBeforeTreeKill) (the root is gone and + /// nothing says what was under it). #[must_use] pub const fn stopped(self) -> bool { - !matches!(self, Self::TimedOut | Self::Unverified) + !matches!( + self, + Self::TimedOut + | Self::Unverified + | Self::DescendantsUnconfirmed + | Self::ExitedBeforeTreeKill + ) } /// Did the process stop without needing a forced kill (or was it already @@ -162,6 +266,8 @@ impl TerminationOutcome { Self::Graceful => "graceful", Self::Forced => "forced", Self::TimedOut => "timed_out", + Self::DescendantsUnconfirmed => "descendants_unconfirmed", + Self::ExitedBeforeTreeKill => "exited_before_tree_kill", } } } @@ -194,11 +300,8 @@ pub fn terminate_verified( grace: Duration, force: bool, ) -> TerminationOutcome { - match identity_state(id) { - IdentityState::Gone => return TerminationOutcome::AlreadyGone, - IdentityState::Recycled => return TerminationOutcome::IdentityMismatch, - IdentityState::Indeterminate => return TerminationOutcome::Unverified, - IdentityState::Matches => {} + if let Some(verdict) = refusal_before_signalling(identity_state(id)) { + return verdict; } let tree = matches!(scope, KillScope::Tree); @@ -261,6 +364,566 @@ pub fn terminate_verified( } } +/// How long a managed stop waits on a recorded process before it escalates. +/// +/// Each recorded process, tree included, gets this long to exit after `SIGTERM` +/// before the stop escalates to `SIGKILL`, and then this long again before it +/// reports a timeout. The one value behind `rocm services stop` and `rocmd`'s +/// `stop_server`: both stop the same records, so neither may wait for a +/// different time. +pub const MANAGED_STOP_GRACE: Duration = Duration::from_secs(10); + +/// What a managed stop did about one PID a service record names. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RecordedPidOutcome { + /// The PID is the stopping process's own. It is reported but never + /// signalled, and it counts as stopped: `rocmd` and an attached `rocm + /// serve` record themselves as the supervisor of the service they launched, + /// and the launcher is not the service. + OwnProcess, + /// The PID was handed to the verified termination, with this verdict. + Terminated(TerminationOutcome), +} + +impl RecordedPidOutcome { + /// The label a stop report carries: `self` for the stopping process's own + /// PID, otherwise [`TerminationOutcome::as_str`]. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::OwnProcess => "self", + Self::Terminated(outcome) => outcome.as_str(), + } + } +} + +/// One recorded PID's fate in a managed stop. +/// +/// Kept per PID because the outcomes that leave a stop unconfirmed are reported +/// differently: a `timed_out` PID was signalled and it, or a process under it, +/// outlived the deadline, while an `unverified` one was not signalled at all. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct RecordedPidStop { + pub pid: u32, + /// Which recorded PID this is: `supervisor`, `engine`, or + /// `supervisor_and_engine` when the record holds the same PID for both. + pub role: &'static str, + pub outcome: RecordedPidOutcome, +} + +impl RecordedPidStop { + /// Whether a signal went out to this PID. A `timed_out` PID counts: the + /// signal was sent and only the exit was never observed, which + /// [`stopped`](Self::stopped) reports separately. + #[must_use] + pub const fn signaled(self) -> bool { + matches!( + self.outcome, + RecordedPidOutcome::Terminated( + TerminationOutcome::Graceful + | TerminationOutcome::Forced + | TerminationOutcome::TimedOut + | TerminationOutcome::DescendantsUnconfirmed + ) + ) + } + + /// Whether the stop escalated to a forced kill for this PID. That is not + /// "the forced kill worked": a forced stop only reaches `timed_out` by way + /// of the escalation, and [`stopped`](Self::stopped) says whether the + /// process actually went away. + #[must_use] + pub const fn forced(self) -> bool { + matches!( + self.outcome, + RecordedPidOutcome::Terminated( + TerminationOutcome::Forced + | TerminationOutcome::TimedOut + | TerminationOutcome::DescendantsUnconfirmed + ) + ) + } + + /// Whether the recorded process is confirmed no longer running. + #[must_use] + pub const fn stopped(self) -> bool { + match self.outcome { + RecordedPidOutcome::OwnProcess => true, + RecordedPidOutcome::Terminated(outcome) => outcome.stopped(), + } + } + + /// The entry a stop report lists for this PID under `pid_outcomes`. + #[must_use] + pub fn report_entry(self) -> serde_json::Value { + serde_json::json!({ + "pid": self.pid, + "role": self.role, + "outcome": self.outcome.as_str(), + }) + } +} + +/// Terminate the processes a service record names, verifying each PID's +/// identity first, and report what happened to each. +/// +/// PIDs are recycled, so a persisted PID is not by itself evidence that the +/// recorded process still holds it. Each PID is paired with **its own** +/// start-time token (`supervisor_start_ticks` with `supervisor_pid`, +/// `engine_start_ticks` with `engine_pid`) and handed to `terminate`, which +/// production supplies as [`terminate_recorded_pid`]. When the two PIDs +/// coincide, the entry is reported once, under the role `supervisor_and_engine`, +/// and keeps a known token in preference to a missing one: a token is never read +/// across from the PID it does not belong to. The supervisor is visited first. +/// +/// A zero PID names nothing and is skipped. The stopping process's own PID is +/// reported as [`RecordedPidOutcome::OwnProcess`] and `terminate` is not called +/// for it. +/// +/// `terminate` is injectable so a caller's test can supply the verdicts it +/// cannot construct on demand (a process that survives `SIGKILL`, one whose +/// identity cannot be read). +/// +/// One verdict is settled across the record rather than per PID. A root that +/// exited by itself before its processes could be taken down +/// ([`TerminationOutcome::ExitedBeforeTreeKill`]) cannot say what it left +/// running. It becomes [`TerminationOutcome::AlreadyGone`] only when another +/// process the record names is confirmed gone (a stop, or itself such a root): +/// the record's other process is the one thing that was under the root and is +/// independently verified. A root that is the only process the record names, or +/// whose sibling is still running or unconfirmed, stays unconfirmed. Processes +/// the record does not name that were under a root can be neither found nor +/// verified, which is the same limit as for a root that was already gone when a +/// stop began. +/// +/// This relies on the engine being visited as its own role, which it is: the +/// engine PID is an entry of its own, with its own token and its own verdict, so +/// a worker the root left running is reached by the engine's own termination +/// when the record names it. Both callers fold every row (`rocm services stop` +/// in `stop_internal_managed_service_confirmed_with`, `rocmd` in +/// `stop_managed_service_with`): the stop is confirmed, and the recorded PIDs and +/// the endpoint key cleared, only when every recorded role is individually +/// stopped. A root that exited alone therefore never confirms a stop on its own +/// row, and the record is kept when the engine role is not confirmed. +pub fn terminate_recorded_service_pids( + record: &crate::ManagedServiceRecord, + terminate: impl Fn(&ProcessIdentity) -> TerminationOutcome, +) -> Vec { + let mut entries: Vec<(u32, &'static str, Option)> = Vec::new(); + for (pid, role, ticks) in [ + ( + Some(record.supervisor_pid), + "supervisor", + record.supervisor_start_ticks, + ), + (record.engine_pid, "engine", record.engine_start_ticks), + ] { + let Some(pid) = pid.filter(|pid| *pid != 0) else { + continue; + }; + if let Some(existing) = entries.iter_mut().find(|(seen, _, _)| *seen == pid) { + // Only the engine entry can land here, aliasing the supervisor: the + // one PID is both, and is reported as both. + existing.1 = "supervisor_and_engine"; + if existing.2.is_none() { + existing.2 = ticks; + } + } else { + entries.push((pid, role, ticks)); + } + } + + let mut stops: Vec = entries + .into_iter() + .map(|(pid, role, ticks)| { + let outcome = if pid == std::process::id() { + RecordedPidOutcome::OwnProcess + } else { + RecordedPidOutcome::Terminated(terminate(&ProcessIdentity::new(pid, ticks))) + }; + RecordedPidStop { pid, role, outcome } + }) + .collect(); + settle_roots_that_exited_alone(&mut stops); + stops +} + +/// Whether `stop` vouches for what was under another recorded root: a recorded +/// process this stop took down ([`Graceful`](TerminationOutcome::Graceful), +/// [`Forced`](TerminationOutcome::Forced)), found already gone +/// ([`AlreadyGone`](TerminationOutcome::AlreadyGone)), or a root that exited by +/// itself ([`TerminationOutcome::ExitedBeforeTreeKill`]). Not +/// [`IdentityMismatch`](TerminationOutcome::IdentityMismatch): it says a stranger +/// holds that PID now, which is "gone" for that process but is no evidence about +/// what was under a different root. The stopping process's own PID is not a +/// service process and never counts. +const fn recorded_process_is_gone(stop: RecordedPidStop) -> bool { + matches!( + stop.outcome, + RecordedPidOutcome::Terminated( + TerminationOutcome::Graceful + | TerminationOutcome::Forced + | TerminationOutcome::AlreadyGone + | TerminationOutcome::ExitedBeforeTreeKill + ) + ) +} + +/// Turn each [`TerminationOutcome::ExitedBeforeTreeKill`] into +/// [`TerminationOutcome::AlreadyGone`] when some other recorded process is +/// confirmed gone, and leave it unconfirmed otherwise. Judged on the verdicts as +/// they came in, so two roots that both exited by themselves corroborate each +/// other. +fn settle_roots_that_exited_alone(stops: &mut [RecordedPidStop]) { + let corroborated: Vec = (0..stops.len()) + .map(|index| { + stops + .iter() + .enumerate() + .any(|(other, stop)| other != index && recorded_process_is_gone(*stop)) + }) + .collect(); + for (stop, corroborated) in stops.iter_mut().zip(corroborated) { + if corroborated + && matches!( + stop.outcome, + RecordedPidOutcome::Terminated(TerminationOutcome::ExitedBeforeTreeKill) + ) + { + stop.outcome = RecordedPidOutcome::Terminated(TerminationOutcome::AlreadyGone); + } + } +} + +/// Terminate one recorded process, and its descendants, after verifying its +/// identity. The production `terminate` of [`terminate_recorded_service_pids`]. +/// +/// The stop is forced: a managed stop is definitive, so it must not report +/// success while an engine worker still holds the GPU. Only a process whose +/// recorded identity is confirmed is ever reached by it. +/// +/// On Windows the descendants are taken separately, first. [`terminate_verified`] +/// cannot walk a process tree there (`process_tree_pids` returns just the root), +/// and the engine runs one level below the recorded PID, because the daemon +/// launches it through an `__engine-serve-http` process: a root-only kill would +/// leave the engine running and still holding the device. +/// +/// The tree kill runs only for a PID that passes the same identity gate as the +/// rest of the stop. A record written with a start-time token carries the +/// process's creation time on Windows ([`process_start_ticks`]), so a PID that +/// an unrelated process has since taken reads back [`IdentityState::Recycled`] +/// and neither it nor its subtree is touched. A record written before Windows +/// recorded a token has none to refute, and for it the gate only turns away a +/// PID that is not running at all: that old record can still name a recycled +/// PID, and `taskkill /T /F` then takes that process and its subtree. A stop +/// that confirms clears the record's PIDs, so that exposure shrinks to the +/// records written before the token existed and not yet stopped. +/// +/// On Windows the PID is held from before the identity check until the stop +/// ends: a process's PID is not handed to another process while any handle to it +/// is open, so the creation time that was compared is the creation time of the +/// process `taskkill` and the root termination then reach by PID. Without the +/// hold, the process could exit and an unrelated one take its PID between the +/// check and the kill. +/// +/// The descendant kill is bounded: a `taskkill` that has not finished within +/// [`MANAGED_STOP_GRACE`] is killed and reaped and counts as a tree kill that did +/// not complete, so a stalled helper cannot hang the stop. +/// +/// A tree kill that did not complete leaves the stop unconfirmed, whatever the +/// root's own termination says: [`terminate_verified`] can only watch the root +/// on Windows, so a descendant that `taskkill` did not take down is one this stop +/// cannot see exit, and reporting the stop confirmed would clear the record and +/// the endpoint key while a worker may still hold the device. +pub fn terminate_recorded_pid(identity: &ProcessIdentity) -> TerminationOutcome { + terminate_recorded_pid_within(identity, MANAGED_STOP_GRACE) +} + +/// [`terminate_recorded_pid`] with the root's grace chosen by the caller. +/// +/// For a process that has nothing to flush, such as a launch child cleaned up +/// because its record could not be published. Everything else is the same +/// routine: the PID hold, the identity gate, and on Windows the descendant kill, +/// whose own deadline stays [`MANAGED_STOP_GRACE`]. +pub fn terminate_recorded_pid_within( + identity: &ProcessIdentity, + grace: std::time::Duration, +) -> TerminationOutcome { + terminate_recorded_pid_with( + identity, + platform_tree_kill(), + hold_pid, + identity_state, + |identity| terminate_verified(identity, KillScope::Tree, grace, true), + ) +} + +/// Whether the separate descendant kill took the tree down. +// Only Windows has the separate step, so nothing builds a value elsewhere; the +// tests build both, and the decision on them is shared by every platform. +#[cfg_attr(not(any(windows, test)), allow(dead_code))] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TreeKill { + /// The kill ran and reported that every process it reached is terminated. + Completed, + /// The kill ran and reported that it found no such process (`taskkill` exits + /// 128): the root was gone by the time it looked. Not by itself a failure, + /// and not a success either; the root's own verdict decides which. + NotFound, + /// The kill could not run, or reported that it did not take every process + /// down. Which descendants survive is not known. + Failed, +} + +/// The verdict for a stop whose descendant kill ended as `tree` and whose root +/// termination answered `outcome`. +/// +/// When the tree kill did not complete ([`TreeKill::Failed`], or +/// [`TreeKill::NotFound`] while the root was still there), a root that this +/// stop saw exit ([`Graceful`](TerminationOutcome::Graceful), +/// [`Forced`](TerminationOutcome::Forced)) or that was already gone is reported +/// [`TerminationOutcome::DescendantsUnconfirmed`]: the root is gone, which is +/// what the wait observed, but nothing watched the processes under it, so the +/// stop cannot be called confirmed. It is not `timed_out`, which would claim +/// that something outlived a deadline. The verdicts that already deny the stop +/// ([`TimedOut`](TerminationOutcome::TimedOut), the root itself outlived the +/// deadline; [`Unverified`](TerminationOutcome::Unverified)) stay as they are. +/// +/// A kill that found no such process ([`TreeKill::NotFound`]) is judged by the +/// root's own re-read, which `outcome` carries. When the root is gone +/// ([`TerminationOutcome::AlreadyGone`]) it exited on its own between the +/// identity check and the kill: that is [`TerminationOutcome::ExitedBeforeTreeKill`], +/// not a stop. `taskkill /T` reaches children only through a live parent, so a +/// child the root left running is out of its reach, and the stop kept no list of +/// what was under the root to check afterwards. The aggregate over the record's +/// processes decides whether another recorded process corroborates the exit +/// ([`terminate_recorded_service_pids`]). When the root is still there, "no such +/// process" contradicts what the termination found, so the kill is treated as +/// failed. +const fn after_tree_kill(tree: TreeKill, outcome: TerminationOutcome) -> TerminationOutcome { + match (tree, outcome) { + (TreeKill::Completed, outcome) => outcome, + (TreeKill::NotFound, TerminationOutcome::AlreadyGone) => { + TerminationOutcome::ExitedBeforeTreeKill + } + ( + TreeKill::NotFound | TreeKill::Failed, + TerminationOutcome::Graceful + | TerminationOutcome::Forced + | TerminationOutcome::AlreadyGone, + ) => TerminationOutcome::DescendantsUnconfirmed, + (TreeKill::NotFound | TreeKill::Failed, outcome) => outcome, + } +} + +/// [`terminate_recorded_pid`] with its platform and system touchpoints supplied +/// by the caller, so a test drives the tree-kill step on every host: +/// `tree_kill` takes one PID's descendants down (`None` where the platform needs +/// no separate step), `hold` keeps the PID reserved, answering `None` when it +/// could not, and is dropped when the call returns, `state_of` reads the PID's +/// identity, and `terminate` is the verified termination of the root. +fn terminate_recorded_pid_with( + identity: &ProcessIdentity, + tree_kill: Option<&dyn Fn(u32) -> TreeKill>, + hold: impl FnOnce(&ProcessIdentity) -> Option, + state_of: impl Fn(&ProcessIdentity) -> IdentityState, + terminate: impl Fn(&ProcessIdentity) -> TerminationOutcome, +) -> TerminationOutcome { + // Before anything reads the identity, and until the end of the call. + let held = hold(identity); + if let Some(tree_kill) = tree_kill { + // The identity read that decides about the tree kill also decides about + // the stop. A refusal here (gone, recycled, unverifiable) is the verdict: + // the root is not handed to `terminate`, which re-reads the identity and + // could find a process that was unverifiable a moment ago verifiable now, + // and then stop the root alone with the tree kill skipped. + if let Some(refusal) = refusal_before_signalling(state_of(identity)) { + return refusal; + } + // An identity that matches is only worth acting on while the PID cannot + // change hands before `taskkill` reaches it. Without a hold nothing + // reserves it between this read and the kill, which takes a whole tree by + // PID, so the stop is not confirmed and nothing is signalled. A process + // that could not be opened and also cannot be read was refused above. + if held.is_none() { + return TerminationOutcome::Unverified; + } + // Taken while the root is still alive: `taskkill /T` resolves children + // through the live parent, so it cannot reach them once the root is gone. + let tree = tree_kill(identity.pid); + // `terminate` still owns the bounded wait and the verdict. A root the + // tree kill already took now reads back as `AlreadyGone`, which would + // deny a stop this command did perform, so it counts as forced; but when + // the kill found no such process the root went by itself, and + // `AlreadyGone` is the truth. + let outcome = match (tree, terminate(identity)) { + (TreeKill::NotFound, outcome) => outcome, + (_, TerminationOutcome::AlreadyGone) => TerminationOutcome::Forced, + (_, outcome) => outcome, + }; + return after_tree_kill(tree, outcome); + } + terminate(identity) +} + +/// What keeps a PID reserved while a stop works on it: on Windows an open +/// handle to the process, which stops the OS giving the PID to another process; +/// nothing elsewhere, where the stop signals by PID and `terminate_verified` +/// re-verifies it. +#[cfg(windows)] +struct PidHold { + _handle: std::os::windows::io::OwnedHandle, +} + +#[cfg(not(windows))] +struct PidHold; + +/// Hold `identity`'s PID, or `None` when the process cannot be opened. A stop +/// whose identity matches but whose PID is not held is not carried out, see +/// [`terminate_recorded_pid_with`]. +#[cfg(windows)] +#[allow(unsafe_code)] // Win32 FFI +fn hold_pid(identity: &ProcessIdentity) -> Option { + use std::os::windows::io::{FromRawHandle, OwnedHandle}; + use windows_sys::Win32::System::Threading::{OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION}; + + // SAFETY: `OpenProcess` takes only plain values and returns a handle or null; + // there is no pointer to keep valid. + let handle = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, identity.pid) }; + if handle.is_null() { + return None; + } + Some(PidHold { + // SAFETY: `handle` is non-null, `OpenProcess` just returned it and nothing + // else owns or closes it, so the `OwnedHandle` may take ownership and + // close it exactly once on drop. + _handle: unsafe { OwnedHandle::from_raw_handle(handle) }, + }) +} + +// The `cfg` sibling calls non-const code, so this arm cannot be `const` without +// giving the two platforms different signatures. +#[cfg(not(windows))] +#[allow(clippy::missing_const_for_fn, clippy::unnecessary_wraps)] +fn hold_pid(_identity: &ProcessIdentity) -> Option { + Some(PidHold) +} + +/// The separate descendant kill this platform needs before the verified +/// termination of a root, if any: `taskkill /T /F` on Windows, none elsewhere, +/// where [`terminate_verified`] walks the tree itself. +// The `cfg` arms differ in what they return, so the non-Windows arm cannot be a +// `const fn` without giving the two platforms different signatures. +#[allow(clippy::missing_const_for_fn)] +fn platform_tree_kill() -> Option<&'static dyn Fn(u32) -> TreeKill> { + #[cfg(windows)] + { + Some(&taskkill_tree) + } + #[cfg(not(windows))] + { + None + } +} + +/// `taskkill`'s exit code for "the process was not found". +#[cfg(any(windows, test))] +const TASKKILL_PROCESS_NOT_FOUND: i32 = 128; + +/// The verdict on one `taskkill` run, from whether it started and the exit code +/// it ended with. A clean exit (0) is a completed tree kill and 128 is +/// [`TreeKill::NotFound`], left to the root's re-read to judge. A failed launch, +/// a run that did not finish within its deadline +/// ([`std::io::ErrorKind::TimedOut`], see [`run_to_exit_code`]), any other +/// nonzero code (a process it could not terminate) and a run that ended without +/// a code are all [`TreeKill::Failed`]. Split out of the process spawn so a test +/// runs it on every host. +#[cfg(any(windows, test))] +fn classify_taskkill(run: std::io::Result>) -> TreeKill { + match run { + Ok(Some(0)) => TreeKill::Completed, + Ok(Some(TASKKILL_PROCESS_NOT_FOUND)) => TreeKill::NotFound, + Ok(_) | Err(_) => TreeKill::Failed, + } +} + +/// How long the descendant kill may run. It happens before +/// [`terminate_verified`]'s bounded waits, so without its own deadline a stalled +/// helper would hang the stop; the same bound as every other wait of a managed +/// stop. +#[cfg(windows)] +const TREE_KILL_DEADLINE: Duration = MANAGED_STOP_GRACE; + +/// Start `command` and wait for it to exit, for at most `deadline`. Returns its +/// exit code (`None` when it ended without one). A child still running at the +/// deadline is killed and reaped, so none is left behind, and the call fails +/// with [`std::io::ErrorKind::TimedOut`]. +#[cfg(any(windows, test))] +fn run_to_exit_code( + command: &mut std::process::Command, + deadline: Duration, +) -> std::io::Result> { + let mut child = command.spawn()?; + wait_for_exit_code(&mut child, deadline) +} + +/// The wait of [`run_to_exit_code`], on a child already started. Kills and reaps +/// `child` when the deadline passes, or when it cannot be polled, rather than +/// leaving it running. +#[cfg(any(windows, test))] +fn wait_for_exit_code( + child: &mut std::process::Child, + deadline: Duration, +) -> std::io::Result> { + let end = Instant::now() + deadline; + loop { + match child.try_wait() { + Ok(Some(status)) => return Ok(status.code()), + Ok(None) => {} + Err(error) => { + let _ = child.kill(); + let _ = child.wait(); + return Err(error); + } + } + let now = Instant::now(); + if now >= end { + let _ = child.kill(); + let _ = child.wait(); + return Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "the process did not exit before its deadline", + )); + } + std::thread::sleep(POLL_INTERVAL.min(end.saturating_duration_since(now))); + } +} + +/// The arguments that take `pid` and everything under it down: by PID, with its +/// tree (`/T`), forced (`/F`). Split out so a test pins them on every host. +#[cfg(any(windows, test))] +fn taskkill_args(pid: u32) -> [String; 4] { + [ + "/PID".to_owned(), + pid.to_string(), + "/T".to_owned(), + "/F".to_owned(), + ] +} + +#[cfg(windows)] +fn taskkill_tree(pid: u32) -> TreeKill { + use std::process::{Command, Stdio}; + classify_taskkill(run_to_exit_code( + Command::new(crate::windows_system_tool("taskkill.exe")) + .args(taskkill_args(pid)) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()), + TREE_KILL_DEADLINE, + )) +} + /// Poll until every process in `members` has exited, or `grace` elapses. /// /// A member is exited once its PID is gone or now belongs to a different process @@ -300,8 +963,9 @@ fn send_signal(pid: u32, signal: Signal, _tree: bool) -> bool { /// Read the kernel start-time (field 22 of `/proc//stat`) for `pid`. /// -/// Returns `None` when the value cannot be read, including on non-Linux -/// platforms, where identity verification degrades to best-effort. +/// Returns `None` when the value cannot be read, including on platforms with +/// neither `/proc` nor a creation time to read (neither Linux nor Windows), +/// where identity verification degrades to best-effort. #[cfg(target_os = "linux")] #[must_use] pub fn process_start_ticks(pid: u32) -> Option { @@ -309,7 +973,56 @@ pub fn process_start_ticks(pid: u32) -> Option { parse_start_ticks(&stat) } -#[cfg(not(target_os = "linux"))] +/// Read the creation time of `pid` (`GetProcessTimes`) as the token that tells +/// the process from a later one that reuses its PID. +/// +/// The value is the creation time as a Windows `FILETIME` (100 ns intervals +/// since 1601), not clock ticks since boot: only equality with a value read +/// earlier from the same PID means anything. `None` when the process cannot be +/// opened for querying (it is gone, or belongs to a more privileged account). +#[cfg(windows)] +#[must_use] +#[allow(unsafe_code)] // Win32 FFI +pub fn process_start_ticks(pid: u32) -> Option { + use windows_sys::Win32::Foundation::{CloseHandle, FILETIME}; + use windows_sys::Win32::System::Threading::{ + GetProcessTimes, OpenProcess, PROCESS_QUERY_LIMITED_INFORMATION, + }; + + if pid == 0 { + return None; + } + // SAFETY: plain Win32 calls. The handle is checked before use and closed on + // every path, and the four out-parameters are valid for the one call. + unsafe { + let handle = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid); + if handle.is_null() { + return None; + } + let mut creation = FILETIME::default(); + let mut exit = FILETIME::default(); + let mut kernel = FILETIME::default(); + let mut user = FILETIME::default(); + let read = GetProcessTimes( + handle, + &raw mut creation, + &raw mut exit, + &raw mut kernel, + &raw mut user, + ); + CloseHandle(handle); + (read != 0).then(|| filetime_ticks(creation.dwLowDateTime, creation.dwHighDateTime)) + } +} + +/// The 64-bit value of a Windows `FILETIME`, which a Win32 call returns as two +/// 32-bit halves. Kept apart from the FFI so a test can run it on every host. +#[cfg(any(windows, test))] +fn filetime_ticks(low: u32, high: u32) -> u64 { + (u64::from(high) << 32) | u64::from(low) +} + +#[cfg(not(any(target_os = "linux", windows)))] #[must_use] // Platform stub. The `cfg` sibling calls non-const code, so making only this // arm `const` would give the two platforms different signatures and push @@ -418,6 +1131,25 @@ mod tests { assert!(TerminationOutcome::AlreadyGone.stopped()); assert!(TerminationOutcome::IdentityMismatch.stopped()); assert!(!TerminationOutcome::TimedOut.stopped()); + // The root is gone but a process under it may not be: not a stop, not + // graceful, and not the label of a timeout. + assert!(!TerminationOutcome::DescendantsUnconfirmed.stopped()); + assert!(!TerminationOutcome::DescendantsUnconfirmed.graceful()); + assert_eq!( + TerminationOutcome::DescendantsUnconfirmed.as_str(), + "descendants_unconfirmed" + ); + assert_ne!( + TerminationOutcome::DescendantsUnconfirmed.as_str(), + TerminationOutcome::TimedOut.as_str() + ); + // A root that exited by itself before its tree kill: not a stop alone. + assert!(!TerminationOutcome::ExitedBeforeTreeKill.stopped()); + assert!(!TerminationOutcome::ExitedBeforeTreeKill.graceful()); + assert_eq!( + TerminationOutcome::ExitedBeforeTreeKill.as_str(), + "exited_before_tree_kill" + ); // Unverified: neither stopped nor graceful — we could not act. assert!(!TerminationOutcome::Unverified.stopped()); @@ -445,7 +1177,7 @@ mod tests { let _ = child.wait(); } - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", windows))] #[test] fn identity_matches_our_own_process() { let pid = std::process::id(); @@ -454,7 +1186,7 @@ mod tests { assert_eq!(identity_state(&id), IdentityState::Matches); } - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", windows))] #[test] fn identity_mismatch_when_start_ticks_differ() { // Same live PID (ours), but a different recorded start-time: a recycled @@ -474,7 +1206,7 @@ mod tests { assert_eq!(identity_state(&id), IdentityState::Gone); } - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", windows))] #[test] fn refuses_to_signal_on_identity_mismatch() { // Our own PID with a wrong start-time. A forced stop must NOT kill us: @@ -614,4 +1346,1221 @@ mod tests { ); reap(child); } + + // ---- The managed-stop routine `rocm` and `rocmd` share. ---- + + use std::cell::RefCell; + + /// A PID no host has in use. + const SUPERVISOR_PID: u32 = 999_999_998; + const ENGINE_PID: u32 = 999_999_997; + + fn record_naming( + supervisor: u32, + supervisor_ticks: Option, + engine: Option, + engine_ticks: Option, + ) -> crate::ManagedServiceRecord { + let paths = crate::AppPaths { + config_dir: std::path::PathBuf::from("config"), + data_dir: std::path::PathBuf::from("data"), + cache_dir: std::path::PathBuf::from("cache"), + }; + let mut record = crate::ManagedServiceRecord::new( + &paths, + "svc-managed-stop", + "vllm", + "m", + "m", + "127.0.0.1", + 9, + "managed", + supervisor, + None, + None, + None, + ); + record.supervisor_start_ticks = supervisor_ticks; + record.engine_pid = engine; + record.engine_start_ticks = engine_ticks; + record + } + + /// The identities `terminate_recorded_service_pids` handed its termination + /// function, in call order, and the stops it reported. + fn stop_with( + record: &crate::ManagedServiceRecord, + outcome: TerminationOutcome, + ) -> (Vec, Vec) { + let seen = RefCell::new(Vec::new()); + let stops = terminate_recorded_service_pids(record, |identity| { + seen.borrow_mut().push(*identity); + outcome + }); + (seen.into_inner(), stops) + } + + #[test] + fn managed_stop_grace_is_ten_seconds() { + assert_eq!(MANAGED_STOP_GRACE, Duration::from_secs(10)); + } + + #[test] + fn managed_stop_terminates_supervisor_then_engine_each_with_its_own_start_time() { + let record = record_naming(SUPERVISOR_PID, Some(11), Some(ENGINE_PID), Some(22)); + + let (seen, stops) = stop_with(&record, TerminationOutcome::Graceful); + + assert_eq!( + seen, + vec![ + ProcessIdentity::new(SUPERVISOR_PID, Some(11)), + ProcessIdentity::new(ENGINE_PID, Some(22)), + ], + "the supervisor first, and no start-time read across from the other PID" + ); + assert_eq!( + stops + .iter() + .map(|stop| (stop.pid, stop.role)) + .collect::>(), + vec![(SUPERVISOR_PID, "supervisor"), (ENGINE_PID, "engine")] + ); + } + + #[test] + fn managed_stop_terminates_a_pid_recorded_twice_once_and_keeps_a_known_start_time() { + // Either entry may be the one that carries the token. + for (supervisor_ticks, engine_ticks, expected) in [ + (None, Some(33), Some(33)), + (Some(44), None, Some(44)), + (Some(44), Some(33), Some(44)), + (None, None, None), + ] { + let record = record_naming( + SUPERVISOR_PID, + supervisor_ticks, + Some(SUPERVISOR_PID), + engine_ticks, + ); + + let (seen, stops) = stop_with(&record, TerminationOutcome::Graceful); + + assert_eq!( + seen, + vec![ProcessIdentity::new(SUPERVISOR_PID, expected)], + "{supervisor_ticks:?} / {engine_ticks:?}" + ); + assert_eq!(stops.len(), 1); + assert_eq!(stops[0].role, "supervisor_and_engine"); + } + } + + #[test] + fn managed_stop_skips_a_zero_pid_and_an_absent_engine() { + let record = record_naming(0, None, Some(0), None); + + let (seen, stops) = stop_with(&record, TerminationOutcome::Graceful); + + assert!(seen.is_empty()); + assert!(stops.is_empty()); + } + + #[test] + fn managed_stop_reports_its_own_pid_as_self_and_never_terminates_it() { + let own = std::process::id(); + let record = record_naming(own, None, Some(ENGINE_PID), None); + + let (seen, stops) = stop_with(&record, TerminationOutcome::Graceful); + + assert_eq!( + seen, + vec![ProcessIdentity::new(ENGINE_PID, None)], + "the stopping process is never signalled" + ); + assert_eq!(stops.len(), 2); + assert_eq!(stops[0].pid, own); + assert_eq!(stops[0].role, "supervisor"); + assert_eq!(stops[0].outcome, RecordedPidOutcome::OwnProcess); + assert_eq!(stops[0].outcome.as_str(), "self"); + assert!(!stops[0].signaled() && !stops[0].forced()); + assert!(stops[0].stopped(), "the launcher is not the service"); + assert_eq!( + stops[0].report_entry(), + serde_json::json!({"pid": own, "role": "supervisor", "outcome": "self"}) + ); + } + + #[test] + fn managed_stop_reports_a_pid_that_is_both_roles_and_its_own_as_self() { + let own = std::process::id(); + let record = record_naming(own, None, Some(own), None); + + let (seen, stops) = stop_with(&record, TerminationOutcome::Graceful); + + assert!(seen.is_empty()); + assert_eq!(stops.len(), 1); + assert_eq!(stops[0].role, "supervisor_and_engine"); + assert_eq!(stops[0].outcome, RecordedPidOutcome::OwnProcess); + } + + #[test] + fn managed_stop_classifies_each_outcome_as_signaled_forced_and_stopped() { + use TerminationOutcome::{ + AlreadyGone, DescendantsUnconfirmed, ExitedBeforeTreeKill, Forced, Graceful, + IdentityMismatch, TimedOut, Unverified, + }; + let record = record_naming(SUPERVISOR_PID, None, None, None); + + // (outcome, signaled, forced, stopped) + let expected = [ + (AlreadyGone, false, false, true), + (IdentityMismatch, false, false, true), + (Unverified, false, false, false), + (Graceful, true, false, true), + (Forced, true, true, true), + (TimedOut, true, true, false), + // Signalled and forced by the tree kill, but not confirmed gone. + (DescendantsUnconfirmed, true, true, false), + // Nothing signalled: the root was gone before the tree kill ran. + (ExitedBeforeTreeKill, false, false, false), + ]; + for (outcome, signaled, forced, stopped) in expected { + let (_, stops) = stop_with(&record, outcome); + assert_eq!(stops.len(), 1); + let stop = stops[0]; + assert_eq!(stop.outcome, RecordedPidOutcome::Terminated(outcome)); + assert_eq!(stop.outcome.as_str(), outcome.as_str()); + assert_eq!( + (stop.signaled(), stop.forced(), stop.stopped()), + (signaled, forced, stopped), + "{outcome:?}" + ); + } + } + + /// What one `terminate_recorded_pid_with` call did, in order. + #[derive(Debug, PartialEq, Eq)] + enum Step { + Hold(u32), + /// The hold could not be taken: nothing reserves the PID. + HoldFailed(u32), + TreeKill(u32), + Terminate(u32), + Release, + } + + /// A stand-in for the Windows PID hold: records that it was taken and, when + /// it is dropped, that it was released. + struct RecordedHold<'a>(&'a RefCell>); + + impl Drop for RecordedHold<'_> { + fn drop(&mut self) { + self.0.borrow_mut().push(Step::Release); + } + } + + /// Run `terminate_recorded_pid_with` for `pid`, with a tree-kill step + /// present or not and ending as `tree`, the identity in `state`, and the + /// root's termination answering `outcome`. Returns the verdict and the steps + /// taken, the hold's release included. + fn tree_kill_run_with( + has_tree_kill: bool, + tree: TreeKill, + state: IdentityState, + outcome: TerminationOutcome, + ) -> (TerminationOutcome, Vec) { + let steps = RefCell::new(Vec::new()); + let tree_kill = |pid: u32| { + steps.borrow_mut().push(Step::TreeKill(pid)); + tree + }; + let verdict = terminate_recorded_pid_with( + &ProcessIdentity::new(SUPERVISOR_PID, None), + has_tree_kill.then_some(&tree_kill as &dyn Fn(u32) -> TreeKill), + |identity| { + steps.borrow_mut().push(Step::Hold(identity.pid)); + Some(RecordedHold(&steps)) + }, + |_| state, + |identity| { + steps.borrow_mut().push(Step::Terminate(identity.pid)); + outcome + }, + ); + (verdict, steps.into_inner()) + } + + /// [`tree_kill_run_with`] for a tree kill that completes. + fn tree_kill_run( + has_tree_kill: bool, + state: IdentityState, + outcome: TerminationOutcome, + ) -> (TerminationOutcome, Vec) { + tree_kill_run_with(has_tree_kill, TreeKill::Completed, state, outcome) + } + + #[test] + fn the_tree_is_killed_before_the_root_is_terminated() { + let (_, steps) = tree_kill_run(true, IdentityState::Matches, TerminationOutcome::Forced); + + assert_eq!( + steps, + vec![ + Step::Hold(SUPERVISOR_PID), + Step::TreeKill(SUPERVISOR_PID), + Step::Terminate(SUPERVISOR_PID), + Step::Release, + ], + "`taskkill /T` resolves children through the live parent, so the \ + tree kill has to come first" + ); + } + + #[test] + fn a_root_the_tree_kill_already_took_is_reported_forced_not_already_gone() { + let (verdict, _) = tree_kill_run( + true, + IdentityState::Matches, + TerminationOutcome::AlreadyGone, + ); + + assert_eq!( + verdict, + TerminationOutcome::Forced, + "the stop did perform the kill; reading the root back as gone must not deny it" + ); + } + + #[test] + fn the_verdict_after_a_tree_kill_is_the_terminations_own_for_every_other_outcome() { + use TerminationOutcome::{Forced, Graceful, IdentityMismatch, TimedOut, Unverified}; + for outcome in [Graceful, Forced, TimedOut, Unverified, IdentityMismatch] { + let (verdict, _) = tree_kill_run(true, IdentityState::Matches, outcome); + + assert_eq!(verdict, outcome); + } + } + + #[test] + fn no_tree_kill_is_run_for_a_pid_that_is_gone_recycled_or_unverifiable() { + for state in [ + IdentityState::Gone, + IdentityState::Recycled, + IdentityState::Indeterminate, + ] { + let (verdict, steps) = tree_kill_run(true, state, TerminationOutcome::AlreadyGone); + + assert_eq!( + steps, + vec![Step::Hold(SUPERVISOR_PID), Step::Release], + "{state:?}: a stranger's process tree must not be taken, and the root \ + is not handed to the termination, which would re-read the identity" + ); + let expected = match state { + IdentityState::Gone => TerminationOutcome::AlreadyGone, + IdentityState::Recycled => TerminationOutcome::IdentityMismatch, + _ => TerminationOutcome::Unverified, + }; + assert_eq!( + verdict, expected, + "{state:?}: the initial refusal is the verdict" + ); + } + } + + #[test] + fn without_a_tree_kill_step_the_root_is_terminated_and_its_verdict_unmapped() { + let (verdict, steps) = tree_kill_run( + false, + IdentityState::Matches, + TerminationOutcome::AlreadyGone, + ); + + assert_eq!( + steps, + vec![ + Step::Hold(SUPERVISOR_PID), + Step::Terminate(SUPERVISOR_PID), + Step::Release + ] + ); + assert_eq!(verdict, TerminationOutcome::AlreadyGone); + } + + /// Every way a recorded token and the live process's token can stand, with + /// what the stop does about each: whether the Windows tree kill runs, and the + /// verdict the stop reports (the label and whether it counts as stopped). + /// + /// The live process is running in every row; the tokens go through the same + /// decision production uses ([`classify_live_identity`] and + /// [`refusal_before_signalling`]), with the system reads and the signalling + /// supplied, so each row runs on every host. A process is refused only when + /// a token was recorded and the live one is provably different. + #[test] + fn what_a_stop_does_for_each_recorded_and_live_token() { + const LIVE: u64 = 0x01DC_0000_0000_0001; + struct Row { + name: &'static str, + recorded: Option, + live: Option, + tree_kill_runs: bool, + label: &'static str, + stopped: bool, + } + let row = |name, recorded, live, tree_kill_runs, label, stopped| Row { + name, + recorded, + live, + tree_kill_runs, + label, + stopped, + }; + let rows = [ + // The shape `service-stop-07` records, and any record older than the + // token: nothing to refute, so the whole tree is taken as before. + row("no token recorded", None, Some(LIVE), true, "forced", true), + row( + "no token recorded, none readable", + None, + None, + true, + "forced", + true, + ), + row( + "token matches", + Some(LIVE), + Some(LIVE), + true, + "forced", + true, + ), + // A recycled PID: the process is a stranger, left alone. + row( + "token differs", + Some(LIVE + 1), + Some(LIVE), + false, + "identity_mismatch", + true, + ), + // A recorded token that cannot be read back is neither signalled nor + // reported stopped. + row( + "token unreadable", + Some(LIVE), + None, + false, + "unverified", + false, + ), + ]; + for Row { + name: row, + recorded, + live, + tree_kill_runs, + label, + stopped, + } in rows + { + let steps = RefCell::new(Vec::new()); + let tree_kill = |pid: u32| { + steps.borrow_mut().push(Step::TreeKill(pid)); + TreeKill::Completed + }; + let verdict = terminate_recorded_pid_with( + &ProcessIdentity::new(SUPERVISOR_PID, recorded), + Some(&tree_kill as &dyn Fn(u32) -> TreeKill), + |id| { + steps.borrow_mut().push(Step::Hold(id.pid)); + Some(RecordedHold(&steps)) + }, + |id| classify_live_identity(id.start_ticks, live), + |id| { + steps.borrow_mut().push(Step::Terminate(id.pid)); + // What the verified termination answers: its refusal for a + // process the identity rules out, otherwise the tree kill + // has already taken the root and it reads back as gone. + refusal_before_signalling(classify_live_identity(id.start_ticks, live)) + .unwrap_or(TerminationOutcome::AlreadyGone) + }, + ); + let expected_steps = if tree_kill_runs { + vec![ + Step::Hold(SUPERVISOR_PID), + Step::TreeKill(SUPERVISOR_PID), + Step::Terminate(SUPERVISOR_PID), + Step::Release, + ] + } else { + // Refused on the initial read: not handed to the termination. + vec![Step::Hold(SUPERVISOR_PID), Step::Release] + }; + assert_eq!(steps.into_inner(), expected_steps, "{row}: steps"); + assert_eq!(verdict.as_str(), label, "{row}: verdict"); + assert_eq!(verdict.stopped(), stopped, "{row}: counts as stopped"); + } + } + + #[test] + fn a_recorded_token_refutes_a_live_one_only_when_both_are_known_and_differ() { + use IdentityState::{Indeterminate, Matches, Recycled}; + assert_eq!(classify_live_identity(None, None), Matches); + assert_eq!(classify_live_identity(None, Some(7)), Matches); + assert_eq!(classify_live_identity(Some(7), Some(7)), Matches); + assert_eq!(classify_live_identity(Some(7), Some(8)), Recycled); + assert_eq!(classify_live_identity(Some(7), None), Indeterminate); + assert_eq!(refusal_before_signalling(Matches), None); + } + + #[test] + fn only_windows_has_a_separate_tree_kill_step() { + // Ties the injected tests above to what production actually runs: the + // step they exercise is the one `terminate_recorded_pid` has on Windows + // and does without elsewhere, where `terminate_verified` walks the tree. + assert_eq!(platform_tree_kill().is_some(), cfg!(windows)); + } + + /// Run `terminate_recorded_pid_with` for a PID whose hold could not be taken + /// (the process could not be opened), the identity reading as `state`. + fn unheld_run( + has_tree_kill: bool, + state: IdentityState, + outcome: TerminationOutcome, + ) -> (TerminationOutcome, Vec) { + let steps = RefCell::new(Vec::new()); + let tree_kill = |pid: u32| { + steps.borrow_mut().push(Step::TreeKill(pid)); + TreeKill::Completed + }; + let verdict = terminate_recorded_pid_with( + &ProcessIdentity::new(SUPERVISOR_PID, None), + has_tree_kill.then_some(&tree_kill as &dyn Fn(u32) -> TreeKill), + |identity| { + steps.borrow_mut().push(Step::HoldFailed(identity.pid)); + None::> + }, + |_| state, + |identity| { + steps.borrow_mut().push(Step::Terminate(identity.pid)); + outcome + }, + ); + (verdict, steps.into_inner()) + } + + #[test] + fn a_tree_kill_is_not_run_for_a_pid_that_could_not_be_held_even_when_its_identity_matches() { + // Nothing reserves the PID between the identity read and `taskkill`, which + // takes a whole tree by PID, so a recycled PID would lose its tree. The + // stop is not confirmed, and neither the tree kill nor the termination runs. + let (verdict, steps) = unheld_run(true, IdentityState::Matches, TerminationOutcome::Forced); + + assert_eq!(verdict, TerminationOutcome::Unverified); + assert!(!verdict.stopped()); + assert_eq!(steps, vec![Step::HoldFailed(SUPERVISOR_PID)]); + } + + #[test] + fn an_unheld_pid_that_is_gone_recycled_or_unverifiable_keeps_its_refusal() { + // The hold only adds a refusal for an identity that matches: a process + // already known to be gone, a stranger, or unreadable keeps the verdict + // the identity read gives it. + for (state, expected) in [ + (IdentityState::Gone, TerminationOutcome::AlreadyGone), + ( + IdentityState::Recycled, + TerminationOutcome::IdentityMismatch, + ), + (IdentityState::Indeterminate, TerminationOutcome::Unverified), + ] { + let (verdict, steps) = unheld_run(true, state, TerminationOutcome::Forced); + + assert_eq!(verdict, expected, "{state:?}"); + assert_eq!(steps, vec![Step::HoldFailed(SUPERVISOR_PID)], "{state:?}"); + } + } + + #[test] + fn without_a_tree_kill_step_an_unheld_pid_is_still_terminated() { + // Elsewhere nothing is held and the termination re-verifies the identity + // itself, so the missing hold must not turn a stop into a refusal. + let (verdict, steps) = + unheld_run(false, IdentityState::Matches, TerminationOutcome::Graceful); + + assert_eq!(verdict, TerminationOutcome::Graceful); + assert_eq!( + steps, + vec![ + Step::HoldFailed(SUPERVISOR_PID), + Step::Terminate(SUPERVISOR_PID) + ] + ); + } + + #[test] + fn the_pid_is_held_before_its_identity_is_read_and_released_after_the_stop() { + // The hold is what makes the creation time that was compared the + // creation time of the process the kill then reaches by PID: taken + // first, released last. `Hold` comes before anything else and `Release` + // after the root's termination, with the tree kill between. + let (_, steps) = tree_kill_run(true, IdentityState::Matches, TerminationOutcome::Forced); + + assert_eq!(steps.first(), Some(&Step::Hold(SUPERVISOR_PID))); + assert_eq!(steps.last(), Some(&Step::Release)); + assert_eq!( + steps + .iter() + .filter(|step| matches!(step, Step::Release)) + .count(), + 1 + ); + } + + #[test] + fn a_tree_kill_that_did_not_complete_leaves_the_stop_unconfirmed() { + use TerminationOutcome::{ + AlreadyGone, DescendantsUnconfirmed, Forced, Graceful, TimedOut, Unverified, + }; + // Every root verdict that would confirm the stop becomes the one that + // says the root is gone but a process under it may still be running, not + // `timed_out`, which would say something outlived a deadline; the + // verdicts that already deny the stop are kept. + for (root, expected) in [ + (Forced, DescendantsUnconfirmed), + (Graceful, DescendantsUnconfirmed), + (AlreadyGone, DescendantsUnconfirmed), + (TimedOut, TimedOut), + (Unverified, Unverified), + ] { + let (verdict, steps) = + tree_kill_run_with(true, TreeKill::Failed, IdentityState::Matches, root); + + assert_eq!(verdict, expected, "root {root:?}"); + assert!( + !verdict.stopped(), + "root {root:?}: the stop is not confirmed" + ); + assert!( + steps.contains(&Step::Terminate(SUPERVISOR_PID)), + "the root is still terminated, best effort: {steps:?}" + ); + } + } + + /// `taskkill` answering "no such process" is judged by the root's own + /// re-read. The root that went away by itself between the identity check and + /// the kill is not reported as a kill this stop performed, and not as a stop + /// either on its own: `taskkill /T` reaches children only through a live + /// parent, so a child the root left running was out of its reach, and nothing + /// recorded what was under it. The record-level settling decides. + #[test] + fn a_tree_kill_that_found_no_process_with_the_root_gone_is_not_a_stop_by_itself() { + let (verdict, steps) = tree_kill_run_with( + true, + TreeKill::NotFound, + IdentityState::Matches, + TerminationOutcome::AlreadyGone, + ); + + assert_eq!( + verdict, + TerminationOutcome::ExitedBeforeTreeKill, + "the root exited on its own and nothing says what it left running" + ); + assert!(!verdict.stopped()); + assert!( + steps.contains(&Step::Terminate(SUPERVISOR_PID)), + "the root is re-read by the termination: {steps:?}" + ); + } + + /// The identity read that gates the tree kill is the one that decides the + /// stop. If it is unverifiable and a second read, inside the termination, + /// would find the process verifiable, the root must not be stopped alone with + /// the tree kill skipped: the initial refusal is the verdict and the + /// termination is never called. + #[test] + fn an_unverifiable_first_read_is_not_overtaken_by_a_verifiable_second_read() { + use std::cell::Cell; + let reads = Cell::new(0_u32); + let terminated = Cell::new(false); + let tree_killed = Cell::new(false); + let tree_kill = |_pid: u32| { + tree_killed.set(true); + TreeKill::Completed + }; + + let verdict = terminate_recorded_pid_with( + &ProcessIdentity::new(SUPERVISOR_PID, Some(7)), + Some(&tree_kill as &dyn Fn(u32) -> TreeKill), + |_| Some(()), + |_| { + // Indeterminate the first time, matching from then on. + reads.set(reads.get() + 1); + if reads.get() == 1 { + IdentityState::Indeterminate + } else { + IdentityState::Matches + } + }, + |_| { + terminated.set(true); + TerminationOutcome::Forced + }, + ); + + assert_eq!(verdict, TerminationOutcome::Unverified); + assert!(!verdict.stopped()); + assert!(!terminated.get(), "the root is not stopped alone"); + assert!(!tree_killed.get()); + assert_eq!(reads.get(), 1, "the initial read is the only one"); + } + + /// A sibling whose PID a stranger now holds ([`IdentityMismatch`]) says + /// nothing about what was under this root, so it does not corroborate it. + #[test] + fn an_identity_mismatch_sibling_does_not_corroborate_a_root_that_exited_alone() { + use TerminationOutcome::{ExitedBeforeTreeKill, IdentityMismatch}; + let record = record_naming(SUPERVISOR_PID, None, Some(ENGINE_PID), None); + + let verdicts = settled( + &record, + &[ + (SUPERVISOR_PID, ExitedBeforeTreeKill), + (ENGINE_PID, IdentityMismatch), + ], + ); + + assert_eq!(verdicts, vec![ExitedBeforeTreeKill, IdentityMismatch]); + assert!(!verdicts[0].stopped()); + } + + /// What the record's processes say together about a root that exited by + /// itself. `exits` maps each recorded PID to the verdict `terminate` gives. + fn settled( + record: &crate::ManagedServiceRecord, + exits: &[(u32, TerminationOutcome)], + ) -> Vec { + terminate_recorded_service_pids(record, |identity| { + exits + .iter() + .find(|(pid, _)| *pid == identity.pid) + .map(|(_, outcome)| *outcome) + .expect("a verdict for every recorded pid") + }) + .into_iter() + .map(|stop| match stop.outcome { + RecordedPidOutcome::Terminated(outcome) => outcome, + RecordedPidOutcome::OwnProcess => panic!("not the stopping process"), + }) + .collect() + } + + /// A parent that exits by itself while its child keeps running is not a + /// stop: with only the parent recorded, nothing independently verifies the + /// child, so the stop stays unconfirmed (and the record, the stop request + /// and the key are kept by the callers). This is the case a root's exit + /// cannot settle. + #[test] + fn a_parent_that_exits_alone_with_nothing_else_recorded_is_not_a_confirmed_stop() { + use TerminationOutcome::ExitedBeforeTreeKill; + let record = record_naming(SUPERVISOR_PID, None, None, None); + + let verdicts = settled(&record, &[(SUPERVISOR_PID, ExitedBeforeTreeKill)]); + + assert_eq!(verdicts, vec![ExitedBeforeTreeKill]); + assert!(!verdicts[0].stopped()); + } + + /// The same parent with its engine recorded: the engine is the process under + /// it that is independently verified, by its own entry. Confirmed gone, the + /// stop is confirmed, which keeps the concern that a root exiting on its own + /// must not leave a stop unconfirmed for nothing. + #[test] + fn a_parent_that_exits_alone_is_confirmed_when_the_recorded_engine_is_confirmed_gone() { + use TerminationOutcome::{AlreadyGone, ExitedBeforeTreeKill, Forced, Graceful}; + let record = record_naming(SUPERVISOR_PID, None, Some(ENGINE_PID), None); + + for engine in [Forced, Graceful, AlreadyGone] { + let verdicts = settled( + &record, + &[(SUPERVISOR_PID, ExitedBeforeTreeKill), (ENGINE_PID, engine)], + ); + + assert_eq!(verdicts, vec![AlreadyGone, engine], "engine {engine:?}"); + assert!(verdicts.iter().all(|verdict| verdict.stopped())); + } + // And the other way round: the engine is the root that exited alone. + let verdicts = settled( + &record, + &[ + (SUPERVISOR_PID, Graceful), + (ENGINE_PID, ExitedBeforeTreeKill), + ], + ); + assert_eq!(verdicts, vec![Graceful, AlreadyGone]); + } + + /// A parent that exits while the recorded engine is still running, or not + /// confirmed, stays unconfirmed: the engine is the child that survived. + #[test] + fn a_parent_that_exits_alone_stays_unconfirmed_while_the_recorded_engine_is_not_gone() { + use TerminationOutcome::{ + DescendantsUnconfirmed, ExitedBeforeTreeKill, TimedOut, Unverified, + }; + let record = record_naming(SUPERVISOR_PID, None, Some(ENGINE_PID), None); + + for engine in [TimedOut, Unverified, DescendantsUnconfirmed] { + let verdicts = settled( + &record, + &[(SUPERVISOR_PID, ExitedBeforeTreeKill), (ENGINE_PID, engine)], + ); + + assert_eq!( + verdicts, + vec![ExitedBeforeTreeKill, engine], + "engine {engine:?}" + ); + assert!( + !verdicts[0].stopped(), + "the parent is not settled by {engine:?}" + ); + } + } + + /// Two roots that both exited by themselves corroborate each other: every + /// process the record names is gone. + #[test] + fn two_roots_that_both_exited_alone_are_a_confirmed_stop() { + use TerminationOutcome::{AlreadyGone, ExitedBeforeTreeKill}; + let record = record_naming(SUPERVISOR_PID, None, Some(ENGINE_PID), None); + + let verdicts = settled( + &record, + &[ + (SUPERVISOR_PID, ExitedBeforeTreeKill), + (ENGINE_PID, ExitedBeforeTreeKill), + ], + ); + + assert_eq!(verdicts, vec![AlreadyGone, AlreadyGone]); + } + + /// The stopping process's own PID is not a service process: it never + /// corroborates a root that exited alone. + #[test] + fn the_stopping_process_does_not_corroborate_a_root_that_exited_alone() { + use TerminationOutcome::ExitedBeforeTreeKill; + let record = record_naming(SUPERVISOR_PID, None, Some(std::process::id()), None); + + let stops = terminate_recorded_service_pids(&record, |_| ExitedBeforeTreeKill); + + assert_eq!( + stops[0].outcome, + RecordedPidOutcome::Terminated(ExitedBeforeTreeKill) + ); + assert_eq!(stops[1].outcome, RecordedPidOutcome::OwnProcess); + assert!(!stops[0].stopped()); + } + + /// "No such process" while the root is still there is a contradiction, so it + /// is a failed kill and the stop stays unconfirmed, as for any other nonzero + /// exit. + #[test] + fn a_tree_kill_that_found_no_process_while_the_root_runs_is_a_failed_kill() { + use TerminationOutcome::{DescendantsUnconfirmed, Forced, Graceful, TimedOut, Unverified}; + // The termination saw the root exit (Graceful, Forced) although the kill + // said there was no such process: the exit was observed, so the report + // must not say it outlived a deadline, but nothing watched what is under + // it. + for (root, expected) in [ + (Forced, DescendantsUnconfirmed), + (Graceful, DescendantsUnconfirmed), + (TimedOut, TimedOut), + (Unverified, Unverified), + ] { + let (verdict, _) = + tree_kill_run_with(true, TreeKill::NotFound, IdentityState::Matches, root); + + assert_eq!(verdict, expected, "root {root:?}"); + assert!(!verdict.stopped(), "root {root:?}"); + } + } + + /// Any other nonzero exit with the root gone still leaves the stop + /// unconfirmed: unlike "not found", it says a process could not be + /// terminated, so a descendant may survive the root. + #[test] + fn a_failed_tree_kill_with_the_root_gone_is_still_unconfirmed() { + let (verdict, _) = tree_kill_run_with( + true, + TreeKill::Failed, + IdentityState::Matches, + TerminationOutcome::AlreadyGone, + ); + + assert_eq!(verdict, TerminationOutcome::DescendantsUnconfirmed); + assert!(!verdict.stopped()); + } + + /// A recorded token that can be read and differs proves another process + /// holds the PID, even when the rest of the query failed; without a readable + /// token the PID stays unverifiable. + #[test] + fn a_readable_token_mismatch_outranks_a_failed_query() { + use crate::process::ProcessProbe::Unqueryable; + let state = |recorded, live| { + identity_state_with( + &ProcessIdentity::new(SUPERVISOR_PID, recorded), + |_| Unqueryable, + |_| false, + |_| live, + ) + }; + + assert_eq!(state(Some(7), Some(8)), IdentityState::Recycled); + assert_eq!(state(Some(7), Some(7)), IdentityState::Indeterminate); + assert_eq!(state(Some(7), None), IdentityState::Indeterminate); + assert_eq!(state(None, Some(8)), IdentityState::Indeterminate); + assert_eq!(state(None, None), IdentityState::Indeterminate); + } + + #[test] + fn a_completed_tree_kill_leaves_the_roots_verdict_alone() { + use TerminationOutcome::{Forced, Graceful, TimedOut, Unverified}; + for outcome in [Forced, Graceful, TimedOut, Unverified] { + let (verdict, _) = + tree_kill_run_with(true, TreeKill::Completed, IdentityState::Matches, outcome); + + assert_eq!(verdict, outcome); + } + } + + /// A command that exits at once with `code`, and one that runs far longer + /// than any test waits, on whichever platform the tests run. + fn exits_with(code: i32) -> std::process::Command { + #[cfg(unix)] + let mut command = { + let mut command = std::process::Command::new("sh"); + command.args(["-c", &format!("exit {code}")]); + command + }; + #[cfg(windows)] + let mut command = { + let mut command = std::process::Command::new("cmd"); + command.args(["/C", &format!("exit {code}")]); + command + }; + command + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()); + command + } + + fn runs_for_thirty_seconds() -> std::process::Command { + #[cfg(unix)] + let mut command = { + let mut command = std::process::Command::new("sleep"); + command.arg("30"); + command + }; + #[cfg(windows)] + let mut command = { + let mut command = std::process::Command::new("ping"); + command.args(["-n", "30", "127.0.0.1"]); + command + }; + command + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()); + command + } + + #[test] + fn a_helper_that_exits_in_time_gives_its_exit_code() { + let deadline = Duration::from_secs(20); + + assert_eq!( + run_to_exit_code(&mut exits_with(0), deadline).unwrap(), + Some(0) + ); + assert_eq!( + run_to_exit_code(&mut exits_with(3), deadline).unwrap(), + Some(3) + ); + } + + #[test] + fn a_helper_that_cannot_be_started_is_an_error() { + let mut command = std::process::Command::new("rocm-core-test-no-such-program"); + + assert!(run_to_exit_code(&mut command, Duration::from_secs(1)).is_err()); + } + + /// A helper that stalls must not hold the stop: it is killed and reaped at + /// its deadline, the call ends close to it, and the verdict is a failed tree + /// kill, which leaves the stop unconfirmed. + #[test] + fn a_helper_that_stalls_is_killed_and_reaped_at_its_deadline() { + let mut command = runs_for_thirty_seconds(); + let mut child = command.spawn().expect("spawn the stalled helper"); + let pid = child.id(); + let started = Instant::now(); + + let run = wait_for_exit_code(&mut child, Duration::from_millis(200)); + let elapsed = started.elapsed(); + + let error = run.expect_err("a stalled helper does not finish"); + assert_eq!(error.kind(), std::io::ErrorKind::TimedOut); + assert!( + elapsed >= Duration::from_millis(200) && elapsed < Duration::from_secs(10), + "returned after {elapsed:?}, far from the deadline" + ); + // Reaped: its status is already collected, and the process is gone. + assert!( + matches!(child.try_wait(), Ok(Some(_))), + "the killed helper was waited for" + ); + assert!( + !crate::process_is_running(pid), + "the helper was killed, not left running" + ); + assert_eq!(classify_taskkill(Err(error)), TreeKill::Failed); + } + + #[test] + fn taskkill_is_asked_for_the_pid_its_tree_and_force() { + assert_eq!(taskkill_args(4242), ["/PID", "4242", "/T", "/F"]); + } + + #[test] + fn only_a_clean_taskkill_exit_is_a_completed_tree_kill_and_128_is_not_found() { + use std::io::{Error, ErrorKind}; + assert_eq!(classify_taskkill(Ok(Some(0))), TreeKill::Completed); + // A process it could not terminate. + assert_eq!(classify_taskkill(Ok(Some(1))), TreeKill::Failed); + assert_eq!(classify_taskkill(Ok(Some(5))), TreeKill::Failed); + // No such process: not a failure by itself, the root's re-read decides. + assert_eq!(classify_taskkill(Ok(Some(128))), TreeKill::NotFound); + // Ended without an exit code. + assert_eq!(classify_taskkill(Ok(None)), TreeKill::Failed); + // Could not be started, or did not finish before its deadline. + assert_eq!( + classify_taskkill(Err(Error::from(ErrorKind::NotFound))), + TreeKill::Failed + ); + assert_eq!( + classify_taskkill(Err(Error::from(ErrorKind::TimedOut))), + TreeKill::Failed + ); + } + + /// A PID that cannot be queried is neither gone nor a match, through the + /// same liveness gate every stop starts with. + #[test] + fn a_pid_that_cannot_be_queried_is_unverified_not_already_gone() { + use crate::process::ProcessProbe::{Gone, Running, Unqueryable}; + let state = |probe, zombie, recorded, live| { + identity_state_with( + &ProcessIdentity::new(SUPERVISOR_PID, recorded), + |_| probe, + |_| zombie, + |_| live, + ) + }; + // Whatever token was recorded, and whatever can be read of the process, + // short of a readable token that differs (`a_readable_token_mismatch_...`). + for (recorded, live) in [ + (None, None), + (None, Some(7)), + (None, Some(8)), + (Some(7), None), + (Some(7), Some(7)), + ] { + assert_eq!( + state(Unqueryable, false, recorded, live), + IdentityState::Indeterminate, + "recorded {recorded:?}, live {live:?}" + ); + } + assert_eq!(state(Gone, false, Some(7), Some(7)), IdentityState::Gone); + assert_eq!(state(Running, true, Some(7), Some(7)), IdentityState::Gone); + assert_eq!( + state(Running, false, Some(7), Some(7)), + IdentityState::Matches + ); + assert_eq!( + state(Running, false, Some(7), Some(8)), + IdentityState::Recycled + ); + assert_eq!( + state(Running, false, Some(7), None), + IdentityState::Indeterminate + ); + } + + /// The whole stop for a live process the caller may not query: not + /// signalled, no tree kill, and reported as unverified rather than as + /// stopped, so the record and the endpoint key are kept. + #[test] + fn a_stop_of_a_live_process_it_cannot_query_is_not_confirmed() { + use crate::process::ProcessProbe::Unqueryable; + let steps = RefCell::new(Vec::new()); + let tree_kill = |pid: u32| { + steps.borrow_mut().push(Step::TreeKill(pid)); + TreeKill::Completed + }; + let state_of = + |id: &ProcessIdentity| identity_state_with(id, |_| Unqueryable, |_| false, |_| None); + let verdict = terminate_recorded_pid_with( + &ProcessIdentity::new(SUPERVISOR_PID, Some(7)), + Some(&tree_kill as &dyn Fn(u32) -> TreeKill), + |_| Some(()), + state_of, + // The verified termination's own gate, with the same read. + |id| refusal_before_signalling(state_of(id)).unwrap_or(TerminationOutcome::Forced), + ); + + assert_eq!(verdict, TerminationOutcome::Unverified); + assert!(!verdict.stopped()); + assert!( + steps.into_inner().is_empty(), + "a process that cannot be told from a stranger is not tree-killed" + ); + } + + #[test] + fn terminate_recorded_pid_reports_an_absent_pid_as_already_gone() { + let outcome = terminate_recorded_pid(&ProcessIdentity::new(SUPERVISOR_PID, None)); + + assert_eq!(outcome, TerminationOutcome::AlreadyGone); + } + + #[cfg(unix)] + #[test] + fn terminate_recorded_pid_stops_a_running_verified_process() { + let child = Command::new("sleep") + .arg("60") + .spawn() + .expect("spawn child"); + let id = ProcessIdentity::capture(child.id()); + + let outcome = terminate_recorded_pid(&id); + + assert!(outcome.stopped(), "{outcome:?}"); + assert!(matches!( + outcome, + TerminationOutcome::Graceful | TerminationOutcome::Forced + )); + reap(child); + assert!(!crate::process_is_running(id.pid)); + } + + #[test] + fn a_filetime_is_its_high_word_above_its_low_word() { + // The Windows creation time arrives as two 32-bit halves; swapping them + // or dropping the shift would make every process read as a different one. + assert_eq!( + filetime_ticks(0x89AB_CDEF, 0x0123_4567), + 0x0123_4567_89AB_CDEF + ); + assert_eq!(filetime_ticks(1, 0), 1); + assert_eq!(filetime_ticks(0, 1), 1 << 32); + assert_eq!(filetime_ticks(u32::MAX, u32::MAX), u64::MAX); + } + + /// The managed stop's own entry point refuses a PID whose recorded identity no + /// longer matches, and reaches neither it nor its tree. Our own PID with a + /// wrong start-time is what a recycled PID looks like; if the stop went + /// ahead, `taskkill /T /F` on Windows (a plain forced kill elsewhere) would + /// end this test process instead of asserting. On Windows this runs the real + /// tree-kill gate: the verdict has to come from the identity check, before + /// any `taskkill`. + #[cfg(any(target_os = "linux", windows))] + #[test] + fn terminate_recorded_pid_refuses_a_pid_whose_recorded_identity_no_longer_matches() { + let pid = std::process::id(); + let real = process_start_ticks(pid).expect("own start-time"); + let stale = ProcessIdentity::new(pid, Some(real.wrapping_add(1))); + + let outcome = terminate_recorded_pid(&stale); + + assert_eq!(outcome, TerminationOutcome::IdentityMismatch); + assert!(crate::process_is_running(pid), "nothing was signalled"); + } + + /// The same refusal against a bystander that is a different process holding + /// the recorded PID (the recycled case), then the stop of that process when + /// its identity does match. Windows only: that is where a tree kill by PID + /// follows the identity gate, and where a creation-time token is read. + #[cfg(windows)] + #[test] + fn terminate_recorded_pid_leaves_a_recycled_pid_alone_and_stops_the_verified_one() { + use std::process::{Command, Stdio}; + + let mut child = Command::new("ping") + .args(["-n", "120", "127.0.0.1"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn test child"); + let pid = child.id(); + let real = process_start_ticks(pid).expect("a live process has a creation time"); + assert_eq!( + process_start_ticks(pid), + Some(real), + "the creation time of one process does not change" + ); + assert_ne!( + Some(real), + process_start_ticks(std::process::id()), + "two processes do not share one" + ); + + let outcome = + terminate_recorded_pid(&ProcessIdentity::new(pid, Some(real.wrapping_add(1)))); + assert_eq!(outcome, TerminationOutcome::IdentityMismatch); + assert!( + child.try_wait().expect("check the bystander").is_none(), + "a PID whose recorded creation time differs is a stranger and must keep running" + ); + + let outcome = terminate_recorded_pid(&ProcessIdentity::capture(pid)); + assert!(outcome.stopped(), "{outcome:?}"); + let _ = child.wait(); + assert!(!crate::process_is_running(pid)); + } + + /// A record with no token (written before Windows recorded one, and the shape + /// `service-stop-07` plants) cannot be refuted and still gets the whole stop: + /// the process ends. Windows only, where a real process is stopped by PID. + #[cfg(windows)] + #[test] + fn terminate_recorded_pid_stops_a_process_recorded_without_a_token() { + use std::process::{Command, Stdio}; + + let mut child = Command::new("ping") + .args(["-n", "120", "127.0.0.1"]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("spawn test child"); + let pid = child.id(); + + let outcome = terminate_recorded_pid(&ProcessIdentity::new(pid, None)); + + assert!(outcome.stopped(), "{outcome:?}"); + assert_eq!(outcome.as_str(), "forced"); + assert!( + child.wait().is_ok() && !crate::process_is_running(pid), + "the process recorded without a token must be gone" + ); + } } diff --git a/crates/rocm-core/src/process.rs b/crates/rocm-core/src/process.rs index 2e285e3169..f7744ba489 100644 --- a/crates/rocm-core/src/process.rs +++ b/crates/rocm-core/src/process.rs @@ -363,24 +363,204 @@ pub fn terminate_process_tree(pid: u32) -> Result<()> { terminate_process(pid) } +/// What one look at a PID could establish. +/// +/// [`process_is_running`] answers a plain yes or no, so a process that is alive +/// but cannot be queried reads the same as one that is gone. A stop that is about +/// to report a process as exited must not make that mistake, so this keeps the +/// third answer apart. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ProcessProbe { + /// The PID names a process that has not exited. + Running, + /// The PID names no process, or one that has exited. + Gone, + /// The PID could not be queried, so it can be neither confirmed running nor + /// confirmed gone. On Windows that includes a live process the caller may not + /// open (`ERROR_ACCESS_DENIED`). + /// + /// The cost of reading this as "may be running" is that a stopped service's + /// recorded PID, taken since by a process the caller cannot open, keeps its + /// record live: the record never reconciles to `stopped`, `services remove` + /// and `prune` refuse it, and a stop reports it unverified. That is the safe + /// direction (the alternative deletes the key and record under a process that + /// may be serving), and there is no better evidence to be had: the creation + /// time that would tell the two processes apart is read with the same access + /// right, so a process that cannot be opened has no readable token to + /// compare. `rocm services remove --force --yes` clears such a record + /// (see `docs/manual-testing.md`, "A record that stays unverified"); it + /// applies only when no recorded process is confirmed running, one could not + /// be queried, and a stop was requested. + Unqueryable, +} + +impl ProcessProbe { + /// Whether the process may still be running: everything but a confirmed + /// exit. The answer for a decision that must not treat a process it could + /// not look at as gone, such as dropping a service's endpoint key, marking + /// its record stopped or deleting it. + #[must_use] + pub const fn may_be_running(self) -> bool { + !matches!(self, Self::Gone) + } +} + +/// Windows `ERROR_INVALID_PARAMETER`: what `OpenProcess` sets for a PID that +/// names no process. +#[cfg(any(windows, test))] +const ERROR_INVALID_PARAMETER: i32 = 87; + +/// Windows `STILL_ACTIVE`: the exit code `GetExitCodeProcess` reports for a +/// process that has not exited. +#[cfg(any(windows, test))] +const STILL_ACTIVE: u32 = 259; + +/// What `GetExitCodeProcess` on a handle that opened says about the process: +/// still active is running, any other exit code is gone, and a failed query is +/// neither. Split out of the FFI, with the open-failure mapping above, so a test +/// runs the whole decision on every host. +#[cfg(any(windows, test))] +pub(crate) const fn classify_exit_query(queried: bool, exit_code: u32) -> ProcessProbe { + match (queried, exit_code) { + (false, _) => ProcessProbe::Unqueryable, + (true, STILL_ACTIVE) => ProcessProbe::Running, + (true, _) => ProcessProbe::Gone, + } +} + +/// What a failed `OpenProcess` says about the PID, from its `GetLastError` code. +/// +/// Only the code for "no such process" counts as gone. `ERROR_ACCESS_DENIED` +/// (5), which a live process the caller may not open sets, and any other or +/// missing code are [`ProcessProbe::Unqueryable`]: a failure to look is not an +/// exit. Split out of the FFI so a test runs it on every host. +#[cfg(any(windows, test))] +pub(crate) const fn classify_open_failure(code: Option) -> ProcessProbe { + match code { + Some(ERROR_INVALID_PARAMETER) => ProcessProbe::Gone, + _ => ProcessProbe::Unqueryable, + } +} + +/// The path of a Windows system tool in the system directory (`System32`). +/// +/// The directory comes from `GetSystemDirectoryW`, not from the environment, so +/// neither a directory earlier on `PATH` nor a `SystemRoot` the caller controls +/// can stand in for it; the bare name when the directory cannot be read (and off +/// Windows). `name` includes the extension (`taskkill.exe`). +#[must_use] +pub fn windows_system_tool(name: &str) -> PathBuf { + system_tool_path(windows_system_directory(), name) +} + +/// [`windows_system_tool`] for a given system directory, so a test builds the +/// path on every host. +fn system_tool_path(system_dir: Option, name: &str) -> PathBuf { + match system_dir.filter(|dir| !dir.as_os_str().is_empty()) { + Some(dir) => dir.join(name), + None => PathBuf::from(name), + } +} + +/// The Windows system directory, read with `GetSystemDirectoryW`. #[cfg(windows)] #[allow(unsafe_code)] // Win32 FFI -pub fn process_is_running(pid: u32) -> bool { +fn windows_system_directory() -> Option { + use std::os::windows::ffi::OsStringExt; + use windows_sys::Win32::System::SystemInformation::GetSystemDirectoryW; + + const MAX_PATH: u32 = 260; + let mut buffer = [0_u16; MAX_PATH as usize]; + // SAFETY: `buffer` is a live local valid for `MAX_PATH` wide characters, and + // that is the size passed, so the call cannot write past it. + let length = unsafe { GetSystemDirectoryW(buffer.as_mut_ptr(), MAX_PATH) } as usize; + // Zero is a failure; a length that does not fit means the path was too long + // for the buffer, which is not a directory to trust either. + (length > 0 && length < buffer.len()) + .then(|| PathBuf::from(std::ffi::OsString::from_wide(&buffer[..length]))) +} + +#[cfg(not(windows))] +#[allow(clippy::missing_const_for_fn)] // keeps the signature of the Windows arm +fn windows_system_directory() -> Option { + None +} + +/// E2E hook: `ROCM_E2E_UNQUERYABLE_PIDS`, a comma-separated list of PIDs, makes +/// [`probe_process`] answer [`ProcessProbe::Unqueryable`] for each of them. +/// +/// A live process this account may not open cannot be staged from outside the +/// binary, and without it the stop, the liveness refresh and `services remove` +/// have no live coverage for that case off Windows. Compiled only into the +/// `e2e-test-hooks` build; production builds never read the variable. +#[cfg(feature = "e2e-test-hooks")] +fn scripted_probe(pid: u32) -> Option { + let listed = std::env::var("ROCM_E2E_UNQUERYABLE_PIDS").ok()?; + listed + .split(',') + .filter_map(|entry| entry.trim().parse::().ok()) + .any(|listed| listed == pid) + .then_some(ProcessProbe::Unqueryable) +} + +/// Look at `pid`, keeping a process that cannot be queried apart from one that +/// is gone. See [`ProcessProbe`]. +#[cfg(windows)] +#[must_use] +#[allow(unsafe_code)] // Win32 FFI +pub fn probe_process(pid: u32) -> ProcessProbe { use windows_sys::Win32::Foundation::CloseHandle; + #[cfg(feature = "e2e-test-hooks")] + if let Some(scripted) = scripted_probe(pid) { + return scripted; + } if pid == 0 { - return false; + return ProcessProbe::Gone; } + // SAFETY: `OpenProcess` takes only plain values and returns a handle or null. let handle = unsafe { OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, 0, pid) }; if handle.is_null() { - return false; + return classify_open_failure(std::io::Error::last_os_error().raw_os_error()); } let mut exit_code = 0; - let ok = unsafe { GetExitCodeProcess(handle, &raw mut exit_code) != 0 }; + // SAFETY: `handle` is the non-null handle opened above, still open, and + // `exit_code` is a live local that is valid for this one call to write to. + let queried = unsafe { GetExitCodeProcess(handle, &raw mut exit_code) != 0 }; + // SAFETY: `handle` is the handle opened above, closed here exactly once and + // not used afterwards. unsafe { CloseHandle(handle); } - ok && exit_code == 259 + classify_exit_query(queried, exit_code) +} + +/// Look at `pid`. `kill(pid, 0)` already tells a live process the caller may +/// not signal (`EPERM`, which counts as running) from one that is gone, so there +/// is no third answer to keep apart here. +#[cfg(not(windows))] +#[must_use] +pub fn probe_process(pid: u32) -> ProcessProbe { + #[cfg(feature = "e2e-test-hooks")] + if let Some(scripted) = scripted_probe(pid) { + return scripted; + } + if process_is_running(pid) { + ProcessProbe::Running + } else { + ProcessProbe::Gone + } +} + +/// Whether `pid` names a running process. +/// +/// On Windows a live process this one may not open reads as not running here; a +/// decision that must not take it for gone (dropping a key, marking a record +/// stopped, deleting it) asks [`probe_process`] instead and keeps +/// [`ProcessProbe::Unqueryable`] apart. +#[cfg(windows)] +pub fn process_is_running(pid: u32) -> bool { + probe_process(pid) == ProcessProbe::Running } #[cfg(not(windows))] @@ -1261,6 +1441,72 @@ mod tests { use crate::test_support::workspace_test_artifact_dir; use std::process::Stdio; + /// A system tool is started from the system directory, never by a bare name + /// that `PATH` could resolve to something else; with no directory it falls + /// back to the bare name. + #[test] + fn a_system_tool_is_resolved_under_the_system_directory() { + let dir = PathBuf::from("C:\\Windows").join("System32"); + let path = system_tool_path(Some(dir.clone()), "taskkill.exe"); + assert_eq!(path, dir.join("taskkill.exe")); + assert!(path.ends_with(Path::new("System32").join("taskkill.exe"))); + assert!(!path.parent().unwrap().as_os_str().is_empty()); + + assert_eq!( + system_tool_path(None, "tasklist.exe"), + PathBuf::from("tasklist.exe") + ); + assert_eq!( + system_tool_path(Some(PathBuf::new()), "tasklist.exe"), + PathBuf::from("tasklist.exe"), + "an empty directory is no directory" + ); + } + + /// A failed `OpenProcess` is a gone process only for the code that means + /// "no such process". Access denied, which a live process the caller may not + /// open sets, and any code that is not known are not an exit. + #[test] + fn a_failed_open_is_gone_only_for_no_such_process() { + const ERROR_ACCESS_DENIED: i32 = 5; + assert_eq!(classify_open_failure(Some(87)), ProcessProbe::Gone); + assert_eq!( + classify_open_failure(Some(ERROR_ACCESS_DENIED)), + ProcessProbe::Unqueryable + ); + // The codes around the two that matter, and no code at all. + for code in [Some(0), Some(6), Some(31), Some(299), Some(-1), None] { + assert_eq!( + classify_open_failure(code), + ProcessProbe::Unqueryable, + "{code:?}" + ); + } + } + + /// An opened handle's exit code: still active is running, any other code is + /// gone, and a failed query (whatever the code it left behind) is neither. + #[test] + fn an_exit_code_query_is_running_gone_or_unqueryable() { + assert_eq!(classify_exit_query(true, 259), ProcessProbe::Running); + assert_eq!(classify_exit_query(true, 0), ProcessProbe::Gone); + assert_eq!(classify_exit_query(true, 1), ProcessProbe::Gone); + assert_eq!(classify_exit_query(false, 0), ProcessProbe::Unqueryable); + assert_eq!(classify_exit_query(false, 259), ProcessProbe::Unqueryable); + } + + /// Off Windows, a PID that is running (including one the caller may not + /// signal) probes as running and an absent one as gone, matching + /// [`process_is_running`]. + #[cfg(not(windows))] + #[test] + fn the_probe_agrees_with_process_is_running_off_windows() { + let own = std::process::id(); + assert_eq!(probe_process(own), ProcessProbe::Running); + assert_eq!(probe_process(0), ProcessProbe::Gone); + assert_eq!(probe_process(999_999_998), ProcessProbe::Gone); + } + const LOCK_CHILD_PATH_ENV: &str = "ROCM_CORE_TEST_LOCK_PATH"; const LOCK_CHILD_READY_ENV: &str = "ROCM_CORE_TEST_LOCK_READY"; const HANDOFF_PORT_ENV: &str = "ROCM_CORE_TEST_HANDOFF_PORT"; diff --git a/crates/rocm-core/src/service_record.rs b/crates/rocm-core/src/service_record.rs index eea3a73715..7d1ec24571 100644 --- a/crates/rocm-core/src/service_record.rs +++ b/crates/rocm-core/src/service_record.rs @@ -109,6 +109,33 @@ pub struct ManagedServiceRecord { /// existed, which is the safe default (keep the key). #[serde(default)] pub stop_requested_unix_ms: Option, + /// The stop marker a liveness refresh cleared off a record that named no + /// process, kept as the evidence that it was cleared and not by a relaunch. + /// + /// A record naming no process has nothing for a pending stop to wait on, so + /// the refresh drops the marker (and the key) at once. A `starting` or + /// `failed` record is left at that status, and a stop that began on such a + /// record would then find it with no marker and nothing recorded, which is + /// also what a relaunch that has not recorded its supervisor yet looks like. + /// This holds the marker's value (the stop's own request stamp), so the stop + /// can tell its own marker having been consumed from a relaunch having + /// cleared it ([`Self::is_covered_by_stop_of`]). Cleared by a restart + /// ([`Self::reset_for_restart`]); absent from records that never had one. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub settled_stop_unix_ms: Option, + /// The generation of a relaunch that has begun but not yet published its + /// processes: stamped by `rocm services restart` once its own stop has + /// confirmed and before it restores the endpoint key or spawns, and cleared + /// when the restart publishes ([`Self::reset_for_restart`]). + /// + /// Until then the record is an empty `stopped` manifest, which is also what a + /// stop's own completion looks like, so a stop that read the record earlier + /// would claim it and delete the key the restart restored for its child. A + /// stop covers the record only while the record carries the generation the + /// stop read ([`Self::is_covered_by_stop_of`]). Absent from records that are + /// not between a restart's stop and its publication. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub relaunch_started_unix_ms: Option, /// When the last inference probe was attempted. Throttles re-probing of a /// service that is listed but still loading — see /// [`crate::INFERENCE_PROBE_RETRY_INTERVAL`]. Absent on records written before @@ -254,6 +281,8 @@ impl ManagedServiceRecord { restart_count: 0, last_restart_unix_ms: None, stop_requested_unix_ms: None, + settled_stop_unix_ms: None, + relaunch_started_unix_ms: None, startup_phase: None, inference_verified_at_unix_ms: None, inference_probe_attempted_at_unix_ms: None, @@ -298,6 +327,10 @@ impl ManagedServiceRecord { // died with the previous run. self.anthropic_endpoint_url = None; self.anthropic_endpoint_mode = None; + // A relaunch is no longer the stop that was settled under it, and it is + // published now, so it is no longer waiting to be. + self.settled_stop_unix_ms = None; + self.relaunch_started_unix_ms = None; self.restart_count = self.restart_count.saturating_add(1); self.last_restart_unix_ms = Some(unix_time_millis()); } @@ -330,6 +363,74 @@ impl ManagedServiceRecord { && self.engine_start_ticks == other.engine_start_ticks } + /// Whether `self`, the record as it is now, is still the record of the + /// service that `stopping`, the copy the stop read before it signalled + /// anything, was stopping: the stop's verdict covers what it names. + /// + /// True when the two name the same processes + /// ([`Self::names_same_processes`]), and also when `self` names none because + /// the same stop was completed under it: a liveness refresh that saw every + /// recorded process gone, with the stop requested, clears the processes and + /// reads the record as `stopped` (the deferred cleanup). A record reading + /// `stopped` that names no process has nothing left that the stop did not + /// handle, so the stop's own confirmation is still true of it. + /// + /// False for a restart that recorded new processes, and equally for one that + /// is mid-launch with none recorded yet: that record reads `starting` (or + /// another live status) with the stop marker cleared, not `stopped`, and + /// claiming it stopped, or dropping the key it will serve with, would be + /// false. That holds when the stopping snapshot itself names no process too: + /// an empty record is covered only while it still carries the stop's marker, + /// once it reads `stopped`, or once a liveness refresh consumed that very + /// marker ([`Self::settled_stop_unix_ms`]), which leaves a `starting` or + /// `failed` record at its status with the marker gone. + /// + /// Never true for a record in another relaunch generation than the stop read + /// ([`Self::relaunch_started_unix_ms`]): a restart that has stopped the + /// service and not yet published its child leaves an empty `stopped` record + /// that an older stop would otherwise take for its own completion. + #[must_use] + pub fn is_covered_by_stop_of(&self, stopping: &Self) -> bool { + if self.relaunch_started_unix_ms != stopping.relaunch_started_unix_ms { + return false; + } + if self.status == "stopped" && self.names_no_process() { + return true; + } + // The same processes. When they are none, that equality is also what a + // relaunch that has not recorded its supervisor yet looks like, so it + // proves nothing: the stop's own marker is what tells the two apart, since + // a restart clears it and a stop that is still finalizing has it on disk. + // A marker that is gone is the stop's own only when the refresh that + // consumed it left its stamp behind; a restart wipes that stamp. + self.names_same_processes(stopping) + && (!stopping.names_no_process() + || self.stop_requested_unix_ms.is_some() + || (stopping.stop_requested_unix_ms.is_some() + && self.settled_stop_unix_ms == stopping.stop_requested_unix_ms)) + } + + /// Stamp the record as a relaunch that has begun and not yet published its + /// processes, as a generation no earlier stop or relaunch carried. See + /// [`Self::relaunch_started_unix_ms`]. + pub fn begin_relaunch(&mut self) { + let now = unix_time_millis(); + self.relaunch_started_unix_ms = Some( + now.max( + self.relaunch_started_unix_ms + .map_or(0, |earlier| earlier + 1), + ), + ); + } + + /// Whether the record names neither a supervisor nor an engine process. + const fn names_no_process(&self) -> bool { + self.supervisor_pid == 0 + && self.supervisor_start_ticks.is_none() + && self.engine_pid.is_none() + && self.engine_start_ticks.is_none() + } + /// Record `pid` as the engine, together with its start-time token. See /// [`Self::record_supervisor_identity`]. pub fn record_engine_identity(&mut self, pid: u32) { @@ -337,6 +438,25 @@ impl ManagedServiceRecord { self.engine_start_ticks = crate::process_start_ticks(pid); } + /// Forget the processes this record names: the supervisor and the engine, + /// each with its start-time token. + /// + /// For a stop that has confirmed every one of them gone, and for nothing + /// else. A recorded PID goes stale the instant its process exits, and a + /// later stop of the same record would signal whatever the OS has since + /// handed that number to; a PID recorded without a token cannot be + /// refuted, and on Windows none can, so the tree kill would take that + /// process and everything under it. `rocm services stop` and `rocmd`'s stop + /// both call this on the confirmed path, so the two leave the same record + /// behind. An unconfirmed stop keeps the PIDs: the record is the only thing + /// that names a survivor. + pub const fn clear_recorded_processes(&mut self) { + self.supervisor_pid = 0; + self.supervisor_start_ticks = None; + self.engine_pid = None; + self.engine_start_ticks = None; + } + pub fn normalize_paths_for_host(&mut self) { self.manifest_path = normalize_runtime_path_for_host(&self.manifest_path); self.log_path = normalize_runtime_path_for_host(&self.log_path); @@ -778,7 +898,8 @@ mod tests { /// been recycled. One live process stands in for both roles, as it does for /// a `rocm serve --background` launch. /// - /// Linux-only: `process_start_ticks` reads `/proc`, and is `None` elsewhere. + /// Linux-only here, because it spawns `sleep`: `process_start_ticks` gives a + /// token on Linux and on Windows (the creation time) and is `None` elsewhere. #[cfg(target_os = "linux")] #[test] fn recording_a_process_identity_writes_the_pid_with_its_own_token() { @@ -802,6 +923,230 @@ mod tests { assert_eq!(record.engine_start_ticks, expected); } + /// What a stop that confirmed its processes gone leaves behind: no PID and + /// no token for either role, and nothing else touched. + #[test] + fn clearing_the_recorded_processes_clears_each_pid_and_its_token_and_nothing_else() { + let mut record = probe_test_record(11437); + record.supervisor_pid = 4242; + record.supervisor_start_ticks = Some(5); + record.engine_pid = Some(4243); + record.engine_start_ticks = Some(77); + record.status = "ready".to_owned(); + record.restart_count = 3; + let before = record.clone(); + + record.clear_recorded_processes(); + + assert_eq!(record.supervisor_pid, 0); + assert_eq!(record.supervisor_start_ticks, None); + assert_eq!(record.engine_pid, None); + assert_eq!(record.engine_start_ticks, None); + assert_eq!(record.service_id, before.service_id); + assert_eq!(record.status, "ready"); + assert_eq!(record.restart_count, 3); + assert_eq!(record.port, before.port); + // A cleared record no longer names what it named, so a stop that read it + // before and finds this after does not claim the same processes. + assert!(!before.names_same_processes(&record)); + assert!(record.names_same_processes(&record.clone())); + } + + /// What a stop's verdict still covers, record by record: the same processes, + /// or none because the same stop was completed under it (`stopped`, nothing + /// recorded). A restart's record is never covered: new processes recorded, or + /// still starting with none recorded yet. + #[test] + fn a_stop_is_covered_by_the_same_processes_or_by_its_own_completion_and_not_by_a_relaunch() { + let mut stopping = probe_test_record(11438); + stopping.supervisor_pid = 4242; + stopping.supervisor_start_ticks = None; + stopping.engine_pid = Some(4243); + stopping.engine_start_ticks = Some(77); + stopping.status = "ready".to_owned(); + let record_with = |status: &str, pid: u32, engine: Option| { + let mut record = stopping.clone(); + status.clone_into(&mut record.status); + record.supervisor_pid = pid; + record.supervisor_start_ticks = None; + record.engine_pid = engine; + record.engine_start_ticks = None; + record + }; + // (what the record on disk is now, whether the stop still covers it) + let rows = [ + ("unchanged", stopping.clone(), true), + // The deferred cleanup of this very stop: every process gone, so the + // record is stopped and names none. + ( + "completed under the stop", + record_with("stopped", 0, None), + true, + ), + // A restart that recorded new processes. + ( + "relaunched, ready", + record_with("ready", 5555, Some(5556)), + false, + ), + ( + "relaunched, starting", + record_with("starting", 5555, None), + false, + ), + // A restart mid-launch that has not recorded anything yet: not a stop. + ( + "restarting, nothing recorded", + record_with("starting", 0, None), + false, + ), + ( + "recovering, nothing recorded", + record_with("recovering", 0, None), + false, + ), + ( + "ready, nothing recorded", + record_with("ready", 0, None), + false, + ), + // Stopped, but naming a process the stop did not handle. + ( + "stopped, other process", + record_with("stopped", 5555, None), + false, + ), + ]; + for (name, record, covered) in rows { + assert_eq!(record.is_covered_by_stop_of(&stopping), covered, "{name}"); + } + + // The same table from a stopping snapshot that names no process: an + // untouched record (the stop's own marker still on it, or completed to + // `stopped`) is that stop; an empty relaunch that has cleared the marker is + // not, and neither is one that has recorded processes. + let mut empty = stopping.clone(); + empty.clear_recorded_processes(); + empty.stop_requested_unix_ms = Some(1); + let with_marker = |status: &str, marker: Option, pid: u32| { + let mut record = empty.clone(); + status.clone_into(&mut record.status); + record.stop_requested_unix_ms = marker; + record.supervisor_pid = pid; + record + }; + // What a liveness refresh leaves when it consumes the stop's marker off an + // empty record: the marker gone, the status as it was, the marker's value + // kept as `settled_stop_unix_ms`. + let settled = |status: &str, stamp: Option| { + let mut record = with_marker(status, None, 0); + record.settled_stop_unix_ms = stamp; + record + }; + let rows = [ + ("empty, untouched", with_marker("ready", Some(1), 0), true), + ("empty, stopped", with_marker("stopped", None, 0), true), + // The stop's own marker consumed by a listing: not a relaunch. + ( + "empty, starting, marker consumed by a listing", + settled("starting", Some(1)), + true, + ), + ( + "empty, failed, marker consumed by a listing", + settled("failed", Some(1)), + true, + ), + // A stamp that is not this stop's (an earlier stop's), or none at all, + // is no evidence about this one. + ( + "empty, starting, another stop's marker consumed", + settled("starting", Some(2)), + false, + ), + ( + "empty, starting relaunch", + with_marker("starting", None, 0), + false, + ), + ( + "empty, recovering relaunch", + with_marker("recovering", None, 0), + false, + ), + ( + "empty, ready relaunch", + with_marker("ready", None, 0), + false, + ), + ( + "relaunched with a process", + with_marker("starting", None, 5555), + false, + ), + ]; + for (name, record, covered) in rows { + assert_eq!(record.is_covered_by_stop_of(&empty), covered, "{name}"); + } + + // Nothing is learned from two absent stamps: a snapshot without a marker + // of its own is not matched by a record that never consumed one. + let mut unmarked = empty.clone(); + unmarked.stop_requested_unix_ms = None; + assert!( + !settled("starting", None).is_covered_by_stop_of(&unmarked), + "None must not equal None as evidence" + ); + } + + /// A restart between its own confirmed stop and the publication of its child + /// leaves an empty `stopped` record. A stop that read the record before that + /// (another generation) must not cover it, however the rest of the record + /// reads, while a stop that read the same generation still does. + #[test] + fn a_stop_does_not_cover_a_record_in_another_relaunch_generation() { + let mut stopping = probe_test_record(11440); + stopping.supervisor_pid = 4242; + stopping.status = "ready".to_owned(); + let mut relaunching = stopping.clone(); + relaunching.clear_recorded_processes(); + "stopped".clone_into(&mut relaunching.status); + assert!( + relaunching.is_covered_by_stop_of(&stopping), + "premise: without a relaunch the completed stop covers it" + ); + relaunching.begin_relaunch(); + assert!( + !relaunching.is_covered_by_stop_of(&stopping), + "an older stop does not cover a record whose relaunch has begun" + ); + // A stop that read the record during the relaunch, and an empty snapshot + // of the same generation, are the same stop as before. + let mut during = relaunching.clone(); + during.stop_requested_unix_ms = Some(9); + assert!(relaunching.is_covered_by_stop_of(&during)); + // The generation is never reused, even within one millisecond. + let first = relaunching.relaunch_started_unix_ms; + relaunching.begin_relaunch(); + assert!(relaunching.relaunch_started_unix_ms > first); + } + + /// A restart supersedes the stop that was settled under it: the stamp a + /// listing left behind does not outlive the relaunch, or a stop that began + /// before it would read the relaunch's empty record as its own completion. + #[test] + fn restarting_forgets_a_settled_stop() { + let mut record = probe_test_record(11439); + record.settled_stop_unix_ms = Some(7); + record.begin_relaunch(); + record.reset_for_restart(); + assert_eq!(record.settled_stop_unix_ms, None); + assert_eq!( + record.relaunch_started_unix_ms, None, + "publishing ends the relaunch generation" + ); + } + /// Writing a service manifest never writes through a file another writer /// already staged next to it. The occupied names are the ones the retired /// clock-and-pid temp naming would have picked, so a write that still used diff --git a/docs/architecture.md b/docs/architecture.md index 581107ad44..124a86c6e7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -44,7 +44,8 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`chat/intent.rs`](../apps/rocm/src/chat/intent.rs) — mechanical relocation out of `chat.rs`, a leaf child of `chat.rs`: the prompt-intent heuristics, which decide from a user's latest message alone what it asks for. `fallback_rocm_tool_call_for_prompt` maps a prompt to the read-only or approval-gated tool call the assistant proposes without asking the local model (with `fallback_config_tool_call_for_prompt`, `fallback_engine_list_tool_call` and `fallback_services_list_tool_call` behind it), `prompt_can_use_read_only_without_local_assistant` says whether a status question can be answered before any model server is up, and `install_sdk_chat_approval_for_prompt` (with `install_sdk_without_prefix_chat_approval`) turns an install or setup request into the `rocm install sdk` approval request. The phrase lists live in the `prompt_asks_running_or_status`, `prompt_asks_engine_install_state`, `prompt_mentions_serving_engine_or_service`, `prompt_requests_rocm_install_or_setup` and `prompt_requests_install_action` predicates, and `latest_user_chat_message` cuts the newest message out of the conversation text the chat builds around it, so older words never steer these heuristics; it moved here with the heuristics because three of them call it, and leaving it with the turn loop would make the two modules call each other. Ten of the functions are `pub(super)`, called from `chat/turn.rs`; the other three are private. Its edges out are [`prompt_text.rs`](../apps/rocm/src/prompt_text.rs), [`rocm_argv.rs`](../apps/rocm/src/rocm_argv.rs), `tool_policy.rs` (`ChatToolApprovalRequest`), `freeform_planner.rs` (`format_structured_tool_call`) and `providers` (the tool-call type); nothing in it calls back into `chat/turn.rs` or `chat.rs`. Its tests, the `fallback_tool_call_*` family and the install-intent checks that call the heuristics directly, live in its own `mod tests`, with their original names. - [`chat_cmd.rs`](../apps/rocm/src/chat_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm chat`'s command handler (`run_chat_command`, the body of the `Command::Chat` arm of `dispatch()`), which hands off to the dashboard chat (`dash::run_chat`) in a terminal with no `--prompt` and otherwise prints the reply or status screen through `chat::render_chat_prompt_text`/`chat::render_chat_text`. It is a module of its own, not part of [`chat.rs`](../apps/rocm/src/chat.rs), because it calls into both `chat.rs` and [`dash.rs`](../apps/rocm/src/dash.rs) while `dash.rs` already calls into `chat.rs`; this module depends on both and neither depends on it. `Provider` lives in [`cli.rs`](../apps/rocm/src/cli.rs) and `ChatInferenceParams` stays in [`main.rs`](../apps/rocm/src/main.rs); `run_chat_command` is `pub(crate)` and re-imported there for `dispatch()`. Four source-text tests in this module's own `mod tests` read this file and assert on the handler's text, so a rename of the function or of the calls they look for has to be made there too. - [`chat_host_facts.rs`](../apps/rocm/src/chat_host_facts.rs) — full domain extraction. -- [`cli.rs`](../apps/rocm/src/cli.rs) — the `rocm` command line definition: `Cli`, `Command`, the subcommand enums nested under it (`BenchCommand`, `InstallTarget`, `EnginesCommand`, `RuntimesCommand`, `StorageCommand`, `ComfyuiCommand`, `ServicesCommand`, `AutomationsCommand`, `ConfigCommand`, `SetupCommand`) and the clap value enums behind `--provider`, `--device`, `--framework` and similar arguments (`Provider`, `DevicePolicyArg`, `FrameworkArg`, `InstallFormat`, `WatcherModeArg`, `TelemetryModeArg`, `SandboxToolArg`, `PermissionsModeArg`), with `SUPPORTED_ENGINES` (the `--engine` choices) and the conversions from a value enum to its domain type. Like [`apps/rocmd`](../apps/rocmd/src/cli.rs)'s `cli.rs`, it holds definitions only: the `dispatch()` that matches on `Command` stays in [`main.rs`](../apps/rocm/src/main.rs). Declaration order sets the order of every nested `--help` command list (the top-level list is sorted alphabetically by `cli_command()` in `main.rs`), so reordering variants reorders help output. Its items are `pub(crate)` and re-imported in `main.rs`, so other modules keep reaching them as `crate::Command`, `crate::Provider` and so on; [`bootstrap.rs`](../apps/rocm/src/bootstrap.rs) and [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs) own their enums and appear in `Command` as a single variant each. The tests that parse the tree or render its help live in its own `mod tests`. +- [`cli.rs`](../apps/rocm/src/cli.rs) — the `rocm` command line definition: `Cli`, `Command`, the subcommand enums nested under it (`BenchCommand`, `InstallTarget`, `EnginesCommand`, `RuntimesCommand`, `StorageCommand`, `ComfyuiCommand`, `ServicesCommand`, `AutomationsCommand`, `ConfigCommand`, `SetupCommand`) and re-exports the clap value enums behind `--provider`, `--device`, `--framework` and similar arguments, which [`cli_args.rs`](../apps/rocm/src/cli_args.rs) defines, with `SUPPORTED_ENGINES` (the `--engine` choices). Like [`apps/rocmd`](../apps/rocmd/src/cli.rs)'s `cli.rs`, it holds definitions only: the `dispatch()` that matches on `Command` stays in [`main.rs`](../apps/rocm/src/main.rs). Declaration order sets the order of every nested `--help` command list (the top-level list is sorted alphabetically by `cli_command()` in `main.rs`), so reordering variants reorders help output. Its items are `pub(crate)` and re-imported in `main.rs`, so other modules keep reaching them as `crate::Command`, `crate::Provider` and so on; [`bootstrap.rs`](../apps/rocm/src/bootstrap.rs) and [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs) own their enums and appear in `Command` as a single variant each. The tests that parse the tree or render its help live in its own `mod tests`. +- [`cli_args.rs`](../apps/rocm/src/cli_args.rs) — mechanical relocation out of [`cli.rs`](../apps/rocm/src/cli.rs), owns types: the clap value enums behind `--provider`, `--device`, `--framework` and similar arguments (`Provider`, `DevicePolicyArg`, `FrameworkArg`, `InstallFormat`, `WatcherModeArg`, `TelemetryModeArg`, `SandboxToolArg`, `PermissionsModeArg`) and the conversions from a value enum to its domain type. `cli.rs` re-exports every one, so other modules keep reaching them as `crate::cli::Provider` and so on. Moved so `cli.rs`, which every new subcommand grows, stays under the module size limit. - [`cli_audit.rs`](../apps/rocm/src/cli_audit.rs) — mechanical relocation, no owned types: the CLI audit and lifecycle log (`record_cli_audit_event`, plus the helpers behind it that write `logs/cli-lifecycle.log` and the per-action files under `logs/cli/`). Its production code depends only on `rocm-core`. `record_cli_audit_event`, `cli_lifecycle_log_path` and `sanitize_log_value` are `pub(crate)` and reached from other modules as `crate::` through a re-import in [`main.rs`](../apps/rocm/src/main.rs); every other item is private. The two regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::`. - [`cli_progress.rs`](../apps/rocm/src/cli_progress.rs) — shared CLI-output component (`Spinner`, `AnimatedSpinner`). - [`cli_report.rs`](../apps/rocm/src/cli_report.rs) — shared CLI-output component (`ActionReport`, `write_marked_row`, `JsonOrText`/`print_json_or_text`). @@ -70,7 +71,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-214): `--device`/`--gpu` parsing (`parse_device_policy`, `parse_gpu_selection`, `parse_gpu_indices_arg`), the managed-launch-lock GPU selection (`select_gpu_indices_under_launch_lock`, `validate_pinned_gpu_index`, `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `busy_gpu_indices`), per-GPU VRAM telemetry (the `GpuVramUsage` type, `gpu_vram_usage`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs`, `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`), and the serve-plan low-VRAM warning (`gpu_low_memory_warning`, `vram_capacity_is_meaningful`, `serve_gpu_low_memory_warning`), plus the best-effort `amd-smi`-based GPU count (`detect_gpu_count`).Both `amd-smi` probes go through `rocm_core::amd_smi_json` (the `/dev/kfd` pre-flight) with this module's own `AMD_SMI_SERVE_PROBE_TIMEOUT`, whose doc records the resulting worst-case `serve` latency. `const fn device_policy_name` sits immediately after this cluster's last function in the original file but stayed in `main.rs` rather than moving — it already had callers outside the cluster (`engine_ipc.rs`, `service_launch.rs`, `serve_cmd.rs`) plus its own test. `GpuVramUsage` (and its fields), `gpu_vram_usage`, `parse_gpu_indices_arg`, `serve_gpu_low_memory_warning`, and `vram_capacity_is_meaningful` are `pub(crate)` and read from [`model_fit.rs`](../apps/rocm/src/model_fit.rs) (`classify_accelerator_memory`, `host_accelerator_memory`) [`main.rs`](../apps/rocm/src/main.rs) (the `dispatch()` `Command::EngineServeHttp` arm) and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) (`collect_serve_notes`); `detect_gpu_count`, `gpu_vram_usage`, `parse_device_policy`, `parse_gpu_selection`, `select_gpu_indices_under_launch_lock`, `serve_gpu_low_memory_warning`, and `validate_pinned_gpu_index` are also `pub(crate)` and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs); `parse_device_policy` is also read from [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs)'s `run_builtin_engine_serve_http` and, as of Phase 6f (ROCMAI-209), from [`service_control.rs`](../apps/rocm/src/service_control.rs)'s `restart_child_argv` (moved out of `main.rs` in that same phase, and since relocated from `services_text.rs`). `effective_gpu_count`, `auto_select_gpu_indices`, `select_auto_gpu_index`, `simulated_low_vram_usage`, `gpu_vram_usage_amd_smi`, `gpu_vram_usage_sysfs` (both the Linux and non-Linux `cfg` variants), `read_drm_vram_usage`, `read_sysfs_u64`, `drm_device_is_amd`, `parse_gpu_vram_usage`, `gpu_low_memory_warning`, and `busy_gpu_indices` have no callers outside this module and stay private. The cluster's regression tests live in `gpu_policy.rs`'s own `mod tests`; two small test-only fixture builders (`vram`, `host_gpu`) are duplicated rather than shared, since `serve_cmd.rs`'s own `serve_notes_pair_low_vram_with_the_vllm_utilization_hint` (testing `collect_serve_notes`) needs them too. - [`install_cmd.rs`](../apps/rocm/src/install_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm install`'s command handler (`install()`, for the `sdk` and `driver` targets), the `--version`/`--build-date` selector parsing and display it uses (`therock_install_version_selector`, `therock_install_version_selector_display`) and `sdk_install_request`, which maps the parsed `install sdk` arguments onto a `therock::SdkInstallRequest` and moved here from [`chat.rs`](../apps/rocm/src/chat.rs) with its two tests. `InstallTarget` and `InstallFormat` live in [`cli.rs`](../apps/rocm/src/cli.rs). `install()` is `pub(crate)` and re-imported in `main.rs` for `dispatch()`; `therock_install_version_selector` is `pub(crate)` too, and `chat/mcp_bridge.rs`'s `render_install_sdk_dry_run_for_args` reaches it as `crate::therock_install_version_selector` through the same kind of re-import, which is why `sdk_install_request` lives here and not in `chat.rs`: this module does not depend on `chat.rs`. `sdk_install_request` keeps the `pub(crate)` it had, although only `install()` and the tests call it now. - [`logging.rs`](../apps/rocm/src/logging.rs) — mechanical relocation, no owned types, reached by qualified path. -- [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-211): managed-service liveness classification (`managed_service_is_live`, `refresh_managed_service_runtime_liveness`, `existing_live_managed_service_in`'s idempotency guard) and the dashboard-sidebar status/counts helpers derived from it (`ManagedServiceSidebarCounts`/`managed_service_sidebar_counts`/`local_server_sidebar_status`), plus the engine display-name helper `friendly_engine_label`, read only by [`chat/summaries.rs`](../apps/rocm/src/chat/summaries.rs). `ManagedServiceSidebarCounts` is `pub(crate)` to match the visibility of the `pub(crate)` functions that return/take it; `friendly_engine_label`, `existing_live_managed_service_in`, `recorded_service_pids`, `refresh_managed_service_runtime_liveness`, `local_server_sidebar_status`, `managed_service_sidebar_counts`, and `load_managed_services` are `pub(crate)` for the same reason — each has a confirmed caller outside this module. [`service_records.rs`](../apps/rocm/src/service_records.rs) uses its liveness helpers (`managed_service_is_live`, `recorded_service_pids`) for the in-use check behind service-record removal and pruning, and `load_managed_services` for prune planning, while [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs) uses `load_managed_services` for the codex-bridge snapshot; [`service_launch.rs`](../apps/rocm/src/service_launch.rs) is the only caller of the `existing_live_managed_service_in` duplicate-launch guard; [`services_text.rs`](../apps/rocm/src/services_text.rs) reads the sidebar helpers; other modules, including [`chat/turn.rs`](../apps/rocm/src/chat/turn.rs), [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs), read the registry and liveness helpers. +- [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) — mechanical relocation, owns types (Phase 6f, ROCMAI-211): managed-service liveness classification (`managed_service_is_live`, `refresh_managed_service_runtime_liveness`, `existing_live_managed_service_in`'s idempotency guard) and the dashboard-sidebar status/counts helpers derived from it (`ManagedServiceSidebarCounts`/`managed_service_sidebar_counts`/`local_server_sidebar_status`), plus the engine display-name helper `friendly_engine_label`, read only by [`chat/summaries.rs`](../apps/rocm/src/chat/summaries.rs). `ManagedServiceSidebarCounts` is `pub(crate)` to match the visibility of the `pub(crate)` functions that return/take it; `friendly_engine_label`, `existing_live_managed_service_in`, `recorded_service_pids`, `refresh_managed_service_runtime_liveness`, `local_server_sidebar_status`, `managed_service_sidebar_counts`, and `load_managed_services` are `pub(crate)` for the same reason — each has a confirmed caller outside this module. [`service_records.rs`](../apps/rocm/src/service_records.rs) uses its liveness helpers (`managed_service_is_live`, `recorded_service_pids`) for the in-use check behind service-record removal and pruning, both of which, like the deferred key cleanup and the reconciliation to `stopped` here, take a recorded process for gone only when `probe_process` confirms it (a process that cannot be queried counts as possibly running; when the deferred cleanup does complete, it also forgets the recorded processes and reads the record as stopped, so a later stop cannot signal a recycled PID; an unconfirmed stop of a container-backed engine, decided by the engine `Stop`, is never completed or read as stopped from the supervisor's PID, and `container_stop_unresolved` makes the in-use check treat it as in use), and `load_managed_services` for prune planning, while [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs) uses `load_managed_services` for the codex-bridge snapshot; [`service_launch.rs`](../apps/rocm/src/service_launch.rs) is the only caller of the `existing_live_managed_service_in` duplicate-launch guard; [`services_text.rs`](../apps/rocm/src/services_text.rs) reads the sidebar helpers; other modules, including [`chat/turn.rs`](../apps/rocm/src/chat/turn.rs), [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs) and [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs), read the registry and liveness helpers. - [`model_catalog_text.rs`](../apps/rocm/src/model_catalog_text.rs) — mechanical relocation out of `examine_report_text.rs`: the `rocm model` (alias `models`) catalog renderer. `render_model_registry_text_with_context_and_host` prints the curated list grouped by hardware target (`model_catalog_platform_heading`, `model_catalog_header`, `model_recipe_memory_label`, `model_recipe_gpu_fit_label`), and `render_model_registry_verbose_text_with_context_and_host` prints the per-recipe diagnostics behind `rocm model --verbose` (`append_model_recipe_registry_source`, `append_model_recipe_metadata_lines`, `append_model_engine_recipe_settings_lines`, `append_model_recipe_staleness_lines`, `append_model_artifact_lines`, `append_artifact_source_policy_lines`, `artifact_source_policy_label`, `append_model_host_ram_fit_lines`, `append_model_fit_lines`, `append_manual_alternative_lines`, `manual_alternative_recommendations`, `recipe_is_manual_fit`, `append_model_engine_support_lines`, `model_registry_adapter_availability_note`, `recipe_display_ref`, `preferred_engine_action_target`, `model_registry_reason`, plus the small formatters `format_list_or_none`, `format_string_map_or_none`, `format_gib` and `once_per_call`). `format_bytes`, the human-readable size formatter, lives here too. The two renderers and `format_bytes` are `pub(crate)`: [`main.rs`](../apps/rocm/src/main.rs) imports them for the `Command::Model` arm, and [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs) (the verbose renderer), [`storage.rs`](../apps/rocm/src/storage.rs) and [`automations.rs`](../apps/rocm/src/automations.rs) (`format_bytes`) reach them as `crate::`. Every other item, including `model_registry_adapter_availability_note`, is private. The module has no edge to `examine_report_text.rs`; it reaches the engine-availability helpers in [`engine_ipc.rs`](../apps/rocm/src/engine_ipc.rs), `PlatformRuleOut` and `engine_ruled_out_by_platform_with` at the crate root, and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs)'s `select_serve_engine`. The catalog's regression tests live in this module's own `mod tests`; they keep the names they had in `examine_report_text.rs`. The `format_bytes` property test's saved failure case lives in [`model_catalog_text.txt`](../apps/rocm/proptest-regressions/model_catalog_text.txt), which proptest finds by the test's source file, so the saved case has to follow the test when it moves (see [Property tests](testing.md#property-tests) for how, which depends on whether the whole file moves). - [`model_fit.rs`](../apps/rocm/src/model_fit.rs) — mechanical relocation, owns a `pub(crate)` type: answering whether a host can run a model for `rocm diagnose --model` (`assess_model_on_this_host`, `assess_model_with_host_paths`, `assess_model_for_host`), the accelerator-memory fact it rests on (`host_accelerator_memory`, `classify_accelerator_memory`, `examination_reports_amd_gpu`) and the platform gate that rules an engine out (`PlatformRuleOut`, `engine_ruled_out_by_platform`, `unsupported_here_for`). It is a module of its own, rather than part of the `diagnose` handler, so [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs) and [`model_catalog_text.rs`](../apps/rocm/src/model_catalog_text.rs), which read `PlatformRuleOut` and the platform gate (`engine_ruled_out_by_platform` and `engine_ruled_out_by_platform_with` respectively), do not depend on a command module. It depends on [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) (`select_serve_engine`) and [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs), neither of which depends on it. `assess_model_on_this_host` (called from [`diagnose_cmd.rs`](../apps/rocm/src/diagnose_cmd.rs)), `PlatformRuleOut` and `engine_ruled_out_by_platform` are `pub(crate)`, and [`main.rs`](../apps/rocm/src/main.rs) re-imports the last two so `examine_report_text.rs` reaches them as `crate::` (`model_catalog_text.rs` reaches `PlatformRuleOut` and `engine_ruled_out_by_platform_with` the same way); every other item is private. The cluster's regression tests live in this module's own `mod tests`; the one that re-runs the test binary on a helper test by its full path (`engine_ruled_out_by_platform_gates_aim_on_docker_reachability`) names the helper's new path, `model_fit::tests::docker_unreachable_subprocess_helper`. - [`prompt_text.rs`](../apps/rocm/src/prompt_text.rs) — text extraction from a natural-language request, shared by `chat/intent.rs`'s prompt-intent heuristics (`fallback_rocm_tool_call_for_prompt`, `fallback_config_tool_call_for_prompt`, `install_sdk_chat_approval_for_prompt` and the `prompt_*` predicates) and the freeform planner, in a leaf module that depends only on `therock::RuntimeVersionSelector` (`tool_policy.rs` does not use it): `any_substring`, the install-folder, TheRock build-date and TheRock version readers (`requested_install_prefix_from_prompt`, `requested_therock_build_date_from_prompt`, `requested_therock_version_from_prompt`, with `requested_bare_install_prefix_from_prompt`, `clean_requested_install_prefix`, `looks_like_user_path` and `looks_like_therock_runtime_version` private beside them) from `chat.rs`, and `infer_engine_from_request` from `freeform_planner.rs`. They moved verbatim, together with `infer_engine_from_request`'s regression test, so that `freeform_planner.rs` no longer imports `chat` for them; `chat.rs` re-exports `any_substring`, which [`freeform_answer.rs`](../apps/rocm/src/freeform_answer.rs) still reaches as `chat::any_substring`. @@ -90,18 +91,18 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) — mechanical relocation, owns types (Phase 6a): `serve()`, plus the engine selection that runs ahead of it (`select_serve_engine`, `recipe_preferred_engine`, `serve_model_ref_for_engine`) and the AIM pre-flight that runs inside it: the Hugging Face token check, the GPU-family profile check and pin, the multi-GPU refusal (`aim_gpu_preflight`), and the board note. The engine-recipe overrides `serve()` applies to the launch hint live in [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs); this module imports them by name. It also holds `collect_serve_notes`, the GPU/device warnings folded into the interactive summary, which was relocated here from `main.rs` with its two tests. Its tests cover engine selection, the AIM pre-flight and the reuse pre-gate; the AIM tests build their hint with `protocol_engine_recipe_hint` and `engine_recipe_with_extra_env` from `serve_recipe.rs`, which `serve()` also calls. - [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs) — mechanical relocation, owns a type (split out of [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), which imports the entry points by name): the pure transforms `serve()` applies to the selected engine's launch recipe hint. `protocol_engine_recipe_hint` maps the catalog's engine record onto the protocol hint, and `render_serve_engine_recipe_lines` prints it for the serve plan. The explicit controls are then layered on top: `--tool-call-parser` (`engine_recipe_with_tool_call_override`), `--gpu-memory-utilization` (`parse_gpu_memory_utilization`, `engine_recipe_with_gpu_memory_utilization_override`), `--temperature`/`--top-p`/`--max-tokens` (`ServeGenerationDefaults`, `engine_recipe_with_generation_defaults`) and `--engine-env` (`engine_recipe_with_extra_env`, `engine_env_ignored_note`, with the env names it refuses to override in `RESERVED_ENGINE_ENV_KEYS`); `engine_recipe_enables_tool_choice` reads the result back. The per-flag rewriters (`set_vllm_tool_call_parser`, `set_vllm_gpu_memory_utilization`, `set_vllm_override_generation_config`, `set_lemonade_generation_defaults`) are private. The entry points above, and `ServeGenerationDefaults`, are `pub(crate)` for `serve()` and for the AIM tests in `serve_cmd.rs`; `RESERVED_ENGINE_ENV_KEYS` is private. Among first-party crates the module depends only on `rocm_core` and `rocm_engine_protocol`, and nothing in it calls back into `serve_cmd.rs`. Its tests cover each override on its own. - [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs) — full domain extraction. -- [`service_control.rs`](../apps/rocm/src/service_control.rs) — mechanical relocation, owns private types: stopping and restarting a managed local server, the process control behind `rocm services stop|restart`, the chat/LLM sandbox tool-calling path and `runtime_services.rs`'s moves onto a new runtime. It holds the sandbox-tool runner (`run_internal_sandbox_tool`), the stop path (`terminate_recorded_service_pids`, the `MANAGED_STOP_GRACE` constant, the `RecordedPidOutcome` and `RecordedStopTermination` types, `scripted_stop_outcome`, the end-to-end hook behind the `e2e-test-hooks` feature, `stop_internal_managed_service`, `stop_internal_managed_service_with`, `stop_internal_managed_service_confirmed_with`, `unload_lemonade_service_model`) and the restart path (`apply_restart_runtime_pin`, `load_service_for_restart`, `restart_child_argv`, `restart_internal_managed_service`, `restart_internal_managed_service_with`, `update_restarting_record`, `record_restarted_child`, `record_restart_readiness`). Only `run_internal_sandbox_tool` and `restart_internal_managed_service` are `pub(crate)`: the first is read from `dispatch()` in [`main.rs`](../apps/rocm/src/main.rs), from [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs)'s `run_approved_service_action` and from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), which passes it to [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s `run_attached_service` as the stop for a Ctrl-C, the second from [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs); each importer reaches them through `crate::service_control` rather than a re-export. Every other item has no caller outside this module and stays private — see [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs)'s and [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s entries above for how those two modules' own doc comments describe this module as the caller of their helpers. The module reads managed-service records through [`services_text.rs`](../apps/rocm/src/services_text.rs) (`load_managed_service`, `read_managed_service_manifest`) and shapes the `stop_server` answer with its `internal_stop_server_value`; `services_text.rs` does not call back into it. This module takes `ensure_public_service_has_endpoint_key` and the spawn helpers from [`service_launch.rs`](../apps/rocm/src/service_launch.rs), which does not import it back. The cluster's regression tests live in this module's own `mod tests`, reaching shared fixtures (`test_paths`, `ScopedTestEnv`, `plant_service_record_on_runtime`, `OLD_RUNTIME_KEY`, `NEW_RUNTIME_KEY`) through `crate::tests::` the same way other mechanically relocated modules do, and `runtime_services::` through its own `use crate::runtime_services;` for the handful of tests that drive a restart through `runtime_services`'s public entry points rather than calling it directly. +- [`service_control.rs`](../apps/rocm/src/service_control.rs) — mechanical relocation, owns private types: stopping and restarting a managed local server, the process control behind `rocm services stop|restart`, the chat/LLM sandbox tool-calling path and `runtime_services.rs`'s moves onto a new runtime. It holds the sandbox-tool runner (`run_internal_sandbox_tool`), the stop path (`stop_recorded_processes`, which hands the recorded PIDs to the routine [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) shares with `rocmd`, `scripted_stop_outcome`, the end-to-end hook behind the `e2e-test-hooks` feature, `stop_internal_managed_service`, `stop_internal_managed_service_with`, `stop_internal_managed_service_confirmed_with`, `unload_lemonade_service_model`) and the restart path (`apply_restart_runtime_pin`, `load_service_for_restart`, `restart_child_argv`, `restart_internal_managed_service`, `restart_internal_managed_service_with`, `update_restarting_record`, `record_restarted_child`, `record_restart_readiness`). Only `run_internal_sandbox_tool` and `restart_internal_managed_service` are `pub(crate)`: the first is read from `dispatch()` in [`main.rs`](../apps/rocm/src/main.rs), from [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs)'s `run_approved_service_action` and from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), which passes it to [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s `run_attached_service` as the stop for a Ctrl-C, the second from [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs); each importer reaches them through `crate::service_control` rather than a re-export. Every other item has no caller outside this module and stays private — see [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs)'s and [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s entries above for how those two modules' own doc comments describe this module as the caller of their helpers. The module reads managed-service records through [`services_text.rs`](../apps/rocm/src/services_text.rs) (`load_managed_service`, `read_managed_service_manifest`) and shapes the `stop_server` answer with its `internal_stop_server_value`; `services_text.rs` does not call back into it. This module takes `ensure_public_service_has_endpoint_key` and the spawn helpers from [`service_launch.rs`](../apps/rocm/src/service_launch.rs), which does not import it back. The cluster's regression tests live in this module's own `mod tests`, reaching shared fixtures (`test_paths`, `ScopedTestEnv`, `plant_service_record_on_runtime`, `OLD_RUNTIME_KEY`, `NEW_RUNTIME_KEY`) through `crate::tests::` the same way other mechanically relocated modules do, and `runtime_services::` through its own `use crate::runtime_services;` for the handful of tests that drive a restart through `runtime_services`'s public entry points rather than calling it directly. - [`serve_port.rs`](../apps/rocm/src/serve_port.rs) — full domain extraction: the central `rocm serve` port policy (automatic vs explicit request, reservation by live managed services, OS preflight lease), used by [`service_launch.rs`](../apps/rocm/src/service_launch.rs); the CLI restart path shares the `rocm-core` lease primitives (`lease_loopback_port_or_bind_error`, `is_canonical_loopback_host`) rather than this module. - [`service_hints.rs`](../apps/rocm/src/service_hints.rs) — shared CLI-output component: the `rocm services stop|restart|remove --yes` hint text, shared by [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs), [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs), [`service_launch.rs`](../apps/rocm/src/service_launch.rs), [`service_records.rs`](../apps/rocm/src/service_records.rs), [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs), and [`services_text.rs`](../apps/rocm/src/services_text.rs) so they cannot go out of sync with each other, as `serve_summary.rs`'s stop row did from the day it was added — #417. It also holds the helpers that map a service tool name to the command word, verb and past tense those hints and messages use (`service_action_command`, `service_action_verb`, `service_action_past_tense`) and to its `SandboxToolArg` (`sandbox_tool_arg_from_service_tool`); [`services_text.rs`](../apps/rocm/src/services_text.rs) reads `service_action_past_tense` from here rather than through the crate root. It quotes the service id with `shell_quote`, so a hint names exactly the argv it runs. - [`service_launch.rs`](../apps/rocm/src/service_launch.rs) — mechanical relocation, since split into a directory of child modules, owns no code of its own beyond its declarations: spawning/attaching to a managed service process and streaming its log, the port allocation transaction and failed-launch cleanup (under the shared launch lock), plus the endpoint-key/auth checks that gate a public bind, divided among the children listed below. The file declares the children, a `test_support` module compiled only for tests (the fixtures more than one child's tests share), and `pub(crate) use` re-exports of the items other modules call, so [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`automations.rs`](../apps/rocm/src/automations.rs), [`service_control.rs`](../apps/rocm/src/service_control.rs) and [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs) still reach them as `crate::service_launch::name`; [`main.rs`](../apps/rocm/src/main.rs) does not. An item that sibling children share is `pub(super)` and reached through `super::`; everything else is private to the child that owns it. `ServicesCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs) rather than moving alongside, and `services()` lives in [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs); the record removal and pruning it dispatches to now lives in [`service_records.rs`](../apps/rocm/src/service_records.rs), and only stop/restart are also reachable from the chat/LLM sandbox tool-calling path. This module does not import [`service_control.rs`](../apps/rocm/src/service_control.rs), which imports it: `run_attached_service` takes the stop for a Ctrl-C as a parameter, and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) passes `run_internal_sandbox_tool`. - [`service_launch/endpoint_auth.rs`](../apps/rocm/src/service_launch/endpoint_auth.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, no owned types: bind-host validation and the endpoint API key policy. `validate_bind_host` refuses a non-loopback host without `--allow-public-bind` (`is_loopback_host` is the inverse of `rocm_engine_protocol::is_public_bind_host`, which `rocmd` shares); `resolve_endpoint_auth` decides the key a launch uses (none for a loopback bind unless a key is required, otherwise the supplied key trimmed or a generated one, refusing an empty key or one with a control character); `drop_orphaned_endpoint_key_on_already_running` clears a key staged for an attempt that spawned nothing; `ensure_public_bind_engine_supported` refuses a public bind with Windows managed Lemonade, whose key cannot be enforced; and `ensure_public_service_has_endpoint_key` is the guard a respawn runs so a public (or `--require-api-key`) service whose key file is gone is not reopened without authentication. All six are `pub(crate)` and re-exported by the parent: [`service_control.rs`](../apps/rocm/src/service_control.rs) reads `ensure_public_service_has_endpoint_key`, [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs) reads `is_loopback_host`, and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) reads the rest. -- [`service_launch/launch_transaction.rs`](../apps/rocm/src/service_launch/launch_transaction.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, owns types: the part of a managed launch that decides before anything spawns. `resolve_managed_port_in_transaction` refreshes liveness, answers an equivalent live service before leasing any port (`ManagedPortDecision`), and otherwise leases a concrete port against both the ports live records still reserve (`live_managed_services`, `reserved_service_ports`) and the real OS through [`serve_port.rs`](../apps/rocm/src/serve_port.rs); a caller must hold the launch lock across it and the record write that follows. `abandon_failed_managed_launch` undoes a failed attempt before that lock is released: a spawned child is terminated first and its record kept, marked stopped, while an attempt that never spawned has every artifact removed. `managed_service_reuse_refusal` (with `ManagedReuseRefusal`) is the one answer to whether a live matching service can serve a request (recipe, `--require-api-key`, device policy), and `no_usable_gpu_refusal` words the no-usable-GPU refusal once for `serve()`'s pre-flight and for the spawn. `managed_service_reuse_refusal` and `no_usable_gpu_refusal` are `pub(crate)`, re-exported, and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), and `ManagedReuseRefusal` is `pub(crate)` too because the first returns it (only [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs) matches on it); `ManagedPortDecision`, `live_managed_services`, `resolve_managed_port_in_transaction` and `abandon_failed_managed_launch` are `pub(super)` for [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs). +- [`service_launch/launch_transaction.rs`](../apps/rocm/src/service_launch/launch_transaction.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, owns types: the part of a managed launch that decides before anything spawns. `resolve_managed_port_in_transaction` refreshes liveness, answers an equivalent live service before leasing any port (`ManagedPortDecision`), and otherwise leases a concrete port against both the ports live records still reserve (`live_managed_services`, `reserved_service_ports`) and the real OS through [`serve_port.rs`](../apps/rocm/src/serve_port.rs); a caller must hold the launch lock across it and the record write that follows. `abandon_failed_managed_launch` undoes a failed attempt before that lock is released: a spawned child is terminated first, through the same `rocm_core::terminate_recorded_pid_within` routine a stop uses (so on Windows its descendants are taken down too), and its record kept, marked stopped, while an attempt that never spawned has every artifact removed. A spawned child whose start token was not captured, on a host that records tokens (Linux and Windows, which records the process creation time), is not signalled (`launch_cleanup_refusal`): it was gone before it could be captured and its pid may belong to another process. `managed_service_reuse_refusal` (with `ManagedReuseRefusal`) is the one answer to whether a live matching service can serve a request (recipe, `--require-api-key`, device policy), and `no_usable_gpu_refusal` words the no-usable-GPU refusal once for `serve()`'s pre-flight and for the spawn. `managed_service_reuse_refusal` and `no_usable_gpu_refusal` are `pub(crate)`, re-exported, and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), and `ManagedReuseRefusal` is `pub(crate)` too because the first returns it (only [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs) matches on it); `ManagedPortDecision`, `live_managed_services`, `resolve_managed_port_in_transaction` and `abandon_failed_managed_launch` are `pub(super)` for [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs). - [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, owns types: the detached engine child that the background and attached serve paths share. The process and stdio plumbing (`detach_background_command`, `attach_background_stdio`, `managed_service_process_command`, `managed_engine_startup_failure_detail`) is `pub(crate)` and, off Windows, re-exported: [`service_control.rs`](../apps/rocm/src/service_control.rs) reads all four for a restart and [`service_launch/helper_autostart.rs`](../apps/rocm/src/service_launch/helper_autostart.rs) reads the first three. On Windows `managed_engine_startup_failure_detail` is re-exported for test builds only, because the [`services_text.rs`](../apps/rocm/src/services_text.rs) log-tail tests call it through `crate::service_launch` there too. `ManagedLaunchReport` (`pub(crate)`) is the engine-neutral result of a launch, and `ManagedSpawn` (`pub(super)`) says whether the launch found a live service or spawned a child. `spawn_managed_engine_child` is the whole serialized transaction under the launch lock (duplicate detection, port lease, manifest and log creation, argv preparation, the bounded immediate-exit check and record publication) and returns before any readiness wait; `launch_managed_engine_child_locked` is its side-effecting half, split out so that every failure after the record's first write goes through `abandon_failed_managed_launch`. The tests reach the spawn through a test-only, thread-scoped hook (`TEST_SPAWN_HOOK`, `pub(super)` so the tests in [`service_launch/start.rs`](../apps/rocm/src/service_launch/start.rs) can observe the state at the instant of the spawn). - [`service_launch/readiness.rs`](../apps/rocm/src/service_launch/readiness.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, owns private items: waiting on a service after it is launched or found live. `wait_for_record_ready` and `wait_on_existing_service` run the wait (one wait is bounded by `READINESS_WAIT`, and `wait_past_readiness_budget` is what `rocm serve --wait` uses to keep polling past it until the server answers or stops being live), `service_pid_is_alive`, `record_still_live` and `service_still_live` decide whether the tracked process is still there, `settle_exited_service` demotes the manifest of one that is not, and `died_during_wait` and `record_wait_outcome` end a wait, writing its outcome back without discarding what changed on disk meanwhile. `e2e_simulate_oom_launch` and `simulate_oom_managed_launch` are the end-to-end fault-injection seam behind the `e2e-oom-fault-injection` feature: the first is a `const fn` returning false without the feature, is `pub(crate)` and re-exported, and is read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs); the second fabricates the on-disk state of a launch that ran out of memory. `wait_for_record_ready`, `wait_on_existing_service`, `service_pid_is_alive`, `died_during_wait`, `record_wait_outcome` and `simulate_oom_managed_launch` are `pub(super)` for [`service_launch/start.rs`](../apps/rocm/src/service_launch/start.rs); the rest are private. - [`service_launch/start.rs`](../apps/rocm/src/service_launch/start.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, no owned types: the background launch path. `start_managed_service` runs the spawn transaction, releases the launch lock once the record is published, waits for HTTP readiness and returns a `ManagedLaunchReport`; if the launch fails before a record exists it drops the endpoint key `serve()` staged for the attempt. `print_managed_launch_plain` renders the report line by line for output that is not interactive, and `wait_begin_client_config` returns the one-time client configuration for a generated key when a wait begins, built by `render_endpoint_client_config` (`pub(super)`, since [`service_launch/attach.rs`](../apps/rocm/src/service_launch/attach.rs) prints it too). The three functions other modules call are `pub(crate)`, re-exported, and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs). - [`service_launch/helper_autostart.rs`](../apps/rocm/src/service_launch/helper_autostart.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, owns a private type: starting the background automation helper (`rocm daemon`) when a watcher needs it. `ensure_background_helper_running` and `ensure_background_helper_running_quiet` do nothing unless an automation is enabled and [`background_helper.rs`](../apps/rocm/src/background_helper.rs)'s `background_helper_already_running` says no helper is running; `ensure_background_helper_running_with` takes the spawn as a parameter so the decision is testable, and `spawn_background_helper` does the detached spawn. Concurrent callers queue on an autostart lock, and `AutostartClaim` (with `AUTOSTART_CLAIM_TTL_MS`, `read_autostart_claim`, `write_autostart_claim` and `autostart_spawn_in_flight`) records a spawn in flight so the next holder defers instead of starting a second helper. Both entry points are `pub(crate)` and re-exported: [`automations.rs`](../apps/rocm/src/automations.rs) calls the first, and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) and [`service_launch/attach.rs`](../apps/rocm/src/service_launch/attach.rs) call the quiet one. - [`service_launch/attach.rs`](../apps/rocm/src/service_launch/attach.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, owns private types: the attached serve path (`--verbose`/`--foreground`). `run_attached_service` spawns the same detached child the background path does and follows its log in the terminal until the user detaches with Ctrl-D (the server keeps running) or stops it with Ctrl-C, or the engine exits; `AttachOutcome` is that result and `detach_key_outcome` maps a key press to it. `stream_attached_logs` reads keys in crossterm raw mode (`RawModeGuard` restores the terminal on every exit path) and `stream_attached_logs_no_tty` follows the log until the process exits when stdin is not a terminal. `finish_attached_stop` turns the answer of the stop that Ctrl-C ran into output: a confirmed stop is reported on stdout, an unconfirmed one becomes the error, and a failed stop signals the child this session launched. `run_attached_service` is `pub(crate)`, re-exported, and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs). -- [`service_records.rs`](../apps/rocm/src/service_records.rs) — mechanical relocation, owns types: removing one managed-service record and every file it owns (`remove_managed_service_record`, `service_record_artifacts`) and the `rocm services prune` plan/render/apply steps (`build_service_prune_plan`, `render_service_prune_plan`, `apply_service_prune_plan`, `prune_managed_service_records`) with the age gate behind them (`DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS`, `service_prune_min_age_hours`). It reads record state through [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) and [`services_text.rs`](../apps/rocm/src/services_text.rs) and builds its advice text with [`service_hints.rs`](../apps/rocm/src/service_hints.rs); none of the three depends on it. `remove_managed_service_record`, `prune_managed_service_records`, `service_prune_min_age_hours` and `DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS` are `pub(crate)` (`services()` in [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs) calls the first three; the `ServicesCommand::Prune` clap definition in [`cli.rs`](../apps/rocm/src/cli.rs) uses the constant), and `ServicePruneOutcome`, the return type of `prune_managed_service_records`, is `pub(crate)` with `pub(crate)` `text`, `removed_records`, `removed_files` and `failures` fields that `services()` reads. Every other item is private. The cluster's regression tests live in this module's own `mod tests`, reaching the shared `test_paths`, `plant_service_record` and `plant_service_record_without_its_key` fixtures through `crate::tests::` (the last two widened to `pub(crate)` for that). +- [`service_records.rs`](../apps/rocm/src/service_records.rs) — mechanical relocation, owns types: removing one managed-service record and every file it owns (`remove_managed_service_record`, `service_record_artifacts`) and the `rocm services prune` plan/render/apply steps (`build_service_prune_plan`, `render_service_prune_plan`, `apply_service_prune_plan`, `prune_managed_service_records`) with the age gate behind them (`DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS`, `service_prune_min_age_hours`). It reads record state through [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) and [`services_text.rs`](../apps/rocm/src/services_text.rs) and builds its advice text with [`service_hints.rs`](../apps/rocm/src/service_hints.rs); none of the three depends on it. `remove_managed_service_record_with` takes `--force` and the read of each recorded PID: `--force` removes a record only when none of its recorded processes is confirmed running, one could not be queried and a stop was requested, and reports which. `remove_managed_service_record_with`, `prune_managed_service_records`, `service_prune_min_age_hours` and `DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS` are `pub(crate)` (`services()` in [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs) calls the first three; the `ServicesCommand::Prune` clap definition in [`cli.rs`](../apps/rocm/src/cli.rs) uses the constant), and `ServicePruneOutcome`, the return type of `prune_managed_service_records`, is `pub(crate)` with `pub(crate)` `text`, `removed_records`, `removed_files` and `failures` fields that `services()` reads. Every other item is private. The cluster's regression tests live in this module's own `mod tests`, reaching the shared `test_paths`, `plant_service_record` and `plant_service_record_without_its_key` fixtures through `crate::tests::` (the last two widened to `pub(crate)` for that). - [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm services`' command handler (`services()`, which dispatches `list`, `logs`, `stop`, `restart`, `remove` and `prune`) and `run_approved_service_action`, the approval-gated stop/restart path. `ServicesCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs). Both functions are `pub(crate)`: `dispatch()` calls `services()` through a re-import, and [`advised_commands.rs`](../apps/rocm/src/advised_commands.rs)'s test reaches `run_approved_service_action` through a test-only re-import in `main.rs`. The module reads from [`services_text.rs`](../apps/rocm/src/services_text.rs), [`service_control.rs`](../apps/rocm/src/service_control.rs), [`service_records.rs`](../apps/rocm/src/service_records.rs), [`service_hints.rs`](../apps/rocm/src/service_hints.rs) and [`cli_audit.rs`](../apps/rocm/src/cli_audit.rs); none of them calls back into it. The cluster's one regression test lives in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::`. - [`services_text.rs`](../apps/rocm/src/services_text.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-209): the CLI/TUI logs browser (`render_logs_text`, `render_logs_browser_text`, `render_logs_browser_page_text`, `render_logs_browser_page_text_with_options`, `logs_browser_total_pages`, the `LogBrowserEntry` type, `collect_log_browser_entries`, `filter_log_browser_entries`, `log_browser_source_label`, `write_past_attempts_hint`), the `rocm services`/`rocm services logs` status surfaces (`render_services_text`, `render_services_json`, `render_service_logs_text`, `render_service_logs_text_with_options`, `render_service_action_result`, `read_managed_service_manifest`, `load_managed_service`), the `--follow` tail behind `rocm logs --service --follow` and `rocm services logs --follow` (`follow_service_logs`, `follow_service_logs_with`, the `FollowCursor`, `LogSegment` and `FollowFooter` types, `write_log_segments`, `newly_ready`; new code written in this module, not relocated; `FollowCursor` is the private cursor that reads what a growing log gained, and is not [`rocm_core::LogTail`](../crates/rocm-core/src/log_tail.rs), the result of a bounded tail read that the log views in this module use), the internal `rocmd`-compatible status surface (`render_internal_status_text`), the `stop_server` answer shape and its renderer (`internal_stop_server_value`, `service_stop_unconfirmed`, `render_unconfirmed_service_stop`, `process_check_command`), plus `validate_service_id`, `list_log_files`, `read_optional_tail_lines`, and the CLI-lifecycle-log tail formatting helpers (`format_cli_lifecycle_tail_line`, `lifecycle_event_label`, `humanize_log_label`, `lifecycle_field`, `read_log_tail`). The managed-service stop and restart implementation that used to live here is in [`service_control.rs`](../apps/rocm/src/service_control.rs). The `LogBrowserEntry` struct moved here from `main.rs` with the logs browser and stays private. `dispatch()` calls directly into this module at several points — `render_internal_status_text`, `render_service_logs_text`, `render_logs_browser_text`, and `render_logs_text` — all re-imported via `use crate::services_text::{...};`; [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs)'s `services()` calls `render_services_text`, `render_services_json` and `render_service_logs_text` through its own imports. `render_service_action_result`, `service_stop_unconfirmed` and `validate_service_id` are also `pub(crate)` and read from `services_cmd.rs`'s `run_approved_service_action`; `validate_service_id` is also read from [`service_records.rs`](../apps/rocm/src/service_records.rs)'s `remove_managed_service_record`/`service_record_artifacts`, and `load_managed_service` from its `remove_managed_service_record`/`apply_service_prune_plan`. `load_managed_service` is `pub(crate)`, read from [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) (reconciling live servers against a runtime activation), from [`service_control.rs`](../apps/rocm/src/service_control.rs) and from [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs)'s own test module; `read_managed_service_manifest` and `internal_stop_server_value` are `pub(crate)` for `service_control.rs`, and `process_check_command` for its tests; `read_log_tail`, `render_logs_text`, `render_logs_browser_text`, `render_services_text`, `render_service_logs_text`, and `validate_service_id` are also `pub(crate)`, read from [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs)'s read-only command runner; `read_optional_tail_lines` and `render_service_action_result` are also `pub(crate)`, read from [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s managed-spawn/attach path. Every other item in this list has no caller outside this module and stays private. The cluster's regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` the same way other mechanically relocated modules do; `assert_no_retry_line` stays here as a `pub(crate)` test helper because it checks the unconfirmed-stop rendering and `service_launch.rs`'s tests read it too. Two tests that call this module's `load_managed_service` only incidentally — their primary subject is `runtime_services::pin_service_record_to_runtime`/`restore_service_record_pin` and `runtime_services::restart_service_onto_runtime_with`'s outcome classification respectively — stay in `main.rs`'s shared `mod tests` instead. - [`setup_cmd.rs`](../apps/rocm/src/setup_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm setup`'s command handler (`setup()`, which runs `setup status` by default and `setup reset`), the text `rocm setup status` prints (`render_setup_status_text`) and the reset that clears the saved first-time prompt state (`reset_setup_prompt_state`). `SetupCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs). `setup()` and `render_setup_status_text` are `pub(crate)` and re-imported in `main.rs`: `dispatch()` calls the first, and [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs)'s read-only handling of `rocm setup` and `rocm setup status` reaches the second as `crate::render_setup_status_text`; `reset_setup_prompt_state` is private. The three regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` and `write_test_pip_runtime` through `crate::test_support::`. @@ -164,7 +165,7 @@ Modularized (ROCMAI-83, Phase 5 of ROCMAI-27's sequencing): - [`persistence.rs`](../apps/rocmd/src/persistence.rs) — `record_event`/`load_managed_services`/`load_service_record`, the automation-event/audit-log and managed-service-registry I/O shared across the daemon's sandbox, MCP, service-lifecycle, and watcher code. - [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) — bubblewrap/native sandbox execution, artifact prefetch policy gating, and the sandbox-tool result shaping for `check_updates`/`driver_plan`/`stop_server`. - [`sandbox_tool.rs`](../apps/rocmd/src/sandbox_tool.rs) — the sandbox-tool vocabulary: the `SandboxToolArg` value enum naming each sandbox tool, and `SandboxToolPolicy`, the artifact-download policy built from the `--allow-*` flags (including the Hugging Face token read from the environment when authenticated downloads are allowed). It uses no other module of the crate, so modules that need the vocabulary do not import [`cli.rs`](../apps/rocmd/src/cli.rs). -- [`service.rs`](../apps/rocmd/src/service.rs) — managed-service stop: terminating the recorded processes of a service after verifying their identity, and the stop report (`stop_managed_service`, `stop_report_confirmed`) that the sandbox and MCP tools read. +- [`service.rs`](../apps/rocmd/src/service.rs) — managed-service stop: the engine `Stop`, the manifest and endpoint-key writes, and the stop report (`stop_managed_service`, `stop_report_confirmed`) that the sandbox and MCP tools read. The termination of the recorded processes is the routine shared with `rocm services stop`, in [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs); this module passes it `terminate_recorded_pid` and keeps no copy of it or of its grace constant. - [`supervise.rs`](../apps/rocmd/src/supervise.rs) — launching and re-launching a managed service's supervisor: `supervise_service` (the `rocmd supervise` entry point: launch the engine child, record its identity, wait for readiness, record the exit) with the serve-log startup-phase polling and readiness wait it uses (its engine child runs on the `rocm` binary that `DaemonHost::rocm_binary` finds, before the first write, since `rocmd` has no engine commands, and it polls readiness through the same host), and `restart_managed_service` (the recovery respawn: re-launch `rocmd supervise` detached for a service and record the new supervisor; the binary that runs `supervise` comes from [`daemon_host.rs`](../apps/rocmd/src/daemon_host.rs), and a restart that cannot find it fails before it writes the record). Writes after a supervisor's first go through the private `update_supervised_record`, which re-reads the record and leaves a stop marker or another supervisor's record alone. Its production code uses only [`common.rs`](../apps/rocmd/src/common.rs), [`daemon_host.rs`](../apps/rocmd/src/daemon_host.rs), [`engine_ipc.rs`](../apps/rocmd/src/engine_ipc.rs), [`persistence.rs`](../apps/rocmd/src/persistence.rs) and `stop_requested` from [`watchers.rs`](../apps/rocmd/src/watchers.rs). [`cli.rs`](../apps/rocmd/src/cli.rs) calls `supervise_service`; [`watchers/server_recover.rs`](../apps/rocmd/src/watchers/server_recover.rs) (the server-recover handler) and [`sandbox.rs`](../apps/rocmd/src/sandbox.rs) (the `restart_server` tool) call `restart_managed_service`. - [`watchers.rs`](../apps/rocmd/src/watchers.rs) — the module root of the built-in watchers: the registry (`reconcile_watcher_snapshots`) and the event collector and dispatcher (`evaluate_watchers`, `collect_automation_events_with_gpu_snapshot`, which takes the GPU snapshot as a closure so tests can inject one, and `evaluate_watchers_for_events`), which route each event to its watcher's handler. `find_recoverable_service` and `stop_requested` are re-exported from here because [`supervise.rs`](../apps/rocmd/src/supervise.rs)'s and [`service.rs`](../apps/rocmd/src/service.rs)'s tests reach them by the `crate::watchers::` path. Each watcher's handler lives in the child below that carries its name, together with the policy they share, and a child reaches only [`policy.rs`](../apps/rocmd/src/watchers/policy.rs) and, for the thermal-protect watcher, [`gpu_metrics.rs`](../apps/rocmd/src/watchers/gpu_metrics.rs) among the others. - [`watchers/policy.rs`](../apps/rocmd/src/watchers/policy.rs) — what a watcher does in each mode and the proposals it queues: `WatcherPolicyAction` and `watcher_policy_action` (observe, queue a proposal or run contained, chosen by the watcher mode), `queue_proposal` and `queue_proposal_with_arguments` with the action-to-tool and action-to-arguments tables behind them, and `pending_proposal_exists`, the check the thermal-protect and server-recover handlers use so they do not queue a duplicate of a proposal that is still pending. It uses no other watcher module. @@ -245,7 +246,7 @@ Further `rocm-core` modules, each with its own doc comment except where noted: - [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) — OpenMPI, libatomic and libnuma detection, the `libmpi_cxx` compatibility shim, and the per-library install plans (`build_openmpi_install_plan`, `build_libatomic_install_plan`, `build_libnuma_install_plan`) and hints, needed by vLLM's ROCm wheels. The distro-generic half of that planning lives in [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs); [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs) re-exports its public items, so `rocm_core::openmpi::running_as_root`, `plan_for_os_release_read` and the rest keep resolving, and `pub(crate)` re-exports `PackageManager` and `resolve_package_manager` for [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs). - [`os_release.rs`](../crates/rocm-core/src/os_release.rs) — the one `/etc/os-release` parser every reader of that file uses. - [`package_plan.rs`](../crates/rocm-core/src/package_plan.rs) — the planning half of a system-package install that does not depend on which library is installed: `PackageManager` and `resolve_package_manager` (from an os-release `ID`/`ID_LIKE`), `SystemPackageInstallPlan` and `InstallCommand`, `plan_for_os_release_at`/`plan_for_os_release_read` (which take the per-library builder as an argument), `InstallHint`, the root and passwordless-`sudo` checks (`running_as_root`, `can_autoinstall`) and the `apt-get -s` helpers (`apt_simulate_argv`, `parse_apt_simulate_removals`, `is_protected_rocm_package`). It never runs anything and depends on no other module of this crate except [`os_release.rs`](../crates/rocm-core/src/os_release.rs) and [`hardware_root.rs`](../crates/rocm-core/src/hardware_root.rs), which it reads os-release through. The module is private; its public items are re-exported from [`openmpi.rs`](../crates/rocm-core/src/openmpi.rs). It is a different module from [`system_packages.rs`](../apps/rocm/src/system_packages.rs) in `apps/rocm`, which runs these plans with consent and `sudo`. -- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches. +- [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) — verified, bounded termination of a process by persisted PID, refusing to signal a PID whose start-time identity no longer matches (`terminate_verified`), and the one managed-stop routine built on it that `rocm services stop` and `rocmd`'s `stop_server` both call: `terminate_recorded_service_pids` walks the processes a service record names (supervisor, then engine, each with its own start-time, the stopping process's own PID reported as `self` and never signalled) and returns one `RecordedPidStop` per PID (each caller, on a stop that confirms every one gone, then clears the record's recorded processes with `ManagedServiceRecord::clear_recorded_processes` in [`service_record.rs`](../crates/rocm-core/src/service_record.rs), so a repeated stop reaches no stale PID; an unconfirmed stop keeps them); `terminate_recorded_pid` terminates one of them, adding on Windows a `taskkill /T /F` of its descendants first, because `terminate_verified` cannot walk a process tree there, and only for a PID that passes the same identity check (`process_start_ticks` is the kernel start-time on Linux and the process creation time on Windows, so a recycled PID is refused before any `taskkill`; a record written before Windows recorded one cannot be refuted), holding the PID open on Windows from before that check until the stop ends so the PID cannot be reused in between (a PID that matches but cannot be held is reported unverified and not signalled), and treating a tree kill that did not complete, or did not finish within `MANAGED_STOP_GRACE` (the `taskkill` is killed and reaped at that deadline), as an unconfirmed stop (`descendants_unconfirmed` when the root itself was seen to exit, `timed_out` only when it outlived its deadline), except that a `taskkill` which found no such process is judged by the root's own re-read, so a root that exited by itself is `exited_before_tree_kill`, which the record-wide settling in `terminate_recorded_service_pids` turns into a stop only when another recorded process (the engine, which is visited as its own role and folded by both callers) is confirmed gone, and starting that `taskkill` from the system directory (`GetSystemDirectoryW`, not the `SystemRoot` variable) through `windows_system_tool`; a PID that cannot be queried at all, as against one that is gone, reads as unverifiable unless a readable recorded token differs, in which case that mismatch wins (`ProcessProbe` and `probe_process` in [`process.rs`](../crates/rocm-core/src/process.rs), whose `may_be_running` is the answer for any decision that must not take an unqueryable process for gone); `MANAGED_STOP_GRACE` is the one grace period. The termination is injectable, so a caller's test supplies outcomes it cannot construct and a test of the Windows step runs on every host. - [`report.rs`](../crates/rocm-core/src/report.rs) — the content of a Doctor report and the rule deciding whether one may exist; it sends nothing. - [`report_delivery.rs`](../crates/rocm-core/src/report_delivery.rs) — how a report leaves the machine: building its link and deciding whether a browser may be started. - [`runtime.rs`](../crates/rocm-core/src/runtime.rs) — the host-platform layer: `RuntimePlatform`/`RuntimeHost`, OS-specific executable and Python-environment names (including `runtime_python_env_root_for_executable`, the one rule for finding an environment's folder from its interpreter's path: only an interpreter directly inside a `bin` or `Scripts` folder, in any case, has one, and everything else gets `None`; the adopt command, the ComfyUI runtime environment and the vLLM launch all call it rather than keeping a copy, since the engine crate cannot reach `apps/rocm`), default config/data/cache directories, path-text and path-list helpers, and `install_root_has_payload`, the one definition of a non-empty install root (dot-entries such as the runtime marker do not count; a path that does not exist holds nothing; a root that cannot be listed is an error) that tarball extraction, update repair detection and activation validation in `apps/rocm` all ask (no module doc comment). diff --git a/docs/manual-testing.md b/docs/manual-testing.md index f4fe0aedd8..a2eb7dd454 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -606,13 +606,78 @@ rocm services restart --yes `stop` prints `Local server stopped` only once every recorded process is confirmed gone; otherwise it prints `Local server stop not confirmed` on stderr, keeps the record and any endpoint key, and exits non-zero. It names each -process that, or a process under it, did not exit, and for one it could not -verify, an operating-system command to check on it. It never prints a `retry:` -line. +process that, or a process under it, did not exit. On Windows it also names one +that did exit while taking the processes under it down did not complete +(`not confirmed: process () exited, but a process under it may still +be running`), and one that was already gone when that step ran with no other +recorded process confirmed gone (`not confirmed: process () was +already gone when the processes under it were to be taken down`). For one it +could not verify, it names an operating-system command to check on it. It never +prints a `retry:` line. `restart` starts the server again only once its stop has confirmed every recorded process gone; otherwise it reports that and starts nothing. +On Windows the stop takes the engine's process tree down with the recorded +process (`taskkill /T /F`, which has to finish within the stop's grace period of +10 seconds, or the stop is not confirmed), and it checks the recorded process +against the creation time written when the server started, so a process that has +since taken the same PID is left alone and reported as `identity_mismatch`. A +process it cannot verify, on either platform, is not signalled and is named with +the check the host has: `ps -p -o pid,args` on Linux and WSL, and +`tasklist /FI "PID eq "` on Windows. There are two ways to be unverifiable: +the recorded start-time can no longer be read back from a running process, or +(Windows only) the PID cannot be queried at all, as for a live process the +account may not open. The text of `rocm services stop` does not list a per-process +outcome for the stopping process itself; the JSON answer of the sandbox +`stop_server` tool does, in `pid_outcomes`, as `self` (never signalled, counted +as stopped). + +#### Windows host check + +The Windows build-and-test job runs the workspace unit tests, which include two +`#[cfg(windows)]` tests in `proc_lifecycle.rs` that stop a real `ping` process +through the PID hold, the creation-time read and the System32 `taskkill`. It does +not run any scenario, and nothing there stages the unrun parts: the end-to-end +tree kill of a parent with a child (`service-stop-07`), a real unopenable +process (the `OpenProcess` failure codes), and `taskkill` exiting 128 or failing +or stalling. None of these has been run on a Windows host, and no Windows result +is claimed. On a Windows host with a `--features rocm/e2e-test-hooks` debug +build and `ROCM_CLI_BINARY` set, run the scenarios that apply to Windows: +`cargo test -p e2e-cucumber --test e2e -- -n "service-stop-0[178]"` (`-01` +plants PID 0, `-07` is Windows-only, `-08` runs on every OS) and +`cargo test -p e2e-cucumber --test e2e -- -n "service-unqueryable-01"`, and +record the result. Do not select `service-stop-02` to `-06` or +`service-unqueryable-02`: they are Linux-only (they need `sleep` or `/proc`), +and `-n` bypasses the harness's OS filter, so they would fail for want of the +platform rather than for any defect. + +#### A record that stays unverified + +If the server is gone but its recorded PID has been taken by a process this +account cannot open, `rocm` cannot tell the two apart, and it treats that as +possibly running, which is the safe direction: the record never reconciles to +`stopped`, `rocm services remove` and `rocm services prune` refuse it, and +`rocm services stop` keeps reporting `unverified`. To clear it: + +1. Run the check command the stop printed. If the process is this server's, end it + yourself (from an elevated prompt if the account may not), then run + `rocm services`; the record reconciles and its endpoint key is dropped. +2. If it is an unrelated process, run `rocm services remove --force + --yes`. The refusal of a plain `remove` names exactly this command. `--force` + applies only when a stop was requested (the stop that reported `unverified` + did that), no recorded process is confirmed running, and at least one could + not be queried; a process `rocm` can see running is never forced past. The + command says which process could not be queried and that, if it is still + running, it keeps running, and it deletes the record, its log, its engine + state and its endpoint key. The server can no longer be restarted afterwards. + `rocm services prune` has no `--force`: it leaves such a record alone and lists it + under `Left alone` with this exact command. + If the record still reads `ready` and nobody has run `rocm services stop` on it, + `--force` is refused and the refusal names that stop first; run it (it reports + the process unverified and records the request), then the forced removal. This is + the intended order, not a gap: `--force` never removes a record nobody tried to stop. + The rest of this section needs a **vLLM** server running, on Linux or WSL, and two runtimes to move it between. Nothing earlier in this document sets either up: section 3 serves Lemonade and then stops it on both of its branches, the diff --git a/docs/testing.md b/docs/testing.md index f048f1b863..099e834a28 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2251,6 +2251,284 @@ directly, exits 0, and records whether that signal worked, adding a second stderr `note:` when it did not (`cargo test -p rocm --bin rocm services_stop_` and `cargo test -p rocm --bin rocm attached_ctrl_c_`). +`rocm services stop` and `rocmd`'s `stop_server` terminate the processes a +record names through one routine, +[`terminate_recorded_service_pids`](../crates/rocm-core/src/proc_lifecycle.rs), +and wait `MANAGED_STOP_GRACE` for each; neither binary keeps a copy. The +routine is tested in `rocm-core` (`cargo test -p rocm-core proc_lifecycle::`): +the supervisor is visited before the engine, each PID with its own start-time, +a PID recorded under both roles once, and the stopping process's own PID is +reported as `self` in `pid_outcomes` and never signalled, in both commands. +On Windows the verified root is preceded by `taskkill /PID /T /F`, because +`terminate_verified` cannot walk a process tree there; that step is tested +through injected functions that run on every host (`the_tree_is_killed_before_the_root_is_terminated` +and its neighbours) and, against a real tree, by a Windows-only scenario +(`service-stop-07`, below). The callers' own tests check +that each reaches the routine and reports what it returns +(`cargo test -p rocm --bin rocm service_control::tests::a_stop_` and +`cargo test -p rocmd service::tests::a_stop_`), and each carries a source check +that no private copy of the routine or its constant has come back. Those checks +pin names only: they are a tripwire for a copy returning under the same name, not +coverage of what either stop does. A confirmed +stop also clears the PIDs and start-time tokens it stopped from the record +(`ManagedServiceRecord::clear_recorded_processes`, one call from both +binaries), and an unconfirmed one keeps them; a PID recorded without a token is +signalled on trust alone once the OS has recycled it, so a stale one must not +survive for a second stop to find. +`a_confirmed_stop_clears_the_recorded_processes_so_a_second_stop_signals_nothing` +runs the real routine with the per-PID signal injected, and asserts the +persisted record after the first stop and that a second stop signals nothing; +`an_unconfirmed_stop_keeps_the_recorded_processes_for_the_next_stop` is the +other direction. `service-stop-07` +(`@requires-os:windows`, no GPU) is the live coverage of the Windows tree kill when it is run on a Windows host, which no CI job does: +it records a PowerShell parent that has started a `ping` child, runs `rocm +services stop --yes`, and asserts that both processes are gone together with the +report (`Local server stopped`, `stopped processes: 1`) and the record (stopped, +no process, no stop request, key file gone). The child is the tree kill's own +work: with the `taskkill /T` step removed the stop ends only the recorded +process and the scenario fails on the surviving child. The scenario keeps the +child's own identity (`E2eWorld::recorded_descendant`) and teardown ends it when +it is still that process, so a stop that fails to take it down does not leave +`ping` running for its five minutes behind a dead parent; the decision is +`e2e_cucumber::stray_process::leftover_action`, which a table test covers on +every host (ended only while it is still the recorded process, never once gone, +reused or unconfirmed). `service-stop-01` plants +PID 0 and `service-stop-02` is Linux-only, so neither reaches that step; the +injected unit tests above cover its ordering and verdicts on every host. + +The tree kill is behind the same identity gate as the rest of the stop. On +Windows `process_start_ticks` is the process creation time (`GetProcessTimes`, a +`FILETIME`), so a record written by this version refutes a PID that an unrelated +process has taken, and the stop reports `identity_mismatch` without running +`taskkill` on it; only a record written before Windows recorded a token has +nothing to refute: a stop refuses a PID only when a token was recorded and the +live one differs, and `what_a_stop_does_for_each_recorded_and_live_token` pins +that for every case on every host (no token, which is also the shape +`service-stop-07` records; a token that matches; one that differs; one that +cannot be read back), asserting whether the tree kill runs and the verdict the +stop reports together. `a_filetime_is_its_high_word_above_its_low_word` runs the +conversion on every host. `terminate_recorded_pid_refuses_a_pid_whose_recorded_identity_no_longer_matches` +runs the real stop against this test's own PID with a wrong token on Linux and +Windows (a stop that went ahead would end the test process), and +`terminate_recorded_pid_leaves_a_recycled_pid_alone_and_stops_the_verified_one` +repeats it against a bystander on Windows only and then stops that bystander +with its true token. `service-stop-08` is the live version on every OS: it +records a running process under a token no live process has, stops the server, +and asserts that the process is still running and the record names no process. +None of the Windows-only paths has been run on a Windows host for this change, +and I am not claiming a Windows result for this head. What the CI configuration +does and does not run there, read from `.github/workflows/ci.yml`: the +`windows-debug` job (`Windows build and test (debug)`), when Rust files change, +runs clippy, the build, `cargo test --workspace --all-targets` and the harness +self-tests. That test run includes unit tests in `proc_lifecycle.rs` that stop a +real process on Windows: `terminate_recorded_pid_leaves_a_recycled_pid_alone_and_stops_the_verified_one` +and `terminate_recorded_pid_stops_a_process_recorded_without_a_token` are +`#[cfg(windows)]` and start a real `ping` child and stop it through +`terminate_recorded_pid`, which takes the PID hold, reads the creation time, +applies the identity gate (a wrong token leaves the process running; a matching +or absent token stops it) and runs the System32 `taskkill /T /F`. The e2e target +is excluded from `cargo test`, so no scenario runs there: the `service-stop-*` +and `service-unqueryable-*` scenarios do not run on that job. +`service-stop-07` (`@requires-os:windows`) and the Windows half of +`service-stop-08` can run only through the harness on a Windows host, which is a +manual step (see "Windows host check" in [manual-testing.md](manual-testing.md)); +on the Linux e2e jobs `-07` is skipped and `-08` runs its Linux half. Not covered +by any CI job: the end-to-end tree kill of a parent with a child (`-07`), the +failure cases that cannot be staged (an `OpenProcess` access-denied process, a +failing or stalled `taskkill`, `taskkill` exiting 128), and the launch-cleanup +branch run against a real Windows child. The failed-launch cleanup terminates its +child through the same routine as a stop (`terminate_recorded_pid_within`, so on +Windows the tree kill runs too), and what it asks of the termination is tested +through an injected function on every host +(`abandoned_launch_asks_the_termination_for_the_spawned_child_with_its_token`: +the child's PID with the token captured at spawn, and nothing at all for a child +with no token where tokens are recorded); that the production wiring reaches the tree +kill is not observable on Linux, where the termination walks the tree itself, so it +rests on the routine's own tests and on review. The unit tests above stop a `ping` that has no child of its own, so they +do not show a descendant being taken down. A stop that cannot +verify a process names `ps -p -o pid,args` on Linux and `tasklist /FI "PID +eq "` on Windows (`process_check_command`). +Three further Windows paths of that stop are tested through injected functions +that run on every host; no scenario forces their failure cases. A PID that cannot be +queried (`ProcessProbe::Unqueryable`, which on Windows includes a live process +the stop may not open) is `Indeterminate`, so the stop reports it unverified +instead of already gone (`a_pid_that_cannot_be_queried_is_unverified_not_already_gone`, +`a_stop_of_a_live_process_it_cannot_query_is_not_confirmed`, and +`a_failed_open_is_gone_only_for_no_such_process` for the error codes). The PID is +held open from before the identity read until the stop returns, so the creation +time that was compared is that of the process the kill then reaches +(`the_pid_is_held_before_its_identity_is_read_and_released_after_the_stop`). A PID +that could not be held (the process could not be opened) while its identity still +reads as matching is not signalled either: nothing would reserve it between the +read and `taskkill`, so the stop is `unverified` and neither the tree kill nor the +termination runs +(`a_tree_kill_is_not_run_for_a_pid_that_could_not_be_held_even_when_its_identity_matches`; +a gone, recycled or unreadable PID keeps its own refusal, +`an_unheld_pid_that_is_gone_recycled_or_unverifiable_keeps_its_refusal`, and +where there is no tree-kill step the missing hold changes nothing, +`without_a_tree_kill_step_an_unheld_pid_is_still_terminated`). And a +`taskkill` that failed to start or exited nonzero leaves the stop unconfirmed, +whatever the root's own termination says, because only the root's exit is +watched on Windows. When the root itself was seen to exit (or was already gone) +the outcome is `descendants_unconfirmed`, which says the root is gone and a +process under it may still be running; it is not `timed_out`, which is kept for +a root that outlived its deadline. `rocm services stop` prints it as a `not +confirmed:` line, and `a_stop_with_an_unconfirmed_tree_kill_says_what_happened_and_keeps_the_service` +asserts that line, that no `still running` line is printed, and the record, stop +request and key that are kept. The warning lasts one run, and the line says so: the +next `rocm services stop` finds the root gone and reports the stop confirmed, and +a listing completes the pending stop the same way, neither with new evidence about +the processes under the root (the same test runs that second stop and asserts it); +keeping it durably would need a marker in the record, which is not done +(`a_tree_kill_that_did_not_complete_leaves_the_stop_unconfirmed`, +`only_a_clean_taskkill_exit_is_a_completed_tree_kill_and_128_is_not_found`). A +`taskkill` that exits 128 (process not found) is judged by the root's own re-read: +with the root still there the kill counts as failed +(`a_tree_kill_that_found_no_process_while_the_root_runs_is_a_failed_kill`); with +the root gone it exited by itself, and the outcome is `exited_before_tree_kill`, +not a stop on its own, because `taskkill /T` reaches children only through a live +parent and nothing recorded what was under the root +(`a_tree_kill_that_found_no_process_with_the_root_gone_is_not_a_stop_by_itself`). +Across the record, such a root becomes `already_gone`, and the stop confirmed, +only when another recorded process is confirmed gone (`a_parent_that_exits_alone_is_confirmed_when_the_recorded_engine_is_confirmed_gone`, +`two_roots_that_both_exited_alone_are_a_confirmed_stop`); with nothing else +recorded, or the engine still running or unconfirmed, it stays unconfirmed +(`a_parent_that_exits_alone_with_nothing_else_recorded_is_not_a_confirmed_stop`, +`a_parent_that_exits_alone_stays_unconfirmed_while_the_recorded_engine_is_not_gone`, +`the_stopping_process_does_not_corroborate_a_root_that_exited_alone`). Both +callers fold every row, so the record, PIDs and key are kept unless every +recorded role is confirmed +(`a_root_that_exited_alone_confirms_the_stop_only_when_the_engine_role_does` in +`rocm` and in `rocmd`, `a_lone_root_that_exited_alone_leaves_the_stop_unconfirmed` +in both). A failed tree kill with the root gone stays unconfirmed +(`a_failed_tree_kill_with_the_root_gone_is_still_unconfirmed`). That `taskkill` runs under +a deadline of `MANAGED_STOP_GRACE`: one that stalls is killed and reaped and +counts as a failed tree kill, so it cannot hang `rocm services stop`, `restart` +or the attached Ctrl-C stop +(`a_helper_that_stalls_is_killed_and_reaped_at_its_deadline` runs a long-sleeping +child under a 200 ms deadline on every host and asserts the error, the bounded +elapsed time, the reaped child and the failed verdict; `a_helper_that_exits_in_time_gives_its_exit_code` +and `a_helper_that_cannot_be_started_is_an_error` are its neighbours). The report and the record +a `timed_out` stop leaves are covered by +`an_unconfirmed_stop_keeps_the_recorded_processes_for_the_next_stop`. The +`taskkill` is started from the system directory that `GetSystemDirectoryW` +returns, not by a bare name `PATH` could resolve and not from the `SystemRoot` +environment variable a caller could redirect +(`a_system_tool_is_resolved_under_the_system_directory` builds the path from a +given directory on every host; the Windows call itself is run only by the +Windows job's tests that start `taskkill`; ComfyUI's own `taskkill` and `tasklist` calls use +the same helper). A readable +recorded token that differs outranks a failed query +(`a_readable_token_mismatch_outranks_a_failed_query`), but the creation time and +the query are read with the same access right, so on Windows this only reaches a +process that opens and then fails the exit-code query; a process that cannot be +opened has no readable token, stays unverifiable, and is cleared with +`rocm services remove --force --yes` (see "A record that stays unverified" +in [manual-testing.md](manual-testing.md); the unit tests +`force_removes_a_record_held_by_an_unqueryable_process_and_says_so`, +`a_record_held_by_an_unqueryable_process_is_refused_naming_the_forced_removal`, +`force_is_refused_for_a_running_process_and_without_a_requested_stop` and +`force_on_a_record_that_is_not_running_is_an_ordinary_removal`, +`a_forced_removal_missing_yes_names_the_command_that_completes_it` and +`prune_names_the_forced_removal_for_a_record_held_by_an_unqueryable_process` (prune +lists the record with the command, and running that command clears it) cover it with the +probe injected, and `service-unqueryable-01` and `-02` drive the stop, the +refusal naming `--force`, and the forced removal through the real binary, with +the `e2e-test-hooks` variable `ROCM_E2E_UNQUERYABLE_PIDS` making the listed PIDs +read as unqueryable; `-02` is Linux-only because it starts a real process). On +Windows the happy paths of the PID hold, the creation-time read and the System32 +`taskkill` are exercised by the two `#[cfg(windows)]` unit tests named above when +`windows-debug` runs the workspace tests (no result for this head is claimed +here). What no test reaches there is the failure side of the Windows API: the +`OpenProcess` failure codes against a real unopenable process, `taskkill` +exiting 128 or failing or stalling, and a descendant actually being taken down; +none of those has been run on a Windows host. What was moved out of the FFI so +that it is unit-tested on Linux and, as plain `cargo test`, on the Windows unit +job: the open-failure code mapping (`classify_open_failure`), the exit-code query +mapping (`classify_exit_query`), the `taskkill` result and arguments +(`classify_taskkill`, `taskkill_args`), the `System32` path building, the +FILETIME join, and the deadline wait (a real child). The ordering of the hold, +the identity read, the tree kill and the termination is tested through injected +functions. + +The same unqueryable-process rule holds wherever a service's liveness decides +something destructive. The deferred key cleanup and the reconciliation to +`stopped` in the liveness refresh (`refresh_managed_service_runtime_liveness`) and +the in-use check behind `rocm services remove` and `prune` read each recorded PID +through `probe_process` and treat anything but a confirmed exit as possibly +running. `a_pending_stop_keeps_its_key_and_marker_while_a_process_cannot_be_queried` +drives the refresh with an unqueryable PID and then a confirmed-gone one, +`an_unverified_stop_keeps_its_key_and_marker_through_a_later_liveness_refresh` runs +an unverified stop and then the refresh over the record it wrote, and +`a_record_is_in_use_unless_every_recorded_process_is_confirmed_gone` covers the +removal guard; each takes the probe as an argument, so they run on every host. +A pending stop of a container-backed engine (`engine_stop_decides_confirmation`, +today `aim`) is never completed that way: its supervisor can be gone while the +container still runs, and only the engine `Stop` decides that stop, so a listing +does not complete it from PID evidence. The key, the marker and the recorded processes stay until a later +`services stop` is confirmed +(`a_pending_stop_of_a_container_engine_is_not_completed_by_a_gone_supervisor`). +The same holds past the cleanup: the refresh does not read such a record as +`stopped` from the supervisor's PID, and `remove` and `prune` count it as in use +(`container_stop_unresolved`), so a listing followed by `remove --yes` or +`prune --yes` deletes neither the record nor the key the container may still be +enforcing (`a_pending_container_stop_survives_refresh_remove_and_prune`, for a +record reading `ready` and one already reading `stopped`; a process engine in the +same state is removed). A record that named no process at all, such as a `starting` +one with a stale marker, has its key and marker cleaned as before but is not read as +`stopped` (`a_stale_stop_marker_on_a_record_naming_no_process_is_cleaned_without_reading_it_stopped`). +When that refresh completes a pending stop (every recorded process confirmed gone) it +also forgets the recorded processes and reads the record as stopped, as a stop that +confirmed the exit itself does, so a later `rocm services stop` visits no PID: a PID +recorded without a token cannot be refuted, and on Windows the tree kill would take +whatever process now holds the number. A process that may still be running stays +named. `a_pending_stop_keeps_its_key_and_marker_while_a_process_cannot_be_queried` +checks both halves (named while unqueryable, forgotten once gone), +`a_completed_pending_stop_reads_stopped_whatever_live_status_it_had` that every live +status, and `failed`, still ends as `stopped` in that one refresh (a `failed` record +cleared but left `failed` would not be covered by `is_covered_by_stop_of`, and the stop +finalizing would report itself unconfirmed), and +`a_stop_completed_by_a_later_listing_leaves_no_pid_for_the_next_stop` runs the real +termination routine over the record the listing wrote and asserts it visits nothing. +Three gaps in the scenario coverage are deliberate. `service-stop-07` records no +token, so the path a record written by this version takes on Windows (a token +recorded, matching, with the tree kill then running) has no scenario; it rests on +`what_a_stop_does_for_each_recorded_and_live_token` and the other injected tests, +and on the `#[cfg(windows)]` unit tests that stop a real process with its true token. And +the `self` row that `rocm` now puts in `pid_outcomes` has no scenario: it appears +only in the JSON answer of the hidden `sandbox-run stop_server` and the sandbox +tool, never in the text of `rocm services stop`, and it names the stopping +process's own PID, which a scenario cannot plant in a record before the process +that will read it exists. `a_stop_reports_its_own_pid_as_self` and +`a_self_row_adds_no_line_to_an_unconfirmed_report` cover the answer and the report. +Third, the `check:` line an unverified stop prints is a diagnostic, not a +remediation: it shows whether the process is still running, and what clears the +condition is that process ending (or, for a record held by a process `rocm` +cannot open, `rocm services remove --force --yes`; for a record still reading +`ready` that nobody has tried to stop, `--force` is refused and names +`rocm services stop --yes` first, which reports the process unverified and +records the request, so the clearing is those two commands, on purpose: +`--force` never removes a record nobody tried to stop, and +`a_ready_record_held_by_an_unqueryable_process_is_cleared_by_stop_then_force` +runs the chain), after which a stop or a +service listing confirms and drops the key. The command is a stock one the host +has, pinned by `the_process_check_command_is_the_one_the_host_has`, and +`service-stop-07` uses the same `tasklist /FI "PID eq "` filter (in CSV +form, which does not depend on the display language) to see whether a process is +running. A `tasklist` that does not exit successfully is no answer: the scenario +fails instead of reading the missing row as an exit +(`tasklist_reports_running` in `tests/e2e-cucumber/src/tasklist.rs`, a pure read of +the status and output covered by `a_failed_run_is_no_answer_whatever_it_printed` on +every host). No test follows the printed command on a Windows host and reads its +default table output, because that output is localised. What follows once the +process is gone is covered by +`a_pending_stop_keeps_its_key_and_marker_while_a_process_cannot_be_queried`. +`service-stop-06` (Linux) is the same stop of a real recorded process through +`rocm services stop`, checking the persisted state: the process exited and the +record names none. Run the Windows scenario on a Windows host with +`cargo test -p e2e-cucumber --test e2e -- -n "service-stop-07"` +and a prebuilt `ROCM_CLI_BINARY`. + No scenario drives the attached Ctrl-C stop end to end. Attaching needs a real `rocm serve` launch — an engine and a runtime the mock lane does not have, which is why the chat scenarios there plant a record behind a mock server @@ -2270,7 +2548,46 @@ verdict to the JSON; `rocm services stop` exits 1. The stop records the request in the manifest before it signals anything, and applies its result to the manifest as it is once the stop ends, so a write made meanwhile is kept — and a restart that recorded new processes meanwhile makes the stop -unconfirmed rather than having its PIDs cleared. +unconfirmed rather than having its PIDs cleared. What the stop's verdict still +covers is decided by `ManagedServiceRecord::is_covered_by_stop_of`: the same +processes, or none recorded because a liveness refresh that saw every process +gone completed this same stop first (the record reads `stopped` and names +nothing). A restart, whether it has recorded new processes or is mid-launch with +none yet (status `starting`), is not covered, and neither is the empty `stopped` +record a `rocm services restart` leaves between its own confirmed stop and the +publication of its child: the restart stamps a relaunch generation +(`relaunch_started_unix_ms`) before it restores the endpoint key or spawns, a +stop covers only the generation it read, and publishing the child clears it +(`an_older_stop_finalizing_before_a_restart_publishes_does_not_delete_its_key` +finalizes an older stop in that window and asserts the restored key survives; +`a_stop_does_not_cover_a_record_in_another_relaunch_generation` is the unit +table; `a_restart_publishing_its_child_ends_the_relaunch_generation_and_respects_a_moved_one` +covers the publication guard). The unit test +`a_stop_is_covered_by_the_same_processes_or_by_its_own_completion_and_not_by_a_relaunch` +is the table, and it runs again from a stopping snapshot that names no process (an +empty record is that stop only while it carries the stop's marker, reads +`stopped`, or a liveness refresh consumed that marker and left its stamp in +`settled_stop_unix_ms`; an empty `starting` relaunch has cleared the marker +without leaving the stamp and is not covered, +`stop_does_not_confirm_over_an_empty_relaunch_of_an_empty_record`); in `rocm`, +`a_listing_clearing_the_marker_of_an_empty_stop_does_not_make_it_unconfirmed` runs the +real refresh between the termination and the final read of an empty `starting` and an +empty `failed` record (which the refresh leaves at their status) and asserts a confirmed +stop, `a_relaunch_after_a_listing_consumed_the_marker_is_still_not_the_stop` that a +restart following that listing is still not covered (it wipes the stamp), and +`a_listing_that_completes_the_stop_before_finalization_does_not_unconfirm_it` +persists a real liveness refresh between the termination and the final read and +asserts a confirmed stop with the marker not restored, +`a_restart_is_not_refused_because_a_listing_completed_its_stop` that a restart then +carries on past its stop, and `stop_does_not_confirm_over_a_restart_that_has_recorded_no_process_yet` +and `stop_does_not_confirm_when_a_restart_recorded_new_processes_meanwhile` that a real +relaunch is still detected; `rocmd` has the same three (`a_stop_completed_by_a_concurrent_listing_is_still_confirmed`, +`a_stop_does_not_claim_a_restart_that_has_recorded_no_process_yet`, +`a_stop_does_not_clear_processes_recorded_while_it_ran`) plus +`a_stop_of_an_empty_record_whose_marker_a_listing_consumed_is_still_confirmed`, the +listing's write made directly in those that need it because the refresh lives in the +`rocm` binary (in the last one it is made in the engine `Stop` callback, the only +step that runs for a record naming no PID, and the test asserts it ran). `features/service_stop.feature` (`service-stop-02`, Linux, mock lane) drives the real `rocmd sandbox-tool stop_server` against a live recorded @@ -2290,7 +2607,9 @@ checks that the line omits it and no key file exists. Because the seam replaces `terminate_verified`, they say nothing about what is signalled. They need a hooks build, as `cargo xtask e2e` makes: `cargo build -p rocm -p rocmd --features rocm/e2e-test-hooks`, then the command above with `-n -"service-stop"`. +"service-stop-0[1-68]"`. Not `-n "service-stop"`: a name filter bypasses the +suite's own host filter, so it would also run the Windows-only +`service-stop-07` on a Linux host, where it fails for want of PowerShell. `features/engine_ipc.feature` (`engine-ipc-01` and `-02`, Linux, mock lane) covers how the daemon reads an engine child's answer, through the same diff --git a/engines/vllm/src/state.rs b/engines/vllm/src/state.rs index 2691dc62fa..bf2fce5385 100644 --- a/engines/vllm/src/state.rs +++ b/engines/vllm/src/state.rs @@ -484,9 +484,10 @@ mod tests { // Uses the test's own live PID rather than a fixed placeholder so that, // on Linux, the `start_ticks` assertion below exercises the real // `rocm_core::process_start_ticks` lookup rather than only checking - // the field is present. On other platforms `process_start_ticks` is a - // stub that always returns `None`, so there the assertion only checks - // `None == None`. + // the field is present. On Windows it is the process creation time and + // the assertion still compares like with like; on any other platform + // `process_start_ticks` is a stub that always returns `None`, so there + // the assertion only checks `None == None`. let own_pid = std::process::id(); let state_path = probe_state_path("therock-env"); let request = ServeHttpRequest { diff --git a/tests/e2e-cucumber/features/service_record_unqueryable.feature b/tests/e2e-cucumber/features/service_record_unqueryable.feature new file mode 100644 index 0000000000..77ecaf04db --- /dev/null +++ b/tests/e2e-cucumber/features/service_record_unqueryable.feature @@ -0,0 +1,49 @@ +Feature: A local server record held by a process rocm cannot query + + # On Windows a live process the account may not open cannot be queried, so + # `rocm` cannot tell it from a dead server whose PID another program took. + # It treats that as possibly running: a stop reports it unverified, and + # `rocm services remove` refuses the record. `remove --force --yes` is the way + # out for exactly that case, and these scenarios drive the whole path through + # the real binary rather than the pieces. + # + # An unqueryable process cannot be staged from outside the binary, so the + # `e2e-test-hooks` build reads ROCM_E2E_UNQUERYABLE_PIDS and answers + # "cannot be queried" for the PIDs it lists; production builds never read it. + # The records are planted as plain JSON, so no scenario needs a GPU. + # + # `service-unqueryable-01` has no OS tag and the hook works on Windows too, + # but no CI job runs this feature on Windows: the Windows job does not run + # scenarios. It has run on Linux only. + # + # A new feature file, as `service_record_cleanup.feature` explains for its + # own: indexes must be sequential per file, and several open branches claim the + # next ones in the files that exist. + + @id:service-unqueryable-remove-force-clears-an-unverified-record + Scenario: service-unqueryable-01 - A record whose process cannot be queried is stopped as unverified, refused by remove, and cleared by remove --force + Given a local server record whose recorded process cannot be queried + When the user stops that local server while its process cannot be queried + Then the CLI reports the stop as not confirmed because the process could not be queried + And every file belonging to that record is still there + When the user removes that local server record while its process cannot be queried + Then the CLI refuses the removal and names the forced removal + And every file belonging to that record is still there + When the user prunes every record whatever its age while its process cannot be queried + Then the CLI leaves the record alone and names the forced removal + And every file belonging to that record is still there + When the user force-removes that local server record while its process cannot be queried + Then the CLI removes the record and says its process could not be queried + And every file belonging to that record is gone + + # `--force` is for a process `rocm` cannot query, never for one it can see + # running. The recorded process here is a real one the scenario started, and a + # stop was requested on the record, so the only thing standing in the way is + # that the process is running. + @id:service-unqueryable-force-never-passes-a-running-process + @requires-os:linux + Scenario: service-unqueryable-02 - --force is refused for a record whose process is confirmed running + Given a local server record whose recorded process is running and whose stop was requested + When the user force-removes that local server record + Then the CLI refuses the forced removal because the record does not qualify + And every file belonging to that record is still there diff --git a/tests/e2e-cucumber/features/service_stop.feature b/tests/e2e-cucumber/features/service_stop.feature index b32ef7e933..2c30645420 100644 --- a/tests/e2e-cucumber/features/service_stop.feature +++ b/tests/e2e-cucumber/features/service_stop.feature @@ -124,3 +124,73 @@ Feature: Stopping a managed service Then the CLI reports the stop as not confirmed and names no stop command And the server's record still names its process and the stop request And the report says only the record was kept, and there is no key file + + # `rocm services stop`'s confirmed half with a real process. service-stop-01 + # plants PID 0, so it never signals anything and cannot show what the stop + # leaves in the record; service-stop-02 shows it for the daemon's tool. This + # runs the CLI's own stop against a real recorded process and checks the + # persisted state it claims: the process exited, and the record names no + # process any more, so a second stop of that record would signal nothing (a + # PID recorded without a start-time token cannot be refuted once the OS + # recycles it). The unit test + # `a_confirmed_stop_clears_the_recorded_processes_so_a_second_stop_signals_nothing` + # covers the second stop itself. + # + # Linux-only for the same reason as service-stop-02: the recorded process + # carries the start-time token `/proc` gives it. + + @id:service-stop-cli-confirms-a-stop-and-clears-the-record + @requires-os:linux + Scenario: service-stop-06 - `rocm services stop` of a running managed server confirms the stop and leaves no process in its record + Given a managed server whose recorded process is running + When the user stops it with --yes + Then the CLI reports the service as stopped + And the recorded process is no longer running + And the server's record reads stopped and names no process + And the server's endpoint key file is gone + + # The Windows half of the stop. A managed engine runs one level below the + # recorded process (the daemon launches it through an `__engine-serve-http` + # process), and `terminate_verified` cannot walk a process tree on Windows, so + # the stop first runs `taskkill /PID /T /F` on the recorded process. + # service-stop-01 plants PID 0 and never reaches that step, and the scenarios + # above need Linux's start-time token, so none of them kills a real tree. + # + # This records a real parent that has started a real child, stops it, and + # checks that BOTH are gone together with the report and the record. Without + # the tree-kill step the stop ends only the recorded process, the child keeps + # running, and the scenario fails on it. No GPU, no engine, no network. The + # record carries no start-time token, like a Windows record written before the + # token was recorded: the case where the PID cannot be refuted, which the tree + # kill and the cleared record exist for. service-stop-08 below is the same stop + # with a token that does not match. + + @id:service-stop-takes-down-the-whole-process-tree-on-windows + @requires-os:windows + Scenario: service-stop-07 - Stopping a managed server on Windows takes down its recorded process together with the process it started + Given a managed server whose recorded process has started another process + When the user stops it with --yes + Then the CLI reports the service as stopped + And the report counts one stopped process + And the recorded process and the process it started are no longer running + And the server's record reads stopped and names no process + And the server's endpoint key file is gone + + # A recorded PID is not proof that the recorded process still holds it: the + # server can exit by itself and the OS hand its PID to an unrelated program. + # The record pairs the PID with the process's start-time token (Linux's + # `/proc` start time, Windows' creation time), and a stop whose token no longer + # matches treats the recorded process as already gone: it reports the stop, + # clears the record and leaves the process that now holds the PID running, + # instead of signalling a stranger (on Windows, with `taskkill /T /F`, its + # whole subtree). The stranger here is a real process, so "left alone" is + # checked on the process itself. Runs on every OS: the planted token is one + # no live process has. + @id:service-stop-leaves-a-process-that-reused-the-recorded-pid-alone + Scenario: service-stop-08 - Stopping a managed server whose recorded PID now belongs to another process leaves that process running + Given a managed server whose recorded process id now belongs to a different process + When the user stops it with --yes + Then the CLI reports the service as stopped + And the process now holding the recorded process id is still running + And the server's record reads stopped and names no process + And the server's endpoint key file is gone diff --git a/tests/e2e-cucumber/src/lib.rs b/tests/e2e-cucumber/src/lib.rs index 27fc415a4f..a89e72dd29 100644 --- a/tests/e2e-cucumber/src/lib.rs +++ b/tests/e2e-cucumber/src/lib.rs @@ -30,6 +30,8 @@ pub mod serve_log; pub mod serve_plan; pub mod shared_runtime; pub mod stand_in_uname; +pub mod stray_process; +pub mod tasklist; pub mod terminal_size; use std::path::{Path, PathBuf}; diff --git a/tests/e2e-cucumber/src/stray_process.rs b/tests/e2e-cucumber/src/stray_process.rs new file mode 100644 index 0000000000..815b5a9271 --- /dev/null +++ b/tests/e2e-cucumber/src/stray_process.rs @@ -0,0 +1,75 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! What scenario teardown does about a process a scenario started and recorded. +//! +//! A scenario that records a process a stop is supposed to take down (a parent +//! and the child it started, say) can fail with one of them left running, and a +//! later stop cannot find a child whose parent is gone. Teardown therefore +//! keeps the child's own identity and ends it itself. Whether to do so is +//! decided here, apart from the system reads, so a table test covers each case +//! on every host: only a process that is still the one recorded is ended, never +//! one that has since exited or whose PID another process has taken. + +/// What the host says about a recorded process now. +/// +/// It is read against the identity the scenario captured for the process. The +/// harness maps the production identity check onto these (`rocm-core` is a +/// dev-dependency of this crate, so the library does not name it). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RecordedState { + /// Running, and the same process (or recorded without a token, which cannot + /// be refuted). + Same, + /// No longer running. + Gone, + /// Running, but provably a different process that took the PID. + Reused, + /// Running, but its identity cannot be confirmed. + Unconfirmed, +} + +/// What teardown does about one recorded process. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Leftover { + /// End it: it is still the process the scenario recorded. + Terminate, + /// Leave it: it is gone, or is not (provably) the recorded process. + Leave, +} + +/// The teardown action for a recorded process whose identity reads as `state`. +/// +/// A process that cannot be confirmed as the recorded one is left alone: ending +/// a stranger that happens to hold a recycled PID is worse than leaving a +/// stray five-minute process to time out. +#[must_use] +pub const fn leftover_action(state: RecordedState) -> Leftover { + match state { + RecordedState::Same => Leftover::Terminate, + RecordedState::Gone | RecordedState::Reused | RecordedState::Unconfirmed => Leftover::Leave, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_a_process_that_is_still_the_recorded_one_is_ended() { + for (state, expected) in [ + // Alive and the same process (or recorded without a token, which + // cannot be refuted): the stray child a failed stop left behind. + (RecordedState::Same, Leftover::Terminate), + // The stop took it down already. + (RecordedState::Gone, Leftover::Leave), + // Its PID has been reused: a stranger. + (RecordedState::Reused, Leftover::Leave), + // Its token cannot be read back, so it cannot be confirmed. + (RecordedState::Unconfirmed, Leftover::Leave), + ] { + assert_eq!(leftover_action(state), expected, "{state:?}"); + } + } +} diff --git a/tests/e2e-cucumber/src/tasklist.rs b/tests/e2e-cucumber/src/tasklist.rs new file mode 100644 index 0000000000..35aea69c67 --- /dev/null +++ b/tests/e2e-cucumber/src/tasklist.rs @@ -0,0 +1,83 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! What `tasklist` says about one process, read so that a failed query cannot +//! pass for an answer. +//! +//! A scenario that wants to know whether a process a stop was meant to take down +//! is still running asks `tasklist /FI "PID eq N" /FO CSV /NH`. A match prints a +//! CSV row naming the PID and no match prints an INFO line without one, but the +//! command can also fail (it cannot be started under that name, it is refused, it +//! times out) and print nothing useful. Reading "no row" from such a run as "the +//! process exited" lets the scenario pass while the process may still be running, +//! so the exit status is judged before the text. The read is a pure function of +//! the command's status and output, so a table test covers every case on every +//! host. + +/// Whether `tasklist` reported process `pid` running. +/// +/// `success` is whether the command exited successfully and `stdout` is what it +/// printed. `Ok(true)` is a CSV row for `pid`, `Ok(false)` is a successful run +/// with no such row, and `Err` is a run that did not succeed: its output says +/// nothing about the process, so there is no answer. +/// +/// # Errors +/// +/// Returns a message naming the PID and what the failed run printed when the +/// command did not exit successfully. +pub fn tasklist_reports_running( + success: bool, + stdout: &str, + stderr: &str, + pid: u32, +) -> Result { + if !success { + return Err(format!( + "`tasklist` did not exit successfully, so it cannot say whether process {pid} \ + is still running (stdout: {:?}, stderr: {:?})", + stdout.trim(), + stderr.trim() + )); + } + Ok(stdout.contains(&format!("\"{pid}\""))) +} + +#[cfg(test)] +mod tests { + use super::*; + + const ROW: &str = "\"ping.exe\",\"4242\",\"Console\",\"1\",\"3,044 K\"\r\n"; + const NO_MATCH: &str = "INFO: No tasks are running which match the specified criteria.\r\n"; + + #[test] + fn a_successful_run_with_a_row_for_the_pid_is_running() { + assert_eq!(tasklist_reports_running(true, ROW, "", 4242), Ok(true)); + } + + #[test] + fn a_successful_run_without_a_row_for_the_pid_is_not_running() { + assert_eq!( + tasklist_reports_running(true, NO_MATCH, "", 4242), + Ok(false) + ); + // A row for another PID is not a row for this one, nor is a PID that + // only contains it. + assert_eq!(tasklist_reports_running(true, ROW, "", 424), Ok(false)); + assert_eq!(tasklist_reports_running(true, ROW, "", 7), Ok(false)); + } + + #[test] + fn a_failed_run_is_no_answer_whatever_it_printed() { + // No output to read: "no row" must not become "exited". + let err = tasklist_reports_running(false, "", "Access is denied.", 4242) + .expect_err("a failed query is not an answer"); + assert!( + err.contains("4242") && err.contains("Access is denied."), + "{err}" + ); + // Nor does a row printed before the failure make it one. + assert!(tasklist_reports_running(false, ROW, "", 4242).is_err()); + assert!(tasklist_reports_running(false, NO_MATCH, "", 4242).is_err()); + } +} diff --git a/tests/e2e-cucumber/tests/e2e.rs b/tests/e2e-cucumber/tests/e2e.rs index 7db98ab508..f433460695 100644 --- a/tests/e2e-cucumber/tests/e2e.rs +++ b/tests/e2e-cucumber/tests/e2e.rs @@ -215,6 +215,11 @@ pub struct E2eWorld { /// reads the record, but a real `rocm services stop e2e-mock --yes` would /// then signal the harness process itself. pub recorded_process: Option, + /// The identity of a process the recorded one started (`service-stop-07`'s + /// `ping`). Teardown ends it on its own account, because a stop that fails + /// to take it down leaves it running with a dead parent, where nothing else + /// can find it. + pub recorded_descendant: Option, /// Whether the Doctor report preview in the dashboard console was the CLI's /// refusal envelope (it carries a `refused` key). Recorded when the preview /// is read, so a later step can check the card against what the CLI said. @@ -386,6 +391,7 @@ impl Default for E2eWorld { skill_symptom: None, helper_identity: None, recorded_process: None, + recorded_descendant: None, dash_report_refused: None, automation_daemon: None, } @@ -734,6 +740,21 @@ impl Drop for E2eWorld { let _ = process.kill(); let _ = process.wait(); } + // The process the recorded one started: ended too when it is still the + // process that was recorded, so a failed scenario does not leave it + // running until its own timeout. + if let Some(descendant) = self.recorded_descendant.take() { + use e2e_cucumber::stray_process::{Leftover, RecordedState, leftover_action}; + let state = match rocm_core::identity_state(&descendant) { + rocm_core::IdentityState::Matches => RecordedState::Same, + rocm_core::IdentityState::Gone => RecordedState::Gone, + rocm_core::IdentityState::Recycled => RecordedState::Reused, + rocm_core::IdentityState::Indeterminate => RecordedState::Unconfirmed, + }; + if leftover_action(state) == Leftover::Terminate { + let _ = rocm_core::terminate_recorded_pid(&descendant); + } + } // A scenario that enabled a watcher started a real detached `rocm daemon`, // which only exits once the watcher is disabled. If a step failed first, // nothing else would stop it, and it would keep polling a config dir diff --git a/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs b/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs index 16f5d589d2..375a3dfa39 100644 --- a/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_cleanup_steps.rs @@ -696,3 +696,201 @@ fn assert_succeeded(world: &E2eWorld) { combined_output(world) ); } + +// ── A record held by a process `rocm` cannot query ───────────────── + +/// A PID no host is using, which the `e2e-test-hooks` build is told to read as +/// "cannot be queried" (`ROCM_E2E_UNQUERYABLE_PIDS`). +const UNQUERYABLE_PID: u32 = 2_000_000_001; + +/// Rewrite fields of the planted manifest. +fn set_manifest_fields(world: &E2eWorld, fields: &[(&str, serde_json::Value)]) { + let manifest = services_dir(world).join(format!("{SERVICE_ID}.json")); + let mut record: serde_json::Value = serde_json::from_slice( + &std::fs::read(&manifest).expect("failed to read the planted manifest"), + ) + .expect("failed to parse the planted manifest"); + for (key, value) in fields { + record[*key] = value.clone(); + } + std::fs::write( + &manifest, + serde_json::to_vec_pretty(&record).expect("failed to serialize record"), + ) + .expect("failed to rewrite the planted manifest"); +} + +/// Make the next `rocm` invocation read [`UNQUERYABLE_PID`] as unqueryable. +fn script_unqueryable(world: &mut E2eWorld) { + world.command_env.push(( + "ROCM_E2E_UNQUERYABLE_PIDS", + UNQUERYABLE_PID.to_string().into(), + )); +} + +#[given("a local server record whose recorded process cannot be queried")] +async fn record_with_unqueryable_process(world: &mut E2eWorld) { + plant_record(world, "ready"); + set_manifest_fields( + world, + &[("supervisor_pid", serde_json::json!(UNQUERYABLE_PID))], + ); +} + +#[given("a local server record whose recorded process is running and whose stop was requested")] +async fn record_with_running_process(world: &mut E2eWorld) { + plant_record(world, "ready"); + // A real process the scenario owns, so the teardown (which stops every + // record left behind) ends it rather than the harness. + let child = std::process::Command::new("sleep") + .arg("300") + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .expect("failed to start the recorded process"); + set_manifest_fields( + world, + &[ + ("supervisor_pid", serde_json::json!(child.id())), + ("stop_requested_unix_ms", serde_json::json!(1)), + ], + ); + world.recorded_process = Some(child); +} + +#[when("the user stops that local server while its process cannot be queried")] +async fn stop_unqueryable(world: &mut E2eWorld) { + script_unqueryable(world); + let (stdout, stderr, rc) = + crate::run_rocm_with_scenario_env(world, &["services", "stop", SERVICE_ID, "--yes"]); + record(world, stdout, stderr, rc); +} + +#[when("the user removes that local server record while its process cannot be queried")] +async fn remove_unqueryable(world: &mut E2eWorld) { + script_unqueryable(world); + let (stdout, stderr, rc) = + crate::run_rocm_with_scenario_env(world, &["services", "remove", SERVICE_ID, "--yes"]); + record(world, stdout, stderr, rc); +} + +#[when("the user force-removes that local server record while its process cannot be queried")] +async fn force_remove_unqueryable(world: &mut E2eWorld) { + script_unqueryable(world); + let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env( + world, + &["services", "remove", SERVICE_ID, "--force", "--yes"], + ); + record(world, stdout, stderr, rc); +} + +#[when("the user prunes every record whatever its age while its process cannot be queried")] +async fn prune_unqueryable(world: &mut E2eWorld) { + script_unqueryable(world); + let (stdout, stderr, rc) = + crate::run_rocm_with_scenario_env(world, &["services", "prune", "--any-age", "--yes"]); + record(world, stdout, stderr, rc); +} + +#[then("the CLI leaves the record alone and names the forced removal")] +async fn prune_leaves_and_names_force(world: &mut E2eWorld) { + assert_succeeded(world); + let combined = combined_output(world); + assert!( + combined.contains(&format!( + "{SERVICE_ID} is " + )) && combined.contains(&format!( + "process {UNQUERYABLE_PID} it recorded could not be queried - remove it with `rocm services remove {SERVICE_ID} --force --yes`" + )), + "{combined}" + ); +} + +#[when("the user force-removes that local server record")] +async fn force_remove(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm( + world, + &["services", "remove", SERVICE_ID, "--force", "--yes"], + ); + record(world, stdout, stderr, rc); +} + +#[then("the CLI reports the stop as not confirmed because the process could not be queried")] +async fn stop_not_confirmed_unverified(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no command rc recorded"); + let combined = combined_output(world); + assert_ne!(rc, 0, "an unverified stop must exit non-zero:\n{combined}"); + assert!( + combined.contains("Local server stop not confirmed"), + "{combined}" + ); + assert!( + combined.contains(&format!("not signaled: process {UNQUERYABLE_PID}")), + "{combined}" + ); + // The check the stop names is the host's own command, and names the PID. + assert!( + combined.contains(&format!("ps -p {UNQUERYABLE_PID} -o pid,args")) + || combined.contains(&format!("tasklist /FI \"PID eq {UNQUERYABLE_PID}\"")), + "{combined}" + ); + // What the stop says it kept is what is on disk: the record names the + // process and the stop request is recorded. + let manifest: serde_json::Value = serde_json::from_slice( + &std::fs::read(services_dir(world).join(format!("{SERVICE_ID}.json"))) + .expect("the record must still be there"), + ) + .expect("the record must parse"); + assert_eq!(manifest["supervisor_pid"], UNQUERYABLE_PID, "{manifest}"); + assert!(manifest["stop_requested_unix_ms"].is_u64(), "{manifest}"); +} + +#[then("the CLI refuses the removal and names the forced removal")] +async fn refuses_and_names_force(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no command rc recorded"); + let combined = combined_output(world); + assert_ne!(rc, 0, "{combined}"); + assert!( + combined.contains(&format!( + "process {UNQUERYABLE_PID} it recorded could not be queried" + )), + "{combined}" + ); + // The command the refusal names is the one the next step runs. + assert!( + combined.contains(&format!("rocm services remove {SERVICE_ID} --force --yes")), + "{combined}" + ); +} + +#[then("the CLI removes the record and says its process could not be queried")] +async fn force_removed(world: &mut E2eWorld) { + assert_succeeded(world); + let combined = combined_output(world); + assert!( + combined.contains(&format!( + "--force: process {UNQUERYABLE_PID} could not be queried" + )), + "{combined}" + ); + assert!( + combined.contains("Local server record removed"), + "{combined}" + ); +} + +#[then("the CLI refuses the forced removal because the record does not qualify")] +async fn refuses_force(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no command rc recorded"); + let combined = combined_output(world); + assert_ne!(rc, 0, "{combined}"); + assert!( + combined.contains("--force removes a record only when"), + "{combined}" + ); + assert!( + combined.contains(&format!("rocm services stop {SERVICE_ID} --yes")), + "{combined}" + ); +} diff --git a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs index 32153a060d..c45a81b7e4 100644 --- a/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/service_stop_steps.rs @@ -166,9 +166,171 @@ fn plant_server_with_running_process(world: &mut E2eWorld, host: &str, with_key: .stderr(Stdio::null()) .spawn() .expect("failed to start the process to record"); + let ticks = start_ticks(process.id()); + plant_server_record(world, host, with_key, process, Some(ticks)); +} + +/// A ready, publicly bound server whose recorded PID is a live process that is +/// not the server: what a recycled PID looks like. The record's start-time token +/// is not the process's own, so the stop has to refuse the PID rather than signal +/// it. The token is the process's real one offset by one on Linux, and `1` (a +/// `FILETIME` in 1601, which no live process has) on Windows. +#[given("a managed server whose recorded process id now belongs to a different process")] +async fn managed_server_with_a_recycled_pid(world: &mut E2eWorld) { + let (program, args): (&str, &[&str]) = if cfg!(windows) { + ("ping", &["-n", "300", "127.0.0.1"]) + } else { + ("sleep", &["300"]) + }; + let bystander = Command::new(program) + .args(args) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("failed to start the process that holds the recorded PID"); + let stale_ticks = if cfg!(windows) { + 1 + } else { + start_ticks(bystander.id()).wrapping_add(1) + }; + plant_server_record(world, "0.0.0.0", true, bystander, Some(stale_ticks)); +} + +/// The stop's promise about the process that holds the recorded PID, checked on +/// the process: it was not signalled, so it is still running. +#[then("the process now holding the recorded process id is still running")] +async fn process_holding_the_pid_still_runs(world: &mut E2eWorld) { + let process = world + .recorded_process + .as_mut() + .expect("scenario recorded no process"); + assert!( + process + .try_wait() + .expect("failed to check the recorded process") + .is_none(), + "the stop signalled a process that is not the recorded server" + ); +} + +/// Where the parent the Windows scenario records writes the PID of the process +/// it started, for the scenario to read. +fn descendant_pid_path(world: &E2eWorld) -> PathBuf { + services_dir(world).join("descendant.pid") +} + +/// A ready, publicly bound server whose recorded supervisor is a parent process +/// that has started a child of its own, recorded the way a Windows record was +/// before Windows recorded a creation time: the PID alone, with no start-time +/// token. (A record written now carries one; this scenario plants the tokenless +/// shape on purpose, the one a PID cannot be refuted by.) +/// +/// The parent is PowerShell, the child `ping` for five minutes: both ship with +/// Windows, need nothing installed, and are disposable. PowerShell starts the +/// child itself, so the child's parent is the recorded PID, which is the +/// relationship `taskkill /T` follows; it writes the child's PID where the +/// scenario can read it and then stays alive. Both outlive the scenario only if +/// the stop fails to take them, and the World's teardown reaps the parent. +#[given("a managed server whose recorded process has started another process")] +async fn managed_server_whose_process_started_another(world: &mut E2eWorld) { + let services = services_dir(world); + std::fs::create_dir_all(&services).expect("failed to create services dir"); + let pid_file = descendant_pid_path(world); + let process = spawn_parent_with_descendant(&pid_file); + let pid = process.id(); + plant_server_record(world, "0.0.0.0", true, process, None); + + // The premise, so the Then step cannot pass for the wrong reason: the + // descendant exists and is running, and so is the parent. + let descendant = wait_for_descendant_pid(&pid_file); + // Kept first, with its creation time where the host reads one, so teardown + // can end the child on its own even when the stop (or a step) fails and the + // parent is gone: a later stop cannot recover a child from a dead parent. + world.recorded_descendant = Some(rocm_core::ProcessIdentity::capture(descendant)); + assert_ne!(descendant, pid, "the child must be a different process"); + assert!( + windows_process_is_running(descendant), + "premise: the process the recorded one started must be running" + ); + assert!( + world + .recorded_process + .as_mut() + .expect("scenario recorded no process") + .try_wait() + .expect("failed to check the recorded process") + .is_none(), + "premise: the recorded process must be running" + ); +} + +fn spawn_parent_with_descendant(pid_file: &std::path::Path) -> std::process::Child { + // A single-quoted PowerShell string: an apostrophe in the path is doubled. + let pid_file = pid_file.display().to_string().replace('\'', "''"); + let script = format!( + "$child = Start-Process -FilePath ping.exe -ArgumentList '-n','300','127.0.0.1' \ + -WindowStyle Hidden -PassThru; \ + Set-Content -LiteralPath '{pid_file}' -Value $child.Id; \ + Start-Sleep -Seconds 300" + ); + Command::new("powershell") + .args(["-NoProfile", "-NonInteractive", "-Command", &script]) + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .expect("failed to start the process to record") +} + +/// The child's PID, once the parent has written it. +fn wait_for_descendant_pid(pid_file: &std::path::Path) -> u32 { + let deadline = std::time::Instant::now() + std::time::Duration::from_mins(1); + loop { + if let Some(pid) = std::fs::read_to_string(pid_file) + .ok() + .and_then(|text| text.trim().parse().ok()) + { + return pid; + } + assert!( + std::time::Instant::now() < deadline, + "the recorded process never wrote the PID of the process it started" + ); + std::thread::sleep(std::time::Duration::from_millis(200)); + } +} + +/// Whether Windows has a process `pid`, by `tasklist`: a match prints a CSV +/// row naming it, and no match prints an INFO line without one. A `tasklist` +/// that fails has said nothing about the process, so the scenario fails rather +/// than read it as "exited" (`e2e_cucumber::tasklist`). +fn windows_process_is_running(pid: u32) -> bool { + let output = Command::new("tasklist") + .args(["/FI", &format!("PID eq {pid}"), "/FO", "CSV", "/NH"]) + .output() + .expect("failed to run tasklist"); + e2e_cucumber::tasklist::tasklist_reports_running( + output.status.success(), + &String::from_utf8_lossy(&output.stdout), + &String::from_utf8_lossy(&output.stderr), + pid, + ) + .unwrap_or_else(|message| panic!("{message}")) +} + +/// Write the record for a ready server whose supervisor is `process`, with +/// `ticks` as its start-time token when there is one, and take `process` into +/// the World so the scenario (and its teardown) can check and reap it. +fn plant_server_record( + world: &mut E2eWorld, + host: &str, + with_key: bool, + process: std::process::Child, + ticks: Option, +) { let pid = process.id(); world.recorded_process = Some(process); - let ticks = start_ticks(pid); let services = services_dir(world); std::fs::create_dir_all(&services).expect("failed to create services dir"); @@ -508,19 +670,63 @@ async fn tool_still_reports_stopped(world: &mut E2eWorld) { #[then("the recorded process is no longer running")] async fn recorded_process_gone(world: &mut E2eWorld) { + assert_recorded_process_exited(world); +} + +/// The Windows tree kill, observed: the recorded process AND the one it started +/// are gone after the stop. Without the `taskkill /T` step the stop ends only +/// the recorded process (`TerminateProcess` on the root), and the child keeps +/// running, which is what the second check catches. +#[then("the recorded process and the process it started are no longer running")] +async fn recorded_process_and_its_child_gone(world: &mut E2eWorld) { + assert_recorded_process_exited(world); + let pid = wait_for_descendant_pid(&descendant_pid_path(world)); + // `taskkill` returns once it has asked for the termination; give the exit + // a moment to land before calling the process a survivor. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(15); + while windows_process_is_running(pid) { + assert!( + std::time::Instant::now() < deadline, + "the stop took the recorded process down but left the process it \ + started (PID {pid}) running: the tree kill did not reach it" + ); + std::thread::sleep(std::time::Duration::from_millis(250)); + } +} + +/// The report says one process was stopped (the recorded one: a tree kill that +/// already took the root is reported as forced, so it counts), checked next to +/// the processes themselves rather than instead of them. +#[then("the report counts one stopped process")] +async fn report_counts_one_stopped_process(world: &mut E2eWorld) { + let combined = combined_output(world); + assert!(combined.contains("stopped processes: 1"), "{combined}"); +} + +fn assert_recorded_process_exited(world: &mut E2eWorld) { let process = world .recorded_process .as_mut() .expect("scenario recorded no process"); // The report's claim, checked against the process itself: it has exited - // (and is reaped here), rather than merely having been signalled. - let exited = process - .try_wait() - .expect("failed to check the recorded process"); - assert!( - exited.is_some(), - "the stop reported the process gone, but it is still running" - ); + // (and is reaped here), rather than merely having been signalled. The stop + // confirms an exit from the OS's exit code, and this handle is signalled by + // the process object's teardown, so allow that teardown a few seconds on a + // loaded host before calling a process a survivor. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + loop { + let exited = process + .try_wait() + .expect("failed to check the recorded process"); + if exited.is_some() { + break; + } + assert!( + std::time::Instant::now() < deadline, + "the stop reported the process gone, but it is still running" + ); + std::thread::sleep(std::time::Duration::from_millis(200)); + } } #[then("the server's record reads stopped and names no process")] diff --git a/tests/e2e-cucumber/tests/feature_naming.rs b/tests/e2e-cucumber/tests/feature_naming.rs index bb1f06cae9..47dcb5d9d6 100644 --- a/tests/e2e-cucumber/tests/feature_naming.rs +++ b/tests/e2e-cucumber/tests/feature_naming.rs @@ -54,6 +54,7 @@ const FEATURE_KEYS: &[(&str, &str)] = &[ ("runtime_lifecycle.feature", "runtime-lifecycle"), ("runtime_setup.feature", "runtime"), ("service_record_cleanup.feature", "service-cleanup"), + ("service_record_unqueryable.feature", "service-unqueryable"), ("service_stop.feature", "service-stop"), ("service_unknown_id.feature", "service-unknown"), ("storage.feature", "storage"),