From 74d86d557bda0354fae4b88fcfdc53b126e03b9b Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 3 Nov 2026 14:02:01 +0200 Subject: [PATCH 1/9] feat(core): watch a Windows detached spawn for an early exit Add spawn_detached_no_inherit_watching_startup next to spawn_detached_no_inherit. It keeps the same creation flags and handle inheritance (DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW | CREATE_UNICODE_ENVIRONMENT, no inherited handles). It also waits on the child's process handle for a bounded settle window before closing it, and returns the exit code as DetachedSpawn::early_exit_code when the child has already exited. The check runs while the CreateProcessW handle is still open, so it does not carry the PID-reuse race that re-opening the process by pid would. GetExitCodeProcess is read only after WAIT_OBJECT_0. Any ambiguous result is reported as "still running". Also export MANAGED_ENGINE_STARTUP_FAILURE_TEST_ENV, the one name the rocm binary and the E2E suite share for the scripted startup death. Signed-off-by: Roman Inflianskas --- crates/rocm-core/src/lib.rs | 18 ++- crates/rocm-core/src/process.rs | 195 +++++++++++++++++++++++++++++++- 2 files changed, 205 insertions(+), 8 deletions(-) diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 2147a8e00c..8baeb79e8d 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -184,17 +184,17 @@ pub use proc_lifecycle::{ IdentityState, KillScope, ProcessIdentity, TerminationOutcome, identity_state, process_start_ticks, terminate_verified, }; +#[cfg(windows)] +pub use process::{ + DetachedSpawn, spawn_detached_no_inherit, spawn_detached_no_inherit_watching_startup, + spawn_hidden_console_no_inherit, spawn_hidden_console_with_log, wait_for_process_exit, +}; pub use process::{ FileLock, LoopbackPortLease, MANAGED_LAUNCH_LOCK_TIMEOUT, detach_command_session, is_canonical_loopback_host, launch_port_argument, lease_loopback_port, lease_loopback_port_or_bind_error, process_is_running, terminate_process, terminate_process_tree, }; -#[cfg(windows)] -pub use process::{ - spawn_detached_no_inherit, spawn_hidden_console_no_inherit, spawn_hidden_console_with_log, - wait_for_process_exit, -}; pub use quoting::{posix_shell_quote, powershell_quote}; pub use report::{ APPROVED_ARCHITECTURES, APPROVED_ARCHITECTURES_SOURCE, GROUP_AXES, REPORT_SCHEMA_VERSION, @@ -247,6 +247,14 @@ pub use vllm_oom::{ vllm_oom_display_line, }; +/// E2E-only switch that makes `rocm serve`'s managed engine die during startup. +/// +/// Honoured only by a `rocm` binary built with the `e2e-test-hooks` feature; a +/// release build has no code that reads it. Defined here, rather than in the +/// binary that reads it, because the E2E suite sets it from another crate: one +/// constant both sides name cannot drift the way two copies of the string could. +pub const MANAGED_ENGINE_STARTUP_FAILURE_TEST_ENV: &str = "ROCM_E2E_MANAGED_ENGINE_STARTUP_FAILURE"; + /// The variable that opts a machine out of rocm-cli choosing its runtime's torch. pub const TORCH_ALIGNMENT_DISABLED_ENV: &str = "ROCM_CLI_DISABLE_TORCH_ALIGNMENT"; diff --git a/crates/rocm-core/src/process.rs b/crates/rocm-core/src/process.rs index 2e285e3169..97e49ef70f 100644 --- a/crates/rocm-core/src/process.rs +++ b/crates/rocm-core/src/process.rs @@ -52,6 +52,27 @@ use std::sync::{Arc, Mutex}; use std::thread; use std::time::{Duration, Instant}; +/// Outcome of a detached Windows spawn that was briefly watched for an early exit. +/// +/// The observation runs while the process handle returned by `CreateProcessW` is +/// still open, so a reported exit code is always the exit code of the process that +/// was just spawned. Re-opening the process by PID after the fact could not offer +/// that guarantee — Windows recycles PIDs, so by then the PID may name an +/// unrelated process. +#[cfg(windows)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DetachedSpawn { + /// PID of the spawned process. + pub pid: u32, + /// `Some(code)` if the process exited at any point within the observation + /// window — known as soon as it exits, since the wait returns then rather + /// than at the end of the window — and `None` if it was still running when + /// the window ran out. `None` is also reported if the wait failed or the + /// exit code could not be read, which degrades to the unwatched behaviour + /// rather than inventing a failure. + pub early_exit_code: Option, +} + #[cfg(windows)] pub fn spawn_detached_no_inherit( program: &Path, @@ -65,6 +86,38 @@ pub fn spawn_detached_no_inherit( DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW | CREATE_UNICODE_ENVIRONMENT, false, None, + None, + ) + .map(|spawn| spawn.pid) +} + +/// As [`spawn_detached_no_inherit`], but watches the new process for up to +/// `settle` before returning, so the caller can tell "started" apart from +/// "started and died immediately". +/// +/// Creation flags and handle inheritance are identical to +/// [`spawn_detached_no_inherit`]: the child stays fully detached and outlives this +/// process. The wait is bounded and delays the caller by at most `settle` — less +/// when the child exits sooner; it is not a join. +/// +/// This exists because the detached spawn primitives hand back a bare PID rather +/// than a `std::process::Child`, so a caller has no equivalent of `try_wait()` to +/// notice a child that failed during startup. +#[cfg(windows)] +pub fn spawn_detached_no_inherit_watching_startup( + program: &Path, + args: &[String], + env_overrides: &[(&str, &Path)], + settle: Duration, +) -> Result { + spawn_windows_no_inherit( + program, + args, + env_overrides, + DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW | CREATE_UNICODE_ENVIRONMENT, + false, + None, + Some(settle), ) } @@ -81,7 +134,9 @@ pub fn spawn_hidden_console_no_inherit( CREATE_NEW_CONSOLE | CREATE_NEW_PROCESS_GROUP | CREATE_UNICODE_ENVIRONMENT, true, None, + None, ) + .map(|spawn| spawn.pid) } #[cfg(windows)] @@ -150,12 +205,13 @@ pub fn spawn_hidden_console_with_log( CREATE_NEW_CONSOLE | CREATE_NEW_PROCESS_GROUP | CREATE_UNICODE_ENVIRONMENT, true, Some((stdout_handle, stderr_handle)), + None, ); unsafe { CloseHandle(stdout_handle); CloseHandle(stderr_handle); } - result + result.map(|spawn| spawn.pid) } #[cfg(windows)] @@ -708,7 +764,8 @@ fn spawn_windows_no_inherit( windows_sys::Win32::Foundation::HANDLE, windows_sys::Win32::Foundation::HANDLE, )>, -) -> Result { + settle: Option, +) -> Result { use std::ptr::{null, null_mut}; use windows_sys::Win32::Foundation::CloseHandle; @@ -752,11 +809,63 @@ fn spawn_windows_no_inherit( std::io::Error::last_os_error() ); } + // Observe the child *before* the handle is closed. Waiting on this handle is + // race-free; re-opening the process later by PID would not be, since Windows + // recycles PIDs and the PID could by then belong to something else. + let early_exit_code = + settle.and_then(|settle| unsafe { observe_early_exit(process_info.hProcess, settle) }); unsafe { CloseHandle(process_info.hThread); CloseHandle(process_info.hProcess); } - Ok(process_info.dwProcessId) + Ok(DetachedSpawn { + pid: process_info.dwProcessId, + early_exit_code, + }) +} + +/// Wait up to `settle` for `process` to exit, reporting its exit code if it did. +/// +/// Returns `None` both when the process is still running and when the exit code +/// could not be read, so a failed query degrades to "assume it is alive" rather +/// than reporting a startup failure that may not have happened. +/// +/// # Safety +/// +/// `process` must be an open process handle that is not closed for the duration +/// of the call; this function does not close it. The only caller passes the +/// `hProcess` that `CreateProcessW` just returned, which carries full access +/// (`PROCESS_ALL_ACCESS`), so the rights the two calls need are always present: +/// `SYNCHRONIZE` for `WaitForSingleObject`, and `PROCESS_QUERY_LIMITED_INFORMATION` +/// (or the broader `PROCESS_QUERY_INFORMATION`) for `GetExitCodeProcess`. +#[cfg(windows)] +#[allow(unsafe_code)] // Win32 FFI +unsafe fn observe_early_exit( + process: windows_sys::Win32::Foundation::HANDLE, + settle: Duration, +) -> Option { + use windows_sys::Win32::Foundation::WAIT_OBJECT_0; + + if unsafe { WaitForSingleObject(process, wait_timeout_millis(settle)) } != WAIT_OBJECT_0 { + return None; + } + let mut exit_code: u32 = 0; + if unsafe { GetExitCodeProcess(process, &raw mut exit_code) } == 0 { + return None; + } + Some(exit_code) +} + +/// Clamp a wait budget to the `u32` milliseconds `WaitForSingleObject` takes, +/// never yielding `INFINITE`. +/// +/// `INFINITE` is `u32::MAX`, so a saturating conversion of a long duration would +/// silently turn a bounded startup check into one that blocks until the child +/// exits. Cap one millisecond below it instead. +#[cfg(any(windows, test))] +fn wait_timeout_millis(budget: Duration) -> u32 { + const LONGEST_FINITE_WAIT_MS: u128 = (u32::MAX - 1) as u128; + budget.as_millis().min(LONGEST_FINITE_WAIT_MS) as u32 } #[cfg(windows)] @@ -1460,6 +1569,86 @@ mod tests { ); } + #[test] + fn wait_timeout_millis_keeps_the_startup_wait_bounded() { + assert_eq!(wait_timeout_millis(Duration::from_millis(200)), 200); + assert_eq!(wait_timeout_millis(Duration::ZERO), 0); + // Sub-millisecond budgets truncate to a poll rather than rounding up. + assert_eq!(wait_timeout_millis(Duration::from_micros(900)), 0); + // A budget past the `u32` millisecond range must never land on + // `INFINITE` (`u32::MAX`), which would block until the child exits. + let huge = wait_timeout_millis(Duration::from_secs(u64::from(u32::MAX))); + assert_eq!(huge, u32::MAX - 1); + assert_ne!(huge, u32::MAX); + } + + /// Watching a detached spawn must surface a child that died during startup. + /// Only meaningful on Windows: the detached spawn primitives return a bare + /// PID there, so this wait is the only chance to notice the exit. + /// + /// It also pins what `DetachedSpawn::early_exit_code` documents: the exit is + /// reported when it happens, not when the window runs out. The window is far + /// longer than `cmd /C exit` takes, so a wait that sat it out fails the + /// elapsed-time check rather than passing slowly. + #[cfg(windows)] + #[test] + fn watched_detached_spawn_reports_a_child_that_exits_immediately() { + let window = Duration::from_secs(30); + let started = std::time::Instant::now(); + // Each token is a separate argument so the assembled command line needs no + // quoting, keeping `cmd.exe`'s quote-stripping rules out of the test. + let spawn = spawn_detached_no_inherit_watching_startup( + &test_system32_tool("cmd.exe"), + &["/C".to_owned(), "exit".to_owned(), "7".to_owned()], + &[], + window, + ) + .expect("spawn should succeed"); + let elapsed = started.elapsed(); + assert_eq!(spawn.early_exit_code, Some(7)); + assert_ne!(spawn.pid, 0); + assert!( + elapsed < window / 2, + "the exit must be reported as soon as the child exits, not at the end of \ + the {window:?} window (took {elapsed:?})" + ); + } + + /// The converse: a child that is still running must not be reported as a + /// startup failure, or every successful launch would be rejected. + #[cfg(windows)] + #[test] + fn watched_detached_spawn_does_not_report_a_child_that_keeps_running() { + // `ping` against localhost is the dependency-free Windows sleep, and ~9s + // outlasts the 200 ms observation window by well over an order of + // magnitude. Spawned directly rather than through `cmd /C` so the + // returned PID is the process the test has to clean up: on Windows + // `terminate_process_tree` is plain `terminate_process`, so an + // intermediate shell would leave `ping` orphaned. + let spawn = spawn_detached_no_inherit_watching_startup( + &test_system32_tool("ping.exe"), + &["-n".to_owned(), "10".to_owned(), "127.0.0.1".to_owned()], + &[], + Duration::from_millis(200), + ) + .expect("spawn should succeed"); + // Clean up before asserting, so a failing assertion cannot leak the child. + let early_exit_code = spawn.early_exit_code; + let was_running = process_is_running(spawn.pid); + let _ = terminate_process(spawn.pid); + assert_eq!(early_exit_code, None); + assert!(was_running); + } + + /// `CreateProcessW` is called with an explicit application name, so it does + /// no `PATH` search — the program has to be a full path. + #[cfg(windows)] + fn test_system32_tool(exe: &str) -> PathBuf { + PathBuf::from(std::env::var_os("SystemRoot").unwrap_or_else(|| r"C:\Windows".into())) + .join("System32") + .join(exe) + } + #[test] fn file_lock_creates_missing_parent_dirs_and_lock_file() { let dir = From b21be9be8bab6cd71da07d47e00bc9c8bd7dfab6 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 10 Nov 2026 11:43:37 +0200 Subject: [PATCH 2/9] fix(serve): never signal a dead engine, and keep a concurrent stop A managed engine that dies at startup was already caught on main, but on Windows the check asked whether its pid was still running 200 ms after the spawn. Windows reuses pids, so a dead engine could be reported as running. The check now reads the exit code through the process handle the spawn returned (spawn_detached_no_inherit_watching_startup), and both platforms report a death through one helper that names the engine's log and its exit status. That death is now a typed error. Cleanup of a failed launch used to terminate the recorded engine pid even when the engine was known to be dead. With no start-time token on Windows, that kill could hit whatever process reused the pid. A dead engine is no longer signalled or put on record; its record and log are kept, marked stopped. rocm services stop and remove do not take the launch lock, yet the launch wrote back the copy of the record it held. A stop settled while the engine spawned was overwritten with "running", the failed-launch cleanup re-created a removed record, and the readiness wait wrote a stale copy over a stop or re-created a removed record. Those writes now go to the record as it is on disk, only while it is still the launch's (update_managed_record_on_disk): - publishing "running" fails the serve if a stop or removal took the claim, and cleanup terminates the unrecorded engine; - the readiness verdict keeps a stop, and the serve then fails with a non-zero exit saying the service was stopped while it was starting, instead of reporting a launch whose summary claimed the server might still be loading. A record settled without a stop marker (the server exited on its own) fails the serve saying it exited and naming its log. A removal fails the serve instead of re-creating the record. The read and write are separate steps, so this narrows the race; it does not close it. The serve spinner is also cleared when the background helper fails to start, and an e2e-test-hooks build can script the engine's startup death through ROCM_E2E_MANAGED_ENGINE_STARTUP_FAILURE. Signed-off-by: Roman Inflianskas --- apps/rocm/src/serve_cmd.rs | 186 +++++-- apps/rocm/src/service_launch.rs | 9 +- .../src/service_launch/launch_transaction.rs | 295 +++++++++-- apps/rocm/src/service_launch/readiness.rs | 156 +++++- apps/rocm/src/service_launch/spawn.rs | 466 ++++++++++++++---- apps/rocm/src/service_launch/test_support.rs | 8 +- docs/architecture.md | 6 +- docs/manual-testing.md | 36 ++ 8 files changed, 991 insertions(+), 171 deletions(-) diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 711e169b6c..e4e3819368 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -79,8 +79,8 @@ use crate::service_launch::{ drop_orphaned_endpoint_key_on_already_running, e2e_simulate_oom_launch, ensure_background_helper_running_quiet, ensure_public_bind_engine_supported, managed_service_reuse_refusal, no_usable_gpu_refusal, print_managed_launch_plain, - resolve_endpoint_auth, run_attached_service, start_managed_service, validate_bind_host, - wait_begin_client_config, + resolve_endpoint_auth, run_attached_service, scripted_managed_engine_startup_failure, + start_managed_service, validate_bind_host, wait_begin_client_config, }; use crate::{ GatedEngineAction, SandboxToolArg, cli_progress, device_policy_name, @@ -499,6 +499,39 @@ pub(crate) struct ServeArgs { pub(crate) engine_env: Vec, } +/// The progress line a background launch paints while it waits. +trait LaunchProgress { + /// Erases the line so whatever prints next starts on a clean one. + fn clear(&mut self); +} + +impl LaunchProgress for cli_progress::Spinner { + fn clear(&mut self) { + Self::clear(self); + } +} + +/// Runs a background serve's managed launch, then starts the background +/// helper, and erases `progress` if either step fails. +/// +/// `Spinner` has no `Drop`, so an error returned past a painted line would +/// print on top of it. Both steps are fallible and run back to back, so they +/// share one clear-on-error here instead of each call site guarding its own. +/// The helper is not started when the launch fails. On success `progress` is +/// left painted: the caller still uses it (the smoke test relabels it) and +/// clears it itself. +fn launch_then_start_helper( + progress: &mut P, + launch: impl FnOnce(&mut P) -> Result, + start_helper: impl FnOnce() -> Result<()>, +) -> Result { + let launched = launch(progress).and_then(|report| start_helper().map(|()| report)); + if launched.is_err() { + progress.clear(); + } + launched +} + pub(crate) fn serve(args: ServeArgs) -> Result<()> { let ServeArgs { model, @@ -877,9 +910,13 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { ] .iter() .any(|variable| std::env::var_os(variable).is_some()); + // The scripted startup-death scenario bypasses the same host precondition, for + // the same reason: it asserts what happens *after* a spawn, so it must reach + // one without real GPU hardware. if !cpu_only && !scripted_backend_failure && !e2e_simulate_oom_launch() + && !scripted_managed_engine_startup_failure() && !reuse_existing && host_has_no_usable_gpu { @@ -996,9 +1033,13 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { device_policy_name(&device_policy) ); } + // Preparation is skipped for the scripted startup-death scenario too: it + // asserts the launch, not the install, and a real runtime download is exactly + // what the ungated lane cannot do. if !matches!(device_policy, DevicePolicy::CpuOnly) && engine_manages_own_runtime(&selected_engine) && resolved_model.is_none() + && !scripted_managed_engine_startup_failure() { ensure_self_managed_engine_ready(&paths, &mut config, &selected_engine)?; } @@ -1183,36 +1224,38 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { }, &mut |_elapsed| spinner.borrow_mut().tick(), ); - // `Spinner` has no `Drop`: clear its row so `Error:` is not glued to it. - let report = match launched { - Ok(report) => report, - Err(error) => { - spinner.borrow_mut().clear(); - return Err(error); - } - }; let mut spinner = spinner.into_inner(); - // Not when the launch was fabricated. `simulate_oom_managed_launch` - // writes a `starting` record for a process that never existed, so the - // helper would be left watching a pid that will never appear, with - // nothing in the scenario's teardown to stop it: observed as `rocm - // daemon` processes outliving their e2e checkout by days, still - // carrying `ROCM_E2E_SIMULATE_OOM_LAUNCH=1`. The helper is for - // supervising a real spawned server, and there is none here. - // - // That — not ordering — is the whole justification. This suppression - // runs *before* the OOM note is computed in the `summary_mode` block - // below, so the previous claim here, that the note "is computed before - // it would have run", was backwards and could not have excused it. - // serve-40's `no background automation daemon was left behind` is what - // holds the branch: deleting it left the scenario green with a `rocm - // daemon` still running. - // - // `e2e_simulate_oom_launch` is a `const fn` returning `false` without - // the test-only feature, so this branch folds away in shipped builds. - if !e2e_simulate_oom_launch() { - ensure_background_helper_running_quiet(summary_mode)?; - } + // `Spinner` has no `Drop`: a failed launch or a failed helper start clears + // its row before the error prints, so `Error:` is not glued to it. + let report = launch_then_start_helper( + &mut spinner, + |_| launched, + // Not when the launch was fabricated. `simulate_oom_managed_launch` + // writes a `starting` record for a process that never existed, so the + // helper would be left watching a pid that will never appear, with + // nothing in the scenario's teardown to stop it: observed as `rocm + // daemon` processes outliving their e2e checkout by days, still + // carrying `ROCM_E2E_SIMULATE_OOM_LAUNCH=1`. The helper is for + // supervising a real spawned server, and there is none here. + // + // That — not ordering — is the whole justification. This suppression + // runs *before* the OOM note is computed in the `summary_mode` block + // below, so the previous claim here, that the note "is computed before + // it would have run", was backwards and could not have excused it. + // serve-40's `no background automation daemon was left behind` is what + // holds the branch: deleting it left the scenario green with a `rocm + // daemon` still running. + // + // `e2e_simulate_oom_launch` is a `const fn` returning `false` without + // the test-only feature, so this branch folds away in shipped builds. + || { + if e2e_simulate_oom_launch() { + Ok(()) + } else { + ensure_background_helper_running_quiet(summary_mode) + } + }, + )?; // An equivalent service was already running, so nothing was spawned and the // freshly generated key is unused — drop it rather than leave it orphaned in @@ -2174,4 +2217,85 @@ mod tests { "non-vLLM engines must not be told to pass a vLLM-only flag: {lemonade:?}" ); } + + /// Records what a background launch did to its progress line. + #[derive(Default)] + struct RecordedProgress { + ticks: usize, + clears: usize, + } + + impl LaunchProgress for RecordedProgress { + fn clear(&mut self) { + self.clears += 1; + } + } + + #[test] + fn a_background_helper_that_fails_to_start_clears_the_launch_progress() { + let mut progress = RecordedProgress::default(); + + let result = launch_then_start_helper( + &mut progress, + |progress| { + progress.ticks += 1; + Ok("report") + }, + || anyhow::bail!("background helper did not start"), + ); + + let error = result.expect_err("the helper's failure is the launch's failure"); + assert_eq!(error.to_string(), "background helper did not start"); + assert_eq!(progress.ticks, 1, "the launch ran before the helper"); + assert_eq!( + progress.clears, 1, + "the painted line is erased before the error prints" + ); + } + + #[test] + fn a_failed_launch_clears_its_progress_and_never_starts_the_helper() { + let mut progress = RecordedProgress::default(); + let mut helper_started = false; + + let result: Result<()> = launch_then_start_helper( + &mut progress, + |_| anyhow::bail!("engine died at startup"), + || { + helper_started = true; + Ok(()) + }, + ); + + let error = result.expect_err("the launch failed"); + assert_eq!(error.to_string(), "engine died at startup"); + assert!( + !helper_started, + "no helper is started for a launch that failed" + ); + assert_eq!(progress.clears, 1); + } + + #[test] + fn a_successful_launch_leaves_its_progress_for_the_caller_to_clear() { + let mut progress = RecordedProgress::default(); + let mut helper_started = false; + + let report = launch_then_start_helper( + &mut progress, + |_| Ok("report"), + || { + helper_started = true; + Ok(()) + }, + ) + .expect("both steps succeed"); + + assert_eq!(report, "report"); + assert!(helper_started); + assert_eq!( + progress.clears, 0, + "the smoke test still relabels the line, so the caller clears it" + ); + } } diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 8653d6a692..9791a5efdc 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -39,8 +39,12 @@ //! //! - `endpoint_auth`: bind-host validation and the endpoint API key checks. //! - `launch_transaction`: the locked port allocation decision, the cleanup of a -//! failed launch, and the reuse refusals. -//! - `spawn`: the detached engine child, its process and stdio plumbing, and +//! failed launch, the reuse refusals, and the guarded read-modify-write +//! (`update_managed_record_on_disk`) every launch write after the claim goes +//! through, so a stop or removal made meanwhile is not overwritten or undone. +//! - `spawn`: the detached engine child, its process and stdio plumbing, the +//! startup check that fails a launch whose engine exits inside the settle +//! window (on Windows through the spawn's process handle), and //! `ManagedLaunchReport`. //! - `readiness`: waiting on a service, its liveness, and the OOM fault-injection //! seam. @@ -69,6 +73,7 @@ pub(crate) use helper_autostart::{ }; pub(crate) use launch_transaction::{managed_service_reuse_refusal, no_usable_gpu_refusal}; pub(crate) use readiness::e2e_simulate_oom_launch; +pub(crate) use spawn::scripted_managed_engine_startup_failure; #[cfg(not(windows))] pub(crate) use spawn::{ attach_background_stdio, detach_background_command, managed_engine_startup_failure_detail, diff --git a/apps/rocm/src/service_launch/launch_transaction.rs b/apps/rocm/src/service_launch/launch_transaction.rs index 13fe90ed1b..95e1738ae0 100644 --- a/apps/rocm/src/service_launch/launch_transaction.rs +++ b/apps/rocm/src/service_launch/launch_transaction.rs @@ -119,39 +119,32 @@ const MANAGED_LAUNCH_CLEANUP_GRACE: Duration = Duration::from_secs(2); /// failed attempt has no use for, is cleared. /// Every step is best-effort and idempotent; cleanup must never mask the /// original failure with a second one. -pub(super) fn abandon_failed_managed_launch(paths: &AppPaths, record: &ManagedServiceRecord) { - if let Some(pid) = record.engine_pid.filter(|pid| *pid != 0) { - // The token captured at spawn, not a fresh `capture`: when the child - // already exited, a fresh one would describe a recycled pid's new owner - // and the forced tree kill would hit it. Where start tokens exist - // (Linux), a missing one means the child was gone before it could be - // captured, so the pid is unverifiable and nothing is signalled - // (`terminate_verified` alone would "best-effort proceed"). Windows - // records no token at all, so there the kill proceeds as before. - let outcome = if record.engine_start_ticks.is_none() && cfg!(target_os = "linux") { - rocm_core::TerminationOutcome::Unverified - } else { - rocm_core::terminate_verified( - &rocm_core::ProcessIdentity::new(pid, record.engine_start_ticks), - rocm_core::KillScope::Tree, - MANAGED_LAUNCH_CLEANUP_GRACE, - true, - ) - }; - record_cli_audit_event( +/// +/// `child_known_dead` is set when the startup check saw the child exit +/// (`EngineDiedAtStartup` in `spawn`). Such a child is not signalled at all: it is gone, +/// and its pid may already belong to an unrelated process — on Windows, which +/// records no start-time token, nothing else would stop the kill from landing +/// on it. Its record and log are still kept, since the error names that log. +/// +/// The `stopped` mark is a guarded write ([`update_managed_record_on_disk`]): +/// a stop that settled the record, a restart that took it over, or a removal +/// is left as it is rather than overwritten or re-created. +pub(super) fn abandon_failed_managed_launch( + paths: &AppPaths, + record: &ManagedServiceRecord, + child_known_dead: bool, +) { + let spawned_pid = record.engine_pid.filter(|pid| *pid != 0); + if child_known_dead || spawned_pid.is_some() { + if let Some(pid) = spawned_pid.filter(|_| !child_known_dead) { + terminate_abandoned_launch_child(paths, record, pid); + } + let _ = update_managed_record_on_disk( paths, - "service", - "managed_service_launch_cleanup", - "warn", - format!( - "terminated engine child pid={pid} after a failed launch publication outcome={}", - outcome.as_str() - ), - Some(&record.service_id), + &record.service_id, + |current| launch_still_owns(current, record), + ManagedServiceRecord::mark_launch_abandoned, ); - let mut abandoned = record.clone(); - abandoned.mark_launch_abandoned(); - let _ = abandoned.write(); endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); return; } @@ -165,6 +158,85 @@ pub(super) fn abandon_failed_managed_launch(paths: &AppPaths, record: &ManagedSe endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); } +/// Terminate the child of an abandoned launch whose state is unknown, and audit +/// the outcome. +fn terminate_abandoned_launch_child(paths: &AppPaths, record: &ManagedServiceRecord, pid: u32) { + // The token captured at spawn, not a fresh `capture`: when the child + // already exited, a fresh one would describe a recycled pid's new owner + // and the forced tree kill would hit it. Where start tokens exist + // (Linux), a missing one means the child was gone before it could be + // captured, so the pid is unverifiable and nothing is signalled + // (`terminate_verified` alone would "best-effort proceed"). Windows + // records no token at all, so there the kill proceeds as before. + let outcome = if record.engine_start_ticks.is_none() && cfg!(target_os = "linux") { + rocm_core::TerminationOutcome::Unverified + } else { + rocm_core::terminate_verified( + &rocm_core::ProcessIdentity::new(pid, record.engine_start_ticks), + rocm_core::KillScope::Tree, + MANAGED_LAUNCH_CLEANUP_GRACE, + true, + ) + }; + record_cli_audit_event( + paths, + "service", + "managed_service_launch_cleanup", + "warn", + format!( + "terminated engine child pid={pid} after a failed launch publication outcome={}", + outcome.as_str() + ), + Some(&record.service_id), + ); +} + +/// Apply one change to a managed service's record as it is on disk now, rather +/// than writing back a copy the caller holds. +/// +/// `ManagedServiceRecord::write` replaces the whole file, and a stop records +/// `stop_requested_unix_ms` and then `"stopped"` without taking the launch lock. +/// A copy read before the stop and written back after it would erase both, +/// bringing back as live a service the stop just reported stopped, and with it +/// the marker that keeps `rocmd`'s recovery off it. +/// +/// Returns `Ok(None)`, writing nothing, when the record is gone or +/// `still_owned` refuses the record as it is now: whoever wrote since holds a +/// truer view. This narrows the lost-update window; it does not close it, since +/// the read and the write are separate steps and a stop takes no lock on the +/// manifest. +pub(super) fn update_managed_record_on_disk( + paths: &AppPaths, + service_id: &str, + still_owned: impl FnOnce(&ManagedServiceRecord) -> bool, + update: impl FnOnce(&mut ManagedServiceRecord), +) -> Result> { + if !paths.service_manifest_path(service_id).exists() { + return Ok(None); + } + let mut current = ManagedServiceRecord::read_manifest(paths, service_id)?; + if !still_owned(¤t) { + return Ok(None); + } + update(&mut current); + current.write()?; + Ok(Some(current)) +} + +/// Whether the launch that wrote `launched` may still write `current`, the same +/// service's record as it is on disk now: it still reads as live, and names the +/// supervisor the launch last recorded — 0 for the claim, before any child is +/// on record. A confirmed stop writes `"stopped"`; a restart records its own +/// supervisor. Either way the record is no longer the launch's to write. +pub(super) fn launch_still_owns( + current: &ManagedServiceRecord, + launched: &ManagedServiceRecord, +) -> bool { + crate::managed_service_status::managed_service_is_live(current) + && current.supervisor_pid == launched.supervisor_pid + && current.supervisor_start_ticks == launched.supervisor_start_ticks +} + /// The ways a live managed service that already matches on liveness, engine and /// canonical model id can still be unusable by this invocation. /// @@ -298,7 +370,6 @@ pub(crate) fn managed_service_reuse_refusal( mod tests { use super::*; use crate::service_hints; - #[cfg(not(windows))] use crate::service_launch::test_support::sleeper_test_name; use crate::service_launch::test_support::{reusable_record, write_live_service_record}; use crate::tests::test_paths; @@ -884,7 +955,7 @@ mod tests { // Simulate a pre-spawn failure: nothing ever ran. record.engine_pid = None; - abandon_failed_managed_launch(&paths, &record); + abandon_failed_managed_launch(&paths, &record, false); assert!(!record.manifest_path.exists()); assert!(!record.log_path.exists()); @@ -925,13 +996,13 @@ mod tests { record.record_engine_identity(bystander.id()); record.engine_start_ticks = record.engine_start_ticks.map(|ticks| ticks + 1); - abandon_failed_managed_launch(&paths, &record); + abandon_failed_managed_launch(&paths, &record, false); let mismatched_left_alone = bystander.try_wait().expect("poll bystander").is_none(); // A child that was gone before its token could be captured leaves none: // that must not fall back to "best-effort" signalling either. record.engine_start_ticks = None; - abandon_failed_managed_launch(&paths, &record); + abandon_failed_managed_launch(&paths, &record, false); let still_running = mismatched_left_alone && bystander.try_wait().expect("poll bystander").is_none(); let _ = bystander.kill(); @@ -969,7 +1040,7 @@ mod tests { let child_pid = child.id(); record.record_engine_identity(child_pid); - abandon_failed_managed_launch(&paths, &record); + abandon_failed_managed_launch(&paths, &record, false); let wait_result = child.wait(); @@ -999,4 +1070,156 @@ mod tests { ); Ok(()) } + + /// A sleeper the test owns, standing in for an unrelated process that now + /// holds a pid the launch once recorded. Spawned with `std`'s `Command` + /// directly, so the test runs on Windows too, where this matters most: no + /// start-time token guards a kill there. + fn spawn_bystander() -> std::process::Child { + std::process::Command::new(std::env::current_exe().expect("current test exe")) + .arg("--exact") + .arg(sleeper_test_name()) + .arg("--ignored") + .arg("--test-threads=1") + .env("ROCM_TEST_SLEEPER_CHILD", "1") + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .expect("spawn bystander") + } + + #[test] + fn an_engine_seen_dead_at_startup_is_never_signalled() -> Result<()> { + // The startup check saw the engine exit, so its pid is free. Here a + // bystander holds it. Cleanup must not signal it — not even through a + // start-time token, which Windows does not record at all — and must + // still keep the record and log of the failed launch for the error + // that names them. + let (root, paths) = test_paths("abandon-known-dead-engine"); + paths.ensure()?; + let mut record = write_live_service_record( + &paths, + "lemonade-abandon-known-dead", + "abandon-known-dead-model", + 11_495, + "starting", + )?; + fs::write(&record.log_path, "engine: bad flag\n")?; + let mut bystander = spawn_bystander(); + record.record_engine_identity(bystander.id()); + + abandon_failed_managed_launch(&paths, &record, true); + + let left_alone = bystander.try_wait().expect("poll bystander").is_none(); + let _ = bystander.kill(); + let _ = bystander.wait(); + let persisted = ManagedServiceRecord::read_manifest(&paths, &record.service_id); + let log_kept = record.log_path.exists(); + let _ = fs::remove_dir_all(root); + assert!( + left_alone, + "a pid the dead engine freed must not be signalled" + ); + assert_eq!(persisted?.status, "stopped"); + assert!(log_kept, "the log the error names must be kept"); + Ok(()) + } + + #[test] + fn abandoning_a_launch_does_not_recreate_a_removed_record() -> Result<()> { + let (root, paths) = test_paths("abandon-removed-record"); + paths.ensure()?; + let record = write_live_service_record( + &paths, + "lemonade-abandon-removed", + "abandon-removed-model", + 11_496, + "starting", + )?; + fs::remove_file(&record.manifest_path)?; + + abandon_failed_managed_launch(&paths, &record, true); + + let recreated = record.manifest_path.exists(); + let _ = fs::remove_dir_all(root); + assert!(!recreated, "cleanup must not bring back a removed record"); + Ok(()) + } + + #[test] + fn the_launch_does_not_publish_over_a_record_a_restart_took_over() -> Result<()> { + // A restart records its own supervisor on a record that still reads as + // live. The launch's claim names supervisor 0, so it is no longer the + // launch's to write. + let (root, paths) = test_paths("launch-owns-supervisor"); + paths.ensure()?; + let launched = write_live_service_record( + &paths, + "lemonade-owns-supervisor", + "owns-supervisor-model", + 11_497, + "starting", + )?; + let mut taken = ManagedServiceRecord::read_manifest(&paths, &launched.service_id)?; + taken.supervisor_pid = launched.supervisor_pid.wrapping_add(7); + taken.write()?; + assert!(crate::managed_service_status::managed_service_is_live( + &taken + )); + + assert!(!launch_still_owns(&taken, &launched)); + let written = update_managed_record_on_disk( + &paths, + &launched.service_id, + |current| launch_still_owns(current, &launched), + |current| current.status = "running".to_owned(), + )?; + let after = ManagedServiceRecord::read_manifest(&paths, &launched.service_id)?; + let _ = fs::remove_dir_all(root); + assert!( + written.is_none(), + "the launch wrote over a restart's record" + ); + assert_eq!(after.status, "starting"); + assert_eq!(after.supervisor_pid, taken.supervisor_pid); + // Same supervisor pid but a different start token: a restart whose new + // supervisor reused the pid. Not the launch's either. + let mut reused = launched.clone(); + reused.supervisor_start_ticks = Some(launched.supervisor_start_ticks.unwrap_or(0) + 1); + assert!(!launch_still_owns(&reused, &launched)); + // Same supervisor, same start token, still live: the launch owns it. + assert!(launch_still_owns(&launched, &launched)); + Ok(()) + } + + #[test] + fn an_engine_seen_dead_before_it_was_recorded_keeps_its_record_and_log() -> Result<()> { + // The Windows shape: the startup check saw the engine exit before + // `record_engine_identity` ran, so no engine pid is on record. Nothing + // is spawned-and-recorded, yet the failure names the log, so the record + // and log must survive, marked `stopped`. + let (root, paths) = test_paths("abandon-dead-engine-never-recorded"); + paths.ensure()?; + let mut record = write_live_service_record( + &paths, + "lemonade-abandon-never-recorded", + "abandon-never-recorded-model", + 11_498, + "starting", + )?; + record.engine_pid = None; + record.supervisor_pid = 0; + fs::write(&record.log_path, "engine: bad flag\n")?; + record.write()?; + + abandon_failed_managed_launch(&paths, &record, true); + + let persisted = ManagedServiceRecord::read_manifest(&paths, &record.service_id); + let log_kept = record.log_path.exists(); + let _ = fs::remove_dir_all(root); + assert_eq!(persisted?.status, "stopped"); + assert!(log_kept, "the log the error names must be kept"); + Ok(()) + } } diff --git a/apps/rocm/src/service_launch/readiness.rs b/apps/rocm/src/service_launch/readiness.rs index 1e14c85599..82dd7bced0 100644 --- a/apps/rocm/src/service_launch/readiness.rs +++ b/apps/rocm/src/service_launch/readiness.rs @@ -13,6 +13,7 @@ // would be unused in some build, and a gate on it would have to be kept in step // with every use site. Keep the full paths. The tests module imports `fs` for // itself because its tests use it on every platform. +use super::launch_transaction::update_managed_record_on_disk; use super::spawn::ManagedLaunchReport; #[cfg(feature = "e2e-oom-fault-injection")] use crate::device_policy_name; @@ -21,7 +22,7 @@ use crate::engine_runtime_select::wait_for_service_http_ready_with_progress; use crate::status_for_readiness; #[cfg(feature = "e2e-oom-fault-injection")] use anyhow::Context; -use anyhow::{Result, anyhow}; +use anyhow::{Result, anyhow, bail}; #[cfg(feature = "e2e-oom-fault-injection")] use rocm_core::format_http_base_url; use rocm_core::{AppPaths, EndpointReadiness, ManagedServiceRecord}; @@ -153,22 +154,68 @@ pub(super) fn died_during_wait( /// only what the wait learned. The copy read at spawn time can be long stale: /// `rocmd` may have recorded a restart or a new supervisor pid, and `rocm logs` /// refreshes the record, so writing it back whole would revert those. +/// +/// The verdict is written only while the record still reads as live. A +/// `rocm services stop` that settled it during the wait keeps its `stopped` +/// and the wait fails saying so: reporting a readiness verdict (and exiting 0) +/// for a server somebody stopped would be false. A record settled without a +/// stop marker means the server exited on its own (without `--wait`, +/// `died_during_wait` does not run, so this is where that is noticed), and the +/// failure points at its log instead. A record removed during the +/// wait (`rocm services remove`) is not re-created from the stale copy; that +/// fails too, since there is no service left to report. pub(super) fn record_wait_outcome( paths: &AppPaths, held: &mut ManagedServiceRecord, status: &str, verified: bool, ) -> Result<()> { - let mut latest = ManagedServiceRecord::read_manifest(paths, &held.service_id) - .unwrap_or_else(|_| held.clone()); - status.clone_into(&mut latest.status); - if verified { - // Latch the verification the wait just performed, so the readiness checks - // behind `services list` and chat read it instead of re-probing. - latest.inference_verified_at_unix_ms = Some(rocm_core::unix_time_millis() as u64); - } - latest.write()?; - *held = latest; + let written = update_managed_record_on_disk( + paths, + &held.service_id, + crate::managed_service_status::managed_service_is_live, + |latest| { + status.clone_into(&mut latest.status); + if verified { + // Latch the verification the wait just performed, so the readiness + // checks behind `services list` and chat read it instead of + // re-probing. + latest.inference_verified_at_unix_ms = Some(rocm_core::unix_time_millis() as u64); + } + }, + )?; + *held = match written { + Some(latest) => latest, + None if paths.service_manifest_path(&held.service_id).exists() => { + let settled = ManagedServiceRecord::read_manifest(paths, &held.service_id)?; + *held = settled; + // A stop leaves its marker; anything else that settled the record + // (the liveness refresh demoting a dead engine, or the engine's own + // `failed`) means the server exited on its own, and the user needs + // its log rather than to hear somebody stopped it. + if held.stop_requested_unix_ms.is_some() { + bail!( + "managed service `{}` was stopped while it was starting (its record now \ + reads `{}`), so no readiness is reported for it; start it again with \ + `rocm serve`", + held.service_id, + held.status + ); + } + bail!( + "managed service `{}` exited while it was starting (its record now reads `{}`); \ + see `rocm logs --service {}` for its output", + held.service_id, + held.status, + held.service_id + ); + } + None => bail!( + "managed service `{}` was removed while `rocm serve` waited for it, so there is \ + no record of it left to report", + held.service_id + ), + }; Ok(()) } @@ -631,4 +678,91 @@ mod tests { assert_eq!(after.status, "starting"); let _ = fs::remove_dir_all(root); } + + #[test] + fn record_wait_outcome_does_not_recreate_a_record_removed_during_the_wait() { + let (root, paths) = test_paths("wait-outcome-removed"); + paths.ensure().expect("paths"); + let mut held = record_tracking_pid(&paths, 111, "starting"); + held.write().expect("write"); + // `rocm services remove` ran while `rocm serve` waited. + fs::remove_file(&held.manifest_path).expect("remove manifest"); + + let error = record_wait_outcome(&paths, &mut held, "ready", true) + .expect_err("a removed service has nothing left to report"); + let recreated = held.manifest_path.exists(); + let _ = fs::remove_dir_all(root); + assert!( + error.to_string().contains("was removed while"), + "the failure must say what happened: {error:#}" + ); + assert!(!recreated, "the stale copy must not re-create the record"); + } + + #[test] + fn an_engine_that_exited_during_the_wait_is_reported_as_exited_not_stopped() { + // Without `--wait`, `died_during_wait` does not run, so a record the + // liveness refresh demoted (or the engine marked `failed`) reaches the + // verdict write. No stop marker: nobody stopped it, it exited. + for settled_status in ["stopped", "failed"] { + let (root, paths) = test_paths(&format!("wait-outcome-exited-{settled_status}")); + paths.ensure().expect("paths"); + let mut held = record_tracking_pid(&paths, 111, "starting"); + held.write().expect("write"); + let mut exited = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); + exited.status = settled_status.to_owned(); + exited.write().expect("write"); + + let error = record_wait_outcome(&paths, &mut held, "starting", false) + .expect_err("a server that exited must not be reported as launched"); + let after = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); + let _ = fs::remove_dir_all(root); + let message = error.to_string(); + assert!( + message.contains("exited while it was starting"), + "{message}" + ); + assert!(!message.contains("was stopped"), "{message}"); + assert!( + message.contains(&format!("rocm logs --service {}", held.service_id)), + "{message}" + ); + assert_eq!( + after.status, settled_status, + "the settled record was overwritten" + ); + assert_eq!(after.stop_requested_unix_ms, None); + } + } + + #[test] + fn a_stop_settled_during_the_wait_fails_the_serve_and_keeps_the_stop() { + let (root, paths) = test_paths("wait-outcome-stopped"); + paths.ensure().expect("paths"); + let mut held = record_tracking_pid(&paths, 111, "starting"); + held.write().expect("write"); + // `rocm services stop` settled the record while `rocm serve` waited. + let mut stopped = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); + stopped.stop_requested_unix_ms = Some(1_234); + stopped.status = "stopped".to_owned(); + stopped.write().expect("write"); + + // An `Err` here is what `rocm serve` turns into a non-zero exit: the + // launch path propagates it with `?` and prints nothing else. + let error = record_wait_outcome(&paths, &mut held, "ready", true) + .expect_err("a server stopped during its wait must not be reported as launched"); + let after = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); + let _ = fs::remove_dir_all(root); + let message = error.to_string(); + assert!( + message.contains("was stopped while it was starting"), + "{message}" + ); + assert!(message.contains(&held.service_id), "{message}"); + // The message and the state it describes, together: the record really + // does read stopped, with its marker, and carries no readiness verdict. + assert_eq!(after.status, "stopped", "the stop was overwritten"); + assert_eq!(after.stop_requested_unix_ms, Some(1_234)); + assert_eq!(after.inference_verified_at_unix_ms, None); + } } diff --git a/apps/rocm/src/service_launch/spawn.rs b/apps/rocm/src/service_launch/spawn.rs index 0bc3d02a34..d73e28cfbf 100644 --- a/apps/rocm/src/service_launch/spawn.rs +++ b/apps/rocm/src/service_launch/spawn.rs @@ -8,8 +8,9 @@ use super::endpoint_auth::ensure_public_service_has_endpoint_key; use super::launch_transaction::{ - ManagedPortDecision, ManagedReuseRefusal, abandon_failed_managed_launch, + ManagedPortDecision, ManagedReuseRefusal, abandon_failed_managed_launch, launch_still_owns, managed_service_reuse_refusal, no_usable_gpu_refusal, resolve_managed_port_in_transaction, + update_managed_record_on_disk, }; use crate::endpoint_keys; use crate::engine_ipc::builtin_engine_serve_http_args; @@ -24,14 +25,13 @@ use crate::service_hints; use crate::services_text::read_optional_tail_lines; use crate::{device_policy_name, record_cli_audit_event}; use anyhow::{Context, Result, bail}; -#[cfg(windows)] -use rocm_core::process_is_running; use rocm_core::{AppPaths, ManagedServiceRecord}; use rocm_engine_protocol::{DevicePolicy, EngineRecipeHint, ResolveModelResponse}; use std::fs; #[cfg(not(windows))] use std::io; use std::path::{Path, PathBuf}; +use std::process::ExitStatus; #[cfg(not(windows))] use std::process::{Command as ProcessCommand, Stdio}; use std::thread; @@ -62,32 +62,127 @@ pub(crate) fn attach_background_stdio( Ok(()) } -/// Windows spawn-liveness check: wait out the startup window, then fail with the -/// startup log when the child is already gone. The wait and the liveness probe -/// are injected so a unit test runs the exit-detected path on every platform, -/// including the Windows lane that actually calls it. +/// How long a freshly spawned managed engine is watched before it is treated as +/// started. Long enough to catch an engine that dies on the way up — arguments it +/// rejects, a runtime it cannot load — and short enough to be invisible next to +/// the readiness wait that follows. Both platforms use the same budget. /// -/// # Errors -/// Fails with the startup-failure detail when `is_running` reports the child dead. -#[cfg_attr(not(windows), allow(dead_code))] -fn ensure_spawned_child_survives_startup( - child_pid: u32, - log_path: &Path, - settle: impl FnOnce(), - is_running: impl FnOnce(u32) -> bool, -) -> Result<()> { - settle(); - if is_running(child_pid) { - Ok(()) - } else { - bail!("{}", managed_engine_startup_failure_detail("", log_path)) +/// A window, not a guarantee: engines bind their port and load the model later +/// than this, so a death there passes the check and the launch is reported as +/// still starting. +pub(crate) const MANAGED_ENGINE_STARTUP_SETTLE: Duration = Duration::from_millis(200); + +/// The error a launch fails with when its engine was seen dead inside +/// [`MANAGED_ENGINE_STARTUP_SETTLE`]. +/// +/// A type of its own so the launch's cleanup can tell a child *known* to be dead +/// from one whose state is unknown: a known-dead child is never signalled, since +/// its pid is free and may already name an unrelated process. +#[derive(Debug)] +struct EngineDiedAtStartup(String); + +impl std::fmt::Display for EngineDiedAtStartup { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) } } +impl std::error::Error for EngineDiedAtStartup {} + +/// Fail the launch when the startup check saw the engine exit, naming the +/// engine's own log and carrying a tail of it. +/// +/// Both platforms report through here — Unix from `Child::try_wait`, Windows +/// from the exit code the watched detached spawn read through the process +/// handle `CreateProcessW` returned — so a death at startup reads the same on +/// each, exit status included. `None` means the engine was still running, or +/// (on Windows) that its state could not be read, which is treated as running. +fn fail_if_engine_died_at_startup(startup_exit: Option, log_path: &Path) -> Result<()> { + let Some(status) = startup_exit else { + return Ok(()); + }; + Err(EngineDiedAtStartup(managed_engine_startup_failure_detail( + &format!(" with status {status}"), + log_path, + )) + .into()) +} + +/// Put a watched spawn's engine on `record`, unless the watch saw it exit. +/// +/// The order is the point: the startup check runs *before* the pid is recorded, +/// so an engine seen dead is never named by the record. Its pid is free, and on +/// Windows, which records no start-time token, nothing else would stop a later +/// cleanup or stop from signalling whatever process reuses it. A live engine is +/// recorded before anything else can fail, so cleanup knows to terminate it. +/// +/// Called on Windows, where the spawn reports the exit itself; kept free of any +/// platform API so its unit test runs on every lane, the Windows one included. +#[cfg_attr(not(windows), allow(dead_code))] +fn adopt_watched_engine_spawn( + record: &mut ManagedServiceRecord, + pid: u32, + startup_exit: Option, +) -> Result { + fail_if_engine_died_at_startup(startup_exit, &record.log_path)?; + record.record_engine_identity(pid); + Ok(pid) +} + +/// Whether `error` is a launch failure for an engine seen dead at startup. +fn engine_died_at_startup(error: &anyhow::Error) -> bool { + error.downcast_ref::().is_some() +} + +/// Read a watched detached spawn's early exit as an `ExitStatus`. +/// +/// `GetExitCodeProcess` yields a bare `u32`, which is what +/// `ExitStatusExt::from_raw` takes on Windows. +#[cfg(windows)] +fn managed_startup_exit(spawn: &rocm_core::DetachedSpawn) -> Option { + use std::os::windows::process::ExitStatusExt as _; + + spawn.early_exit_code.map(ExitStatus::from_raw) +} + +/// Whether the E2E-only switch that makes the managed engine die during startup +/// is armed, so the black-box suite can assert what a user sees when it does. +/// +/// Read in the parent, which spawns the engine, so nothing test-only reaches +/// the child's own code path. The child is the real `rocm` binary, really +/// spawned and really dead — only its arguments are swapped for ones the CLI +/// rejects, so the launch takes the path a broken engine takes. +/// +/// Armed by [`rocm_core::MANAGED_ENGINE_STARTUP_FAILURE_TEST_ENV`], which the +/// E2E suite names too. +#[cfg(feature = "e2e-test-hooks")] +pub(crate) fn scripted_managed_engine_startup_failure() -> bool { + std::env::var_os(rocm_core::MANAGED_ENGINE_STARTUP_FAILURE_TEST_ENV).is_some() +} + +/// Release builds do not enable `e2e-test-hooks`, so the switch does not exist +/// there and no environment variable can arm it. +#[cfg(not(feature = "e2e-test-hooks"))] +pub(crate) const fn scripted_managed_engine_startup_failure() -> bool { + false +} + +/// Arguments that make the spawned `rocm` exit at once with a non-zero status. +/// Used only under [`scripted_managed_engine_startup_failure`]. +/// +/// An unrecognised *flag*, deliberately: clap rejects it with exit code 2 before +/// any work. An unrecognised subcommand would not do — the CLI falls back to +/// natural-language request parsing for those and exits 0. +#[cfg(feature = "e2e-test-hooks")] +fn managed_engine_startup_failure_args() -> Vec { + vec!["--e2e-managed-engine-startup-failure".to_owned()] +} + /// `status_clause` is a pre-formatted fragment inserted right after "exited -/// immediately" (e.g. `" with status exit status: 1"` on Unix, where a real -/// `ExitStatus` is available; empty on Windows, where the liveness check -/// below only has a yes/no answer, not an exit code). +/// immediately", e.g. `" with status exit status: 1"`. The launch passes one on +/// both platforms: the exit status comes from `Child::try_wait` on Unix and from +/// the exit code the watched Windows spawn read through the process handle. It +/// may be empty for a caller that has no status to name. pub(crate) fn managed_engine_startup_failure_detail( status_clause: &str, log_path: &Path, @@ -374,7 +469,8 @@ pub(super) fn spawn_managed_engine_child( // record's first write: `record.engine_pid` tells it whether a child // ever ran, so a pre-spawn failure deletes everything and a post-spawn // one keeps the record and log (marked `status = "stopped"`) instead. - abandon_failed_managed_launch(paths, &record); + // An engine seen dead at startup is not signalled: its pid is free. + abandon_failed_managed_launch(paths, &record, engine_died_at_startup(&error)); Err(error) } } @@ -446,6 +542,14 @@ fn launch_managed_engine_child_locked( &record.engine_state_path, Some(&record.log_path), )?; + // E2E-only: swap in arguments the child rejects at once, so the launch below + // observes a real engine that really died during startup. + #[cfg(feature = "e2e-test-hooks")] + let serve_args = if scripted_managed_engine_startup_failure() { + managed_engine_startup_failure_args() + } else { + serve_args + }; let engine_envs_root = env_root_for_service(paths, engine, runtime_id, env_id)?; // Hand the child the *path* to the endpoint key file (public bind only) via the // environment. A path — not the secret value — is what the detached-spawn @@ -488,21 +592,17 @@ fn launch_managed_engine_child_locked( if let Some(key_file) = endpoint_key_file.as_deref() { env_refs.push((rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file)); } - let child_pid = rocm_core::spawn_detached_no_inherit(¤t_exe, &serve_args, &env_refs) - .context("failed to launch managed engine process")?; - // Set before the immediate-exit check below, so a bail! here still leaves - // `record.engine_pid` telling cleanup that a child was spawned. The - // start-time token is captured here too, while the child is certainly - // ours: cleanup binds its kill to it rather than to whatever owns the - // pid by then. - record.record_engine_identity(child_pid); - ensure_spawned_child_survives_startup( - child_pid, - &record.log_path, - || thread::sleep(Duration::from_millis(200)), - process_is_running, - )?; - child_pid + // Watched through the process handle `CreateProcessW` returned, before + // it is closed: re-checking the pid afterwards could find a different + // process that reused it, and report a dead engine as running. + let spawn = rocm_core::spawn_detached_no_inherit_watching_startup( + ¤t_exe, + &serve_args, + &env_refs, + MANAGED_ENGINE_STARTUP_SETTLE, + ) + .context("failed to launch managed engine process")?; + adopt_watched_engine_spawn(record, spawn.pid, managed_startup_exit(&spawn))? }; #[cfg(not(windows))] let child_pid = { @@ -533,19 +633,11 @@ fn launch_managed_engine_child_locked( // ours: cleanup binds its kill to it rather than to whatever owns the // pid by then. record.record_engine_identity(child_pid); - thread::sleep(Duration::from_millis(200)); - if let Some(status) = child + thread::sleep(MANAGED_ENGINE_STARTUP_SETTLE); + let startup_exit = child .try_wait() - .context("failed to check managed engine startup state")? - { - bail!( - "{}", - managed_engine_startup_failure_detail( - &format!(" with status {status}"), - &record.log_path - ) - ); - } + .context("failed to check managed engine startup state")?; + fail_if_engine_died_at_startup(startup_exit, &record.log_path)?; child_pid }; // The child is both roles until the engine state file names a separate @@ -555,15 +647,36 @@ fn launch_managed_engine_child_locked( // engine PID recorded without one could only be signalled blind. // The engine's token was already captured right after the spawn (so cleanup // can bind to it); re-capturing here could replace it with a later owner's. - record.record_supervisor_identity(child_pid); - record.status = "running".to_owned(); - if let Err(error) = record.write() { - // Cleanup is the caller's job now: `record.engine_pid` (set above) is - // already enough for `abandon_failed_managed_launch` to terminate this - // child, so there is nothing left for this inner path to do beyond - // surfacing the error. - return Err(error.context("failed to publish the managed service record")); - } + // + // Applied to the record as it is on disk, and only while it is still the + // claim this launch wrote: a `rocm services stop` or `remove` does not take + // the launch lock, so it can settle the claim while the child is spawned. + // Writing the held copy back would revive that service as `running`. + // + // On any error here cleanup is the caller's job: `record.engine_pid` (set + // above) is enough for `abandon_failed_managed_launch` to terminate this + // child, which nothing on record names when the claim was taken away. + let (engine_pid, engine_start_ticks) = (record.engine_pid, record.engine_start_ticks); + let published = update_managed_record_on_disk( + paths, + service_id, + |current| launch_still_owns(current, record), + |current| { + current.engine_pid = engine_pid; + current.engine_start_ticks = engine_start_ticks; + current.record_supervisor_identity(child_pid); + current.status = "running".to_owned(); + }, + ) + .context("failed to publish the managed service record")?; + let Some(published) = published else { + bail!( + "managed service `{service_id}` was stopped or removed while it was launching, so \ + the engine it started (pid {child_pid}) is not recorded on it; the failed launch \ + terminates it" + ); + }; + *record = published; Ok(child_pid) } @@ -579,26 +692,41 @@ mod tests { use crate::tests::test_paths; use rocm_core::DEFAULT_LOCAL_HOST; + /// An `ExitStatus` for `code`, built the way each platform's startup + /// check gets one: `try_wait` on Unix, `GetExitCodeProcess` on Windows. + fn exit_status(code: u8) -> ExitStatus { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt as _; + ExitStatus::from_raw(i32::from(code) << 8) + } + #[cfg(windows)] + { + use std::os::windows::process::ExitStatusExt as _; + ExitStatus::from_raw(u32::from(code)) + } + } + #[test] fn a_child_that_exited_during_the_startup_window_is_reported_with_its_log() -> Result<()> { let dir = tempfile::tempdir()?; let log = dir.path().join("engine.log"); fs::write(&log, "boom: engine could not start\n")?; - let settled = std::cell::Cell::new(false); - let error = ensure_spawned_child_survives_startup( - 4242, - &log, - || settled.set(true), - |pid| { - assert_eq!(pid, 4242); - assert!(settled.get(), "liveness must be probed after the wait"); - false - }, - ) - .expect_err("a dead child must fail the launch"); - assert!(settled.get(), "the startup window must elapse first"); - assert!(format!("{error:#}").contains("boom"), "{error:#}"); - ensure_spawned_child_survives_startup(4242, &log, || {}, |_| true)?; + let error = fail_if_engine_died_at_startup(Some(exit_status(2)), &log) + .expect_err("a dead child must fail the launch"); + let message = format!("{error:#}"); + assert!(message.contains("boom"), "{message}"); + assert!(message.contains(&log.display().to_string()), "{message}"); + // The exit status is named on both platforms, not only where a `Child` + // is at hand. + let status_clause = format!(" with status {}", exit_status(2)); + assert!(message.contains(&status_clause), "{message}"); + // Typed, so cleanup knows not to signal a pid that is already free. + assert!(engine_died_at_startup(&error)); + fail_if_engine_died_at_startup(None, &log)?; + assert!(!engine_died_at_startup(&anyhow::anyhow!( + "some other launch failure" + ))); Ok(()) } @@ -984,19 +1112,20 @@ mod tests { } #[test] - fn a_failed_spawn_past_the_first_write_terminates_the_child_and_keeps_an_abandoned_record() + fn an_engine_that_dies_at_startup_keeps_an_abandoned_record_and_is_never_signalled() -> Result<()> { // Drives the real call site's *post*-spawn cleanup branch, which // neither `abandoned_launch_*` test reaches (they call // `abandon_failed_managed_launch` directly). An engine name this // binary does not build in makes the real spawned child (a second // `rocm` process) exit almost immediately via - // `run_builtin_engine_serve_http`'s `other => bail!(...)` arm, after - // `launch_managed_engine_child_locked` has already set - // `record.engine_pid` — so cleanup must terminate (already-dead) the - // child, keep the record and log for inspection, mark the record - // `stopped` (a `failed` one would be respawned by rocmd recovery), and - // clear only the endpoint key. + // `run_builtin_engine_serve_http`'s `other => bail!(...)` arm. The + // startup check sees it exit (`try_wait` on Unix, the watched spawn's + // exit code on Windows), so cleanup must keep the record and log for + // inspection, mark the record `stopped` (a `failed` one would be + // respawned by rocmd recovery), clear only the endpoint key, and never + // signal the engine: its pid is free. This drives the real mapping from + // the startup check's error to that cleanup, on every platform. // // The child always dies near-instantly, but `spawn_managed_engine_child` // only reports that within a fixed startup-check window. On a loaded or @@ -1099,7 +1228,53 @@ mod tests { live.is_empty(), "an abandoned record must not still reserve its port" ); + assert_eq!( + persisted.engine_pid, None, + "the record must not name an engine seen dead: its pid is free" + ); + let audit = fs::read_to_string(paths.audit_events_path()).unwrap_or_default(); let _ = fs::remove_dir_all(root); + assert!( + !audit.contains("managed_service_launch_cleanup"), + "cleanup must not signal an engine the startup check saw die:\n{audit}" + ); + Ok(()) + } + + #[test] + fn a_watched_spawn_seen_dead_is_never_put_on_record() -> Result<()> { + // The Windows ordering, on every lane: the startup check runs before + // the pid is recorded, so an engine seen dead is never named. + let dir = tempfile::tempdir()?; + let (_root, paths) = test_paths("adopt-watched-spawn"); + let mut record = write_live_service_record( + &paths, + "lemonade-adopt-watched", + "adopt-watched-model", + 11_499, + "starting", + )?; + record.engine_pid = None; + record.log_path = dir.path().join("engine.log"); + fs::write(&record.log_path, "engine: bad flag\n")?; + + let error = adopt_watched_engine_spawn(&mut record, 4242, Some(exit_status(2))) + .expect_err("an engine seen dead must fail the launch"); + assert!(engine_died_at_startup(&error), "{error:#}"); + assert_eq!( + record.engine_pid, None, + "a dead engine's pid was put on record" + ); + + assert_eq!( + adopt_watched_engine_spawn(&mut record, std::process::id(), None)?, + std::process::id() + ); + assert_eq!( + record.engine_pid, + Some(std::process::id()), + "a live engine must be on record before anything else can fail" + ); Ok(()) } @@ -1303,4 +1478,129 @@ mod tests { result?; Ok(()) } + + #[test] + #[cfg(not(windows))] + fn a_stop_settled_while_the_engine_spawns_is_kept_and_the_engine_terminated() -> Result<()> { + let (root, paths) = test_paths("launch-stopped-during-spawn"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + let port = std::net::TcpListener::bind(("127.0.0.1", 0))? + .local_addr()? + .port(); + let lease = rocm_core::lease_loopback_port(port)?; + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-stopped-during-spawn", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + port, + "managed", + 0, + None, + None, + None, + ); + record.write()?; + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + let manifest_paths = paths.clone(); + let service_id = record.service_id.clone(); + let spawn_override: &dyn Fn(&mut ProcessCommand) -> io::Result = + &|_command| { + // `rocm services stop` settles the claim while the child is + // being spawned; it takes no launch lock. + let mut stopped = ManagedServiceRecord::read_manifest(&manifest_paths, &service_id) + .expect("claim on disk"); + stopped.stop_requested_unix_ms = Some(4_242); + stopped.status = "stopped".to_owned(); + stopped.write().expect("write stop"); + ProcessCommand::new(std::env::current_exe().expect("test binary")) + .arg(sleeper_test_name()) + .arg("--exact") + .arg("--ignored") + .env("ROCM_TEST_SLEEPER_CHILD", "1") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + }; + + let result = launch_managed_engine_child_locked( + &paths, + &mut record, + lease, + "lemonade", + "lemonade-stopped-during-spawn", + &resolve, + "127.0.0.1", + &resolve.device_policy, + &[], + None, + None, + None, + Some(spawn_override), + ); + let child_pid = record.engine_pid.expect("the child is known to cleanup"); + let error = match result { + Ok(pid) => { + let _ = rocm_core::terminate_process(pid); + let _ = fs::remove_dir_all(&root); + panic!("a launch whose record was stopped must not report success"); + } + Err(error) => error, + }; + // What `spawn_managed_engine_child` does with that error. + abandon_failed_managed_launch(&paths, &record, engine_died_at_startup(&error)); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while super::super::readiness::service_pid_is_alive(child_pid) + && std::time::Instant::now() < deadline + { + thread::sleep(Duration::from_millis(50)); + } + let child_alive = super::super::readiness::service_pid_is_alive(child_pid); + if child_alive { + let _ = rocm_core::terminate_process(child_pid); + } + let after = ManagedServiceRecord::read_manifest(&paths, &record.service_id); + let _ = fs::remove_dir_all(root); + let after = after?; + assert!( + error + .to_string() + .contains("was stopped or removed while it was launching"), + "{error:#}" + ); + assert_eq!(after.status, "stopped", "the stop was overwritten"); + assert_eq!( + after.stop_requested_unix_ms, + Some(4_242), + "the stop marker was erased" + ); + assert_eq!( + after.supervisor_pid, 0, + "the unrecorded child was put on record" + ); + assert!( + !child_alive, + "the engine nothing records must be terminated" + ); + Ok(()) + } } diff --git a/apps/rocm/src/service_launch/test_support.rs b/apps/rocm/src/service_launch/test_support.rs index 70a5504e09..5366b5f708 100644 --- a/apps/rocm/src/service_launch/test_support.rs +++ b/apps/rocm/src/service_launch/test_support.rs @@ -6,9 +6,7 @@ use anyhow::Result; use rocm_core::{AppPaths, ManagedServiceRecord}; -#[cfg(not(windows))] use std::thread; -#[cfg(not(windows))] use std::time::Duration; /// A record with every reuse-relevant field set to the "reusable" value, so @@ -64,9 +62,10 @@ pub(super) fn write_live_service_record( /// process that outlives the spawn they are checking. It is `#[ignore]`d so a /// normal run skips it, and it only sleeps when the tests that spawn it set /// `ROCM_TEST_SLEEPER_CHILD`. They select it by [`sleeper_test_name`]. +// Not gated to Unix: `an_engine_seen_dead_at_startup_is_never_signalled` in +// `launch_transaction` spawns it on Windows too. #[test] -#[cfg(not(windows))] -#[ignore = "spawned as a child process by abandoned_launch_terminates_the_spawned_child"] +#[ignore = "spawned as a child process by the abandoned-launch tests"] fn managed_launch_cleanup_sleeper_child() { if std::env::var_os("ROCM_TEST_SLEEPER_CHILD").is_some() { thread::sleep(Duration::from_mins(2)); @@ -81,7 +80,6 @@ fn managed_launch_cleanup_sleeper_child() { /// cannot be left naming a test that no longer exists. libtest names a test by /// its path inside the crate, while the item path starts with the crate name, /// which is dropped. The assertions check that against the module path as well. -#[cfg(not(windows))] pub(super) fn sleeper_test_name() -> String { let item_path = std::any::type_name_of_val(&managed_launch_cleanup_sleeper_child); let name = item_path diff --git a/docs/architecture.md b/docs/architecture.md index 6130718426..f0c4ef64e5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -95,9 +95,9 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`service_hints.rs`](../apps/rocm/src/service_hints.rs) — shared CLI-output component: the `rocm services stop|restart|remove --yes` hint text, shared by [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs), [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs), [`service_launch.rs`](../apps/rocm/src/service_launch.rs), [`service_records.rs`](../apps/rocm/src/service_records.rs), [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs), and [`services_text.rs`](../apps/rocm/src/services_text.rs) so they cannot go out of sync with each other, as `serve_summary.rs`'s stop row did from the day it was added — #417. It also holds the helpers that map a service tool name to the command word, verb and past tense those hints and messages use (`service_action_command`, `service_action_verb`, `service_action_past_tense`) and to its `SandboxToolArg` (`sandbox_tool_arg_from_service_tool`); [`services_text.rs`](../apps/rocm/src/services_text.rs) reads `service_action_past_tense` from here rather than through the crate root. It quotes the service id with `shell_quote`, so a hint names exactly the argv it runs. - [`service_launch.rs`](../apps/rocm/src/service_launch.rs) — mechanical relocation, since split into a directory of child modules, owns no code of its own beyond its declarations: spawning/attaching to a managed service process and streaming its log, the port allocation transaction and failed-launch cleanup (under the shared launch lock), plus the endpoint-key/auth checks that gate a public bind, divided among the children listed below. The file declares the children, a `test_support` module compiled only for tests (the fixtures more than one child's tests share), and `pub(crate) use` re-exports of the items other modules call, so [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`automations.rs`](../apps/rocm/src/automations.rs), [`service_control.rs`](../apps/rocm/src/service_control.rs) and [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs) still reach them as `crate::service_launch::name`; [`main.rs`](../apps/rocm/src/main.rs) does not. An item that sibling children share is `pub(super)` and reached through `super::`; everything else is private to the child that owns it. `ServicesCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs) rather than moving alongside, and `services()` lives in [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs); the record removal and pruning it dispatches to now lives in [`service_records.rs`](../apps/rocm/src/service_records.rs), and only stop/restart are also reachable from the chat/LLM sandbox tool-calling path. This module does not import [`service_control.rs`](../apps/rocm/src/service_control.rs), which imports it: `run_attached_service` takes the stop for a Ctrl-C as a parameter, and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) passes `run_internal_sandbox_tool`. - [`service_launch/endpoint_auth.rs`](../apps/rocm/src/service_launch/endpoint_auth.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, no owned types: bind-host validation and the endpoint API key policy. `validate_bind_host` refuses a non-loopback host without `--allow-public-bind` (`is_loopback_host` is the inverse of `rocm_engine_protocol::is_public_bind_host`, which `rocmd` shares); `resolve_endpoint_auth` decides the key a launch uses (none for a loopback bind unless a key is required, otherwise the supplied key trimmed or a generated one, refusing an empty key or one with a control character); `drop_orphaned_endpoint_key_on_already_running` clears a key staged for an attempt that spawned nothing; `ensure_public_bind_engine_supported` refuses a public bind with Windows managed Lemonade, whose key cannot be enforced; and `ensure_public_service_has_endpoint_key` is the guard a respawn runs so a public (or `--require-api-key`) service whose key file is gone is not reopened without authentication. All six are `pub(crate)` and re-exported by the parent: [`service_control.rs`](../apps/rocm/src/service_control.rs) reads `ensure_public_service_has_endpoint_key`, [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs) reads `is_loopback_host`, and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) reads the rest. -- [`service_launch/launch_transaction.rs`](../apps/rocm/src/service_launch/launch_transaction.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, owns types: the part of a managed launch that decides before anything spawns. `resolve_managed_port_in_transaction` refreshes liveness, answers an equivalent live service before leasing any port (`ManagedPortDecision`), and otherwise leases a concrete port against both the ports live records still reserve (`live_managed_services`, `reserved_service_ports`) and the real OS through [`serve_port.rs`](../apps/rocm/src/serve_port.rs); a caller must hold the launch lock across it and the record write that follows. `abandon_failed_managed_launch` undoes a failed attempt before that lock is released: a spawned child is terminated first and its record kept, marked stopped, while an attempt that never spawned has every artifact removed. `managed_service_reuse_refusal` (with `ManagedReuseRefusal`) is the one answer to whether a live matching service can serve a request (recipe, `--require-api-key`, device policy), and `no_usable_gpu_refusal` words the no-usable-GPU refusal once for `serve()`'s pre-flight and for the spawn. `managed_service_reuse_refusal` and `no_usable_gpu_refusal` are `pub(crate)`, re-exported, and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), and `ManagedReuseRefusal` is `pub(crate)` too because the first returns it (only [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs) matches on it); `ManagedPortDecision`, `live_managed_services`, `resolve_managed_port_in_transaction` and `abandon_failed_managed_launch` are `pub(super)` for [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs). -- [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, owns types: the detached engine child that the background and attached serve paths share. The process and stdio plumbing (`detach_background_command`, `attach_background_stdio`, `managed_service_process_command`, `managed_engine_startup_failure_detail`) is `pub(crate)` and, off Windows, re-exported: [`service_control.rs`](../apps/rocm/src/service_control.rs) reads all four for a restart and [`service_launch/helper_autostart.rs`](../apps/rocm/src/service_launch/helper_autostart.rs) reads the first three. On Windows `managed_engine_startup_failure_detail` is re-exported for test builds only, because the [`services_text.rs`](../apps/rocm/src/services_text.rs) log-tail tests call it through `crate::service_launch` there too. `ManagedLaunchReport` (`pub(crate)`) is the engine-neutral result of a launch, and `ManagedSpawn` (`pub(super)`) says whether the launch found a live service or spawned a child. `spawn_managed_engine_child` is the whole serialized transaction under the launch lock (duplicate detection, port lease, manifest and log creation, argv preparation, the bounded immediate-exit check and record publication) and returns before any readiness wait; `launch_managed_engine_child_locked` is its side-effecting half, split out so that every failure after the record's first write goes through `abandon_failed_managed_launch`. The tests reach the spawn through a test-only, thread-scoped hook (`TEST_SPAWN_HOOK`, `pub(super)` so the tests in [`service_launch/start.rs`](../apps/rocm/src/service_launch/start.rs) can observe the state at the instant of the spawn). -- [`service_launch/readiness.rs`](../apps/rocm/src/service_launch/readiness.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, owns private items: waiting on a service after it is launched or found live. `wait_for_record_ready` and `wait_on_existing_service` run the wait (one wait is bounded by `READINESS_WAIT`, and `wait_past_readiness_budget` is what `rocm serve --wait` uses to keep polling past it until the server answers or stops being live), `service_pid_is_alive`, `record_still_live` and `service_still_live` decide whether the tracked process is still there, `settle_exited_service` demotes the manifest of one that is not, and `died_during_wait` and `record_wait_outcome` end a wait, writing its outcome back without discarding what changed on disk meanwhile. `e2e_simulate_oom_launch` and `simulate_oom_managed_launch` are the end-to-end fault-injection seam behind the `e2e-oom-fault-injection` feature: the first is a `const fn` returning false without the feature, is `pub(crate)` and re-exported, and is read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs); the second fabricates the on-disk state of a launch that ran out of memory. `wait_for_record_ready`, `wait_on_existing_service`, `service_pid_is_alive`, `died_during_wait`, `record_wait_outcome` and `simulate_oom_managed_launch` are `pub(super)` for [`service_launch/start.rs`](../apps/rocm/src/service_launch/start.rs); the rest are private. +- [`service_launch/launch_transaction.rs`](../apps/rocm/src/service_launch/launch_transaction.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, owns types: the part of a managed launch that decides before anything spawns. `resolve_managed_port_in_transaction` refreshes liveness, answers an equivalent live service before leasing any port (`ManagedPortDecision`), and otherwise leases a concrete port against both the ports live records still reserve (`live_managed_services`, `reserved_service_ports`) and the real OS through [`serve_port.rs`](../apps/rocm/src/serve_port.rs); a caller must hold the launch lock across it and the record write that follows. `abandon_failed_managed_launch` undoes a failed attempt before that lock is released: a spawned child is terminated first and its record kept, marked stopped, while an attempt that never spawned has every artifact removed; an engine the startup check saw die is never signalled, since its pid is free, but its record and log are kept. `update_managed_record_on_disk` is the guarded read-modify-write every launch write after the claim goes through (the `running` record, the cleanup's `stopped` mark, the readiness verdict): it applies a change to the record as it is on disk, and only while `launch_still_owns` (or the caller's own predicate) accepts it, so a `rocm services stop` or `remove`, which do not take the launch lock, is not overwritten or undone. `managed_service_reuse_refusal` (with `ManagedReuseRefusal`) is the one answer to whether a live matching service can serve a request (recipe, `--require-api-key`, device policy), and `no_usable_gpu_refusal` words the no-usable-GPU refusal once for `serve()`'s pre-flight and for the spawn. `managed_service_reuse_refusal` and `no_usable_gpu_refusal` are `pub(crate)`, re-exported, and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), and `ManagedReuseRefusal` is `pub(crate)` too because the first returns it (only [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs) matches on it); `ManagedPortDecision`, `live_managed_services`, `resolve_managed_port_in_transaction`, `abandon_failed_managed_launch`, `update_managed_record_on_disk` and `launch_still_owns` are `pub(super)` for [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs), and the guarded write for [`service_launch/readiness.rs`](../apps/rocm/src/service_launch/readiness.rs)'s `record_wait_outcome` too. +- [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, owns types: the detached engine child that the background and attached serve paths share. The process and stdio plumbing (`detach_background_command`, `attach_background_stdio`, `managed_service_process_command`, `managed_engine_startup_failure_detail`) is `pub(crate)` and, off Windows, re-exported: [`service_control.rs`](../apps/rocm/src/service_control.rs) reads all four for a restart and [`service_launch/helper_autostart.rs`](../apps/rocm/src/service_launch/helper_autostart.rs) reads the first three. On Windows `managed_engine_startup_failure_detail` is re-exported for test builds only, because the [`services_text.rs`](../apps/rocm/src/services_text.rs) log-tail tests call it through `crate::service_launch` there too. The startup check (`fail_if_engine_died_at_startup`, settle window `MANAGED_ENGINE_STARTUP_SETTLE`; on Windows through `adopt_watched_engine_spawn`, which records the engine's pid only after the check passes) fails a launch whose engine exited inside the window with the typed `EngineDiedAtStartup`; on Windows the exit code comes from `rocm_core::spawn_detached_no_inherit_watching_startup`, which reads it through the process handle rather than re-checking a pid Windows may have reused. `scripted_managed_engine_startup_failure` is the `e2e-test-hooks` switch that scripts that death, `pub(crate)` and re-exported for [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs). `ManagedLaunchReport` (`pub(crate)`) is the engine-neutral result of a launch, and `ManagedSpawn` (`pub(super)`) says whether the launch found a live service or spawned a child. `spawn_managed_engine_child` is the whole serialized transaction under the launch lock (duplicate detection, port lease, manifest and log creation, argv preparation, the bounded immediate-exit check and record publication) and returns before any readiness wait; `launch_managed_engine_child_locked` is its side-effecting half, split out so that every failure after the record's first write goes through `abandon_failed_managed_launch`. The tests reach the spawn through a test-only, thread-scoped hook (`TEST_SPAWN_HOOK`, `pub(super)` so the tests in [`service_launch/start.rs`](../apps/rocm/src/service_launch/start.rs) can observe the state at the instant of the spawn). +- [`service_launch/readiness.rs`](../apps/rocm/src/service_launch/readiness.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, owns private items: waiting on a service after it is launched or found live. `wait_for_record_ready` and `wait_on_existing_service` run the wait (one wait is bounded by `READINESS_WAIT`, and `wait_past_readiness_budget` is what `rocm serve --wait` uses to keep polling past it until the server answers or stops being live), `service_pid_is_alive`, `record_still_live` and `service_still_live` decide whether the tracked process is still there, `settle_exited_service` demotes the manifest of one that is not, and `died_during_wait` and `record_wait_outcome` end a wait, writing its outcome back without discarding what changed on disk meanwhile; a record a stop settled during the wait fails the serve saying it was stopped while starting, one settled without a stop marker fails it saying the server exited, and a record removed during it fails the serve rather than being re-created. `e2e_simulate_oom_launch` and `simulate_oom_managed_launch` are the end-to-end fault-injection seam behind the `e2e-oom-fault-injection` feature: the first is a `const fn` returning false without the feature, is `pub(crate)` and re-exported, and is read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs); the second fabricates the on-disk state of a launch that ran out of memory. `wait_for_record_ready`, `wait_on_existing_service`, `service_pid_is_alive`, `died_during_wait`, `record_wait_outcome` and `simulate_oom_managed_launch` are `pub(super)` for [`service_launch/start.rs`](../apps/rocm/src/service_launch/start.rs); the rest are private. - [`service_launch/start.rs`](../apps/rocm/src/service_launch/start.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, no owned types: the background launch path. `start_managed_service` runs the spawn transaction, releases the launch lock once the record is published, waits for HTTP readiness and returns a `ManagedLaunchReport`; if the launch fails before a record exists it drops the endpoint key `serve()` staged for the attempt. `print_managed_launch_plain` renders the report line by line for output that is not interactive, and `wait_begin_client_config` returns the one-time client configuration for a generated key when a wait begins, built by `render_endpoint_client_config` (`pub(super)`, since [`service_launch/attach.rs`](../apps/rocm/src/service_launch/attach.rs) prints it too). The three functions other modules call are `pub(crate)`, re-exported, and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs). - [`service_launch/helper_autostart.rs`](../apps/rocm/src/service_launch/helper_autostart.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, owns a private type: starting the background automation helper (`rocm daemon`) when a watcher needs it. `ensure_background_helper_running` and `ensure_background_helper_running_quiet` do nothing unless an automation is enabled and [`background_helper.rs`](../apps/rocm/src/background_helper.rs)'s `background_helper_already_running` says no helper is running; `ensure_background_helper_running_with` takes the spawn as a parameter so the decision is testable, and `spawn_background_helper` does the detached spawn. Concurrent callers queue on an autostart lock, and `AutostartClaim` (with `AUTOSTART_CLAIM_TTL_MS`, `read_autostart_claim`, `write_autostart_claim` and `autostart_spawn_in_flight`) records a spawn in flight so the next holder defers instead of starting a second helper. Both entry points are `pub(crate)` and re-exported: [`automations.rs`](../apps/rocm/src/automations.rs) calls the first, and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) and [`service_launch/attach.rs`](../apps/rocm/src/service_launch/attach.rs) call the quiet one. - [`service_launch/attach.rs`](../apps/rocm/src/service_launch/attach.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, owns private types: the attached serve path (`--verbose`/`--foreground`). `run_attached_service` spawns the same detached child the background path does and follows its log in the terminal until the user detaches with Ctrl-D (the server keeps running) or stops it with Ctrl-C, or the engine exits; `AttachOutcome` is that result and `detach_key_outcome` maps a key press to it. `stream_attached_logs` reads keys in crossterm raw mode (`RawModeGuard` restores the terminal on every exit path) and `stream_attached_logs_no_tty` follows the log until the process exits when stdin is not a terminal. `finish_attached_stop` turns the answer of the stop that Ctrl-C ran into output: a confirmed stop is reported on stdout, an unconfirmed one becomes the error, and a failed stop signals the child this session launched. `run_attached_service` is `pub(crate)`, re-exported, and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs). diff --git a/docs/manual-testing.md b/docs/manual-testing.md index c0431d6c65..2a02695ef4 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -599,6 +599,42 @@ line. `restart` starts the server again only once its stop has confirmed every recorded process gone; otherwise it reports that and starts nothing. +**An engine that dies at startup.** On Windows and Linux alike, a managed +engine that exits within its first 200 ms fails `rocm serve --managed` +instead of reporting a server that is still starting. +To see it without a broken engine, build the CLI with its test hooks and arm +the scripted death, which makes the spawned engine reject its arguments: + +```powershell +cargo build -p rocm --features e2e-test-hooks +$env:ROCM_E2E_MANAGED_ENGINE_STARTUP_FAILURE = "1" # bash: export ROCM_E2E_MANAGED_ENGINE_STARTUP_FAILURE=1 +./target/debug/rocm serve qwen --engine lemonade --managed +./target/debug/rocm services list --all +``` + +Expected result: + +- The serve exits non-zero with `managed engine exited immediately with status + ...; inspect `, where `` is the service's own log, plus a + tail of that log when the engine wrote anything to it. On Windows the status + is the engine's exit code too. +- `rocm services list --all` shows the attempt as `stopped`. Running the same + `rocm serve` again fails again with the same `managed engine exited + immediately ...` error, rather than reporting the earlier attempt as already + running, and `rocm services list --all` then shows two `stopped` records. +- An engine that dies later than that — while loading its runtime or model — + is not caught by this check; the serve then waits for readiness and reports + the service as still starting, as before. + +Unset the variable afterwards; a build without `--features e2e-test-hooks` +ignores it. + +To see the stop-during-start case, run `rocm serve --managed` in one +terminal and, while it is still waiting for the server, `rocm services stop + --yes` from another. The serve exits non-zero with an error saying +the managed service "was stopped while it was starting", and +`rocm services list --all` shows the service as `stopped`. + The rest of this section needs a **vLLM** server running, on Linux or WSL, and two runtimes to move it between. Nothing earlier in this document sets either up: section 3 serves Lemonade and then stops it on both of its branches, the From 8d5e481c074483d4ef8fe45bbf61e66a062fd86a Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 10 Nov 2026 11:43:43 +0200 Subject: [PATCH 3/9] test(e2e): cover an engine that dies at startup (serve-54) serve-54 scripts the managed engine's startup death. It checks that the serve fails naming the engine's own log, that the record left on disk reads "stopped" and names no pid, and that serving the same model again reaches the engine launch again. It carries no host tag, so every lane built with e2e-test-hooks runs it, the self-hosted Windows lane among them. The required Windows job builds without the hooks; the Windows watch is covered there by rocm-core's watched_detached_spawn_* unit tests. docs/testing.md gains an entry for the scenario and the focused unit tests behind it. Signed-off-by: Roman Inflianskas --- docs/testing.md | 44 ++++++ .../features/model_serving.feature | 26 ++++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 141 ++++++++++++++++++ 3 files changed, 211 insertions(+) diff --git a/docs/testing.md b/docs/testing.md index f630364d0d..ffe89a0866 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1786,6 +1786,50 @@ same refusal on the required Windows lane. The vLLM live GPU acceptance script should return a clean skip on native Windows. It remains a strict GPU-required test on Linux/WSL. +Managed-launch startup-death and record-write focused tests: + +```bash +cargo test -p rocm --bin rocm service_launch::spawn::tests::a_child_that_exited_during_the_startup_window +cargo test -p rocm --bin rocm service_launch::launch_transaction::tests::an_engine_seen_dead_at_startup_is_never_signalled +cargo test -p rocm --bin rocm service_launch::launch_transaction::tests::abandoning_a_launch_does_not_recreate_a_removed_record +cargo test -p rocm --bin rocm service_launch::spawn::tests::a_stop_settled_while_the_engine_spawns +cargo test -p rocm --bin rocm service_launch::readiness::tests::a_stop_settled_during_the_wait +cargo test -p rocm --bin rocm service_launch::readiness::tests::record_wait_outcome +cargo test -p rocm --bin rocm service_launch::launch_transaction::tests::the_launch_does_not_publish_over +cargo test -p rocm --bin rocm service_launch::launch_transaction::tests::an_engine_seen_dead_before +cargo test -p rocm --bin rocm service_launch::spawn::tests::an_engine_that_dies_at_startup_keeps +cargo test -p rocm --bin rocm service_launch::spawn::tests::a_watched_spawn_seen_dead +cargo test -p rocm --bin rocm service_launch::readiness::tests::an_engine_that_exited_during_the_wait +cargo test -p rocm-core watched_detached_spawn # Windows only +``` + +They pin that an engine seen dead inside the 200 ms startup window fails the +launch naming its own log, on Windows from the exit code read through the +process handle `CreateProcessW` returned (not by re-checking a pid Windows may +already have reused); that cleanup never signals such an engine, since its pid is +free; and that the launch's writes after its claim — the `running` record, the +`stopped` mark of a failed launch, the readiness verdict — go to the record as it +is on disk, so a `rocm services stop` settled meanwhile keeps its `stopped` and +stop marker, and a `rocm services remove` is not undone by re-creating the record. +A serve whose server is stopped during its readiness wait fails (non-zero exit) saying +the service was stopped while it was starting, instead of reporting a launch; one whose +server exited on its own during the wait (a record settled without a stop marker) fails +saying it exited and naming `rocm logs --service `. +The `watched_detached_spawn_*` tests run on the required Windows lane. + +`serve-54` (`@id:serve-managed-engine-dies-at-startup`) is the black-box half: an +`e2e-test-hooks` build scripts the managed engine's death (the spawned `rocm` +gets an argument it rejects), and the scenario checks that the serve fails naming +the engine's own log, that the record left on disk reads `stopped` and names no +pid, and that serving the same model again reaches the engine launch again. It +carries no host tag, so every lane built with the hooks runs it, including the +self-hosted Windows lane; the required `windows-build-and-test` lane builds +without the hooks and does not. Run it locally with: + +```bash +cargo xtask e2e -- -n "serve-54 " +``` + Serve resolver focused tests: ```bash diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index fcc3d09f2f..00679ffcf1 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -849,3 +849,29 @@ Feature: Model serving When the user sends a tool-use request to the Anthropic messages endpoint Then the response contains a tool_use content block And the tool_use block names a tool and arguments from the request + + # A managed engine is spawned detached, so if it dies on the way up there is no + # terminal for it to report to: the child's own service log is the only account + # of why. The launch fails naming that log, and leaves its record `stopped` + # (not `failed`, which `rocmd`'s recovery would respawn) without naming the + # dead child: its pid is free, and on Windows no start-time token would stop a + # later cleanup from signalling whatever process reuses it. Serving the same + # model again then reaches the engine launch again. The engine is scripted to + # die at startup in test builds (`rocm/e2e-test-hooks`), which also waives the + # no-GPU pre-flight and engine preparation so this reaches a real spawn without + # GPU hardware or a runtime download — the death itself is real, only its + # trigger is scripted. + # + # Carries no host tag on purpose, so it runs on every lane built with the + # hooks, the self-hosted Windows lane among them: `@requires-no-gpu` would skip + # it on any host that HAS a GPU, and the premise is a dead engine, not an + # absent GPU. The required `windows-build-and-test` lane does not build the + # hooks, so it cannot run this; the Windows watch itself is covered there by + # `rocm-core`'s `watched_detached_spawn_*` unit tests. + @id:serve-managed-engine-dies-at-startup + Scenario: serve-54 - An engine that dies at startup fails the serve and names its log + Given the managed engine dies during startup + When the user serves a model with Lemonade + Then serving fails and names the engine's own log + When the user serves the same model again + Then that serve reaches the engine launch again and fails the same way diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index c6eb6cb412..ae212db2d3 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -3361,3 +3361,144 @@ async fn no_background_automation_daemon_was_left_behind(world: &mut E2eWorld) { std::fs::read_to_string(&claim_path).ok() ); } + +/// Arms the scripted startup death. Also waives serve's no-GPU pre-flight and +/// engine preparation, so the black-box test reaches a real spawn on a host with +/// neither GPU hardware nor an installed runtime. +#[given("the managed engine dies during startup")] +async fn managed_engine_dies_during_startup(world: &mut E2eWorld) { + world.command_env.push(( + rocm_core::MANAGED_ENGINE_STARTUP_FAILURE_TEST_ENV, + "1".into(), + )); +} + +#[then("serving fails and names the engine's own log")] +async fn assert_startup_death_names_log(world: &mut E2eWorld) { + let output = serve_output(world); + // A binary built without `rocm/e2e-test-hooks` never arms the scripted death: + // on a GPU lane it launches a real engine, which this exit-code check is the + // first to notice — so it names the feature as the likely cause. + assert_ne!( + world.cli_rc, + Some(0), + "an engine that died at startup must fail the serve (if it did not die at \ + all, check the binary under test was built with `rocm/e2e-test-hooks`):\n{output}" + ); + // See the note in assert_lemonade_preparation_retry_is_bounded: the seam that + // scripts the death also waives the no-GPU pre-flight, so this refusal can + // only appear when the binary under test was built without the feature. That + // diagnosis is specific to a host with no usable GPU, where the pre-flight is + // what stops such a binary. + assert!( + !output.contains("no usable AMD GPU detected"), + "serve stopped at the no-GPU pre-flight, so the binary under test was \ + built without the `rocm/e2e-test-hooks` feature and never reached the \ + engine launch:\n{output}" + ); + assert!( + output.contains("managed engine exited immediately"), + "expected the launch to report the engine's immediate exit:\n{output}" + ); + // The child is detached, so its own log is the only account of why it died — + // the user has to be told where it is. The path is read off the record the + // launch left behind, not matched as a `.log` suffix that any incidental + // mention of a log would satisfy; and it is the path `rocm` itself recorded, + // so a Windows 8.3 short form cannot make the comparison disagree with itself. + // + // The two platforms reach this through different branches of the message. + // On Unix the parent redirects the child's stdio into the log, so the dead + // child's output is there and the message carries a tail. On Windows the + // detached spawn does not redirect, and the scripted child never receives + // `--log`, so the log stays empty and only the no-tail branch runs. Naming + // the path is the claim both branches share, and it is the one pinned here. + let services = world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .join("data") + .join("services"); + let records: Vec = std::fs::read_dir(&services) + .unwrap_or_else(|e| panic!("no services dir at {}: {e}", services.display())) + .filter_map(|entry| std::fs::read_to_string(entry.ok()?.path()).ok()) + .filter_map(|json| serde_json::from_str::(&json).ok()) + .filter(|record| record["engine"] == "lemonade") + .collect(); + let log_paths: Vec<&str> = records + .iter() + .filter_map(|record| record["log_path"].as_str()) + .collect(); + assert!( + !log_paths.is_empty(), + "the failed launch left no lemonade service record in {}", + services.display() + ); + assert!( + log_paths.iter().any(|path| output.contains(path)), + "expected the failure to name the engine's own service log {log_paths:?}:\n{output}" + ); + // Read raw from disk, not through `rocm services`, whose liveness refresh + // would rewrite the very fields pinned here. `stopped`, not `failed`: + // `rocmd`'s recovery restarts a managed `failed` record that carries no stop + // marker, which would respawn the launch this serve just reported as failed. + let statuses: Vec<&str> = records + .iter() + .filter_map(|record| record["status"].as_str()) + .collect(); + assert_eq!( + statuses, + ["stopped"], + "the failed launch must leave its record `stopped`, not claiming the engine \ + and model" + ); + // The startup check saw the child die, so the record never names it: the + // pid is free, and a later stop or cleanup must not signal whatever process + // reuses it. On Windows, which records no start-time token, nothing else + // would tell that process from the dead engine. Both platforms leave the + // claim as it was written, before any child was put on record. + let pids: Vec<(serde_json::Value, serde_json::Value)> = records + .iter() + .map(|record| { + ( + record["supervisor_pid"].clone(), + record["engine_pid"].clone(), + ) + }) + .collect(); + assert_eq!( + pids, + [(serde_json::json!(0), serde_json::Value::Null)], + "a child the startup check saw die must not be named by the record" + ); +} + +#[when("the user serves the same model again")] +async fn user_serves_the_same_model_again(world: &mut E2eWorld) { + // Re-arm the scripted death: the runner consumes the scenario env on each + // invocation, and the retry has to die the same way to be recognisable. + world.command_env.push(( + rocm_core::MANAGED_ENGINE_STARTUP_FAILURE_TEST_ENV, + "1".into(), + )); + user_serves_with_failing_lemonade_preparation(world).await; +} + +#[then("that serve reaches the engine launch again and fails the same way")] +async fn assert_retry_reaches_the_engine_launch(world: &mut E2eWorld) { + // The retry reaches the engine launch again and dies there the same way, + // rather than being answered by the first launch's record as an engine already + // starting. The `stopped` status asserted by the previous step is what frees + // the engine and model for it. + let output = serve_output(world); + assert_ne!( + world.cli_rc, + Some(0), + "the retry should fail the same way, not succeed:\n{output}" + ); + assert!( + output.contains("managed engine exited immediately"), + "the retry must reach the engine launch again rather than being turned \ + away by the previous failed launch:\n{output}" + ); +} From 20913e566d2d5ee3d9f7049913fc62aeacf1c9a4 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Fri, 13 Nov 2026 12:33:40 +0200 Subject: [PATCH 4/9] refactor(serve): leave the helper-start spinner fix to its own change Clearing the serve spinner when the background helper fails to start has nothing to do with an engine that dies at startup. Put serve() back to main's inline clear-on-error after the launch and drop LaunchProgress, launch_then_start_helper and its three tests; the helper-start case can come back as a separate change. Signed-off-by: Roman Inflianskas --- apps/rocm/src/serve_cmd.rs | 174 +++++++------------------------------ 1 file changed, 29 insertions(+), 145 deletions(-) diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index d1e984a798..93fc3e939c 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -499,39 +499,6 @@ pub(crate) struct ServeArgs { pub(crate) engine_env: Vec, } -/// The progress line a background launch paints while it waits. -trait LaunchProgress { - /// Erases the line so whatever prints next starts on a clean one. - fn clear(&mut self); -} - -impl LaunchProgress for cli_progress::Spinner { - fn clear(&mut self) { - Self::clear(self); - } -} - -/// Runs a background serve's managed launch, then starts the background -/// helper, and erases `progress` if either step fails. -/// -/// `Spinner` has no `Drop`, so an error returned past a painted line would -/// print on top of it. Both steps are fallible and run back to back, so they -/// share one clear-on-error here instead of each call site guarding its own. -/// The helper is not started when the launch fails. On success `progress` is -/// left painted: the caller still uses it (the smoke test relabels it) and -/// clears it itself. -fn launch_then_start_helper( - progress: &mut P, - launch: impl FnOnce(&mut P) -> Result, - start_helper: impl FnOnce() -> Result<()>, -) -> Result { - let launched = launch(progress).and_then(|report| start_helper().map(|()| report)); - if launched.is_err() { - progress.clear(); - } - launched -} - pub(crate) fn serve(args: ServeArgs) -> Result<()> { let ServeArgs { model, @@ -1224,38 +1191,36 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { }, &mut |_elapsed| spinner.borrow_mut().tick(), ); + // `Spinner` has no `Drop`: clear its row so `Error:` is not glued to it. + let report = match launched { + Ok(report) => report, + Err(error) => { + spinner.borrow_mut().clear(); + return Err(error); + } + }; let mut spinner = spinner.into_inner(); - // `Spinner` has no `Drop`: a failed launch or a failed helper start clears - // its row before the error prints, so `Error:` is not glued to it. - let report = launch_then_start_helper( - &mut spinner, - |_| launched, - // Not when the launch was fabricated. `simulate_oom_managed_launch` - // writes a `starting` record for a process that never existed, so the - // helper would be left watching a pid that will never appear, with - // nothing in the scenario's teardown to stop it: observed as `rocm - // daemon` processes outliving their e2e checkout by days, still - // carrying `ROCM_E2E_SIMULATE_OOM_LAUNCH=1`. The helper is for - // supervising a real spawned server, and there is none here. - // - // That — not ordering — is the whole justification. This suppression - // runs *before* the OOM note is computed in the `summary_mode` block - // below, so the previous claim here, that the note "is computed before - // it would have run", was backwards and could not have excused it. - // serve-40's `no background automation daemon was left behind` is what - // holds the branch: deleting it left the scenario green with a `rocm - // daemon` still running. - // - // `e2e_simulate_oom_launch` is a `const fn` returning `false` without - // the test-only feature, so this branch folds away in shipped builds. - || { - if e2e_simulate_oom_launch() { - Ok(()) - } else { - ensure_background_helper_running_quiet(summary_mode) - } - }, - )?; + // Not when the launch was fabricated. `simulate_oom_managed_launch` + // writes a `starting` record for a process that never existed, so the + // helper would be left watching a pid that will never appear, with + // nothing in the scenario's teardown to stop it: observed as `rocm + // daemon` processes outliving their e2e checkout by days, still + // carrying `ROCM_E2E_SIMULATE_OOM_LAUNCH=1`. The helper is for + // supervising a real spawned server, and there is none here. + // + // That — not ordering — is the whole justification. This suppression + // runs *before* the OOM note is computed in the `summary_mode` block + // below, so the previous claim here, that the note "is computed before + // it would have run", was backwards and could not have excused it. + // serve-40's `no background automation daemon was left behind` is what + // holds the branch: deleting it left the scenario green with a `rocm + // daemon` still running. + // + // `e2e_simulate_oom_launch` is a `const fn` returning `false` without + // the test-only feature, so this branch folds away in shipped builds. + if !e2e_simulate_oom_launch() { + ensure_background_helper_running_quiet(summary_mode)?; + } // An equivalent service was already running, so nothing was spawned and the // freshly generated key is unused — drop it rather than leave it orphaned in @@ -2217,85 +2182,4 @@ mod tests { "non-vLLM engines must not be told to pass a vLLM-only flag: {lemonade:?}" ); } - - /// Records what a background launch did to its progress line. - #[derive(Default)] - struct RecordedProgress { - ticks: usize, - clears: usize, - } - - impl LaunchProgress for RecordedProgress { - fn clear(&mut self) { - self.clears += 1; - } - } - - #[test] - fn a_background_helper_that_fails_to_start_clears_the_launch_progress() { - let mut progress = RecordedProgress::default(); - - let result = launch_then_start_helper( - &mut progress, - |progress| { - progress.ticks += 1; - Ok("report") - }, - || anyhow::bail!("background helper did not start"), - ); - - let error = result.expect_err("the helper's failure is the launch's failure"); - assert_eq!(error.to_string(), "background helper did not start"); - assert_eq!(progress.ticks, 1, "the launch ran before the helper"); - assert_eq!( - progress.clears, 1, - "the painted line is erased before the error prints" - ); - } - - #[test] - fn a_failed_launch_clears_its_progress_and_never_starts_the_helper() { - let mut progress = RecordedProgress::default(); - let mut helper_started = false; - - let result: Result<()> = launch_then_start_helper( - &mut progress, - |_| anyhow::bail!("engine died at startup"), - || { - helper_started = true; - Ok(()) - }, - ); - - let error = result.expect_err("the launch failed"); - assert_eq!(error.to_string(), "engine died at startup"); - assert!( - !helper_started, - "no helper is started for a launch that failed" - ); - assert_eq!(progress.clears, 1); - } - - #[test] - fn a_successful_launch_leaves_its_progress_for_the_caller_to_clear() { - let mut progress = RecordedProgress::default(); - let mut helper_started = false; - - let report = launch_then_start_helper( - &mut progress, - |_| Ok("report"), - || { - helper_started = true; - Ok(()) - }, - ) - .expect("both steps succeed"); - - assert_eq!(report, "report"); - assert!(helper_started); - assert_eq!( - progress.clears, 0, - "the smoke test still relabels the line, so the caller clears it" - ); - } } From 540e21664f509e50dbb705b16c754460cf583c8e Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Fri, 13 Nov 2026 12:33:47 +0200 Subject: [PATCH 5/9] fix(serve): name every test hook that waives the no-GPU bail once The bail checked three test hooks (scripted Lemonade failure, simulated OOM launch, scripted startup death), but gpu_refusal_waived_for_reuse still subtracted only the first, and the reuse pre-gate's doc and the waiver's comment named only that one too. Compute the waiver once and use it in both places, so a hook that waives the bail can no longer leave the reuse waiver claiming otherwise. The pre-gate's own ensure_self_managed_engine_ready call is skipped under the scripted startup death as well, like the later call already was: that scenario asserts the launch, not a runtime download. All of this is test-build only; none of the hooks exists in a shipped binary. Signed-off-by: Roman Inflianskas --- apps/rocm/src/serve_cmd.rs | 42 +++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 93fc3e939c..382cbfe56d 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -421,12 +421,15 @@ fn aim_board_note( /// AMD GPU. /// /// Not "exactly when the bail fires", and the difference is only in test-hook -/// builds: the bail also requires `!scripted_backend_failure` (see the `if` -/// below it), which is `cfg!(feature = "e2e-test-hooks")`/env-gated. Under that -/// scenario the gate can open while the bail cannot fire. That is deliberate — -/// the hook stands in for the GPU the lane does not have, so the reuse path it -/// exercises must still behave as it would on a real GPU-less host — and it -/// costs nothing in a shipped binary, where the term can never be true. +/// builds: the bail is also waived by `gpu_bail_waived_by_test_hook` in +/// `serve()` — the scripted Lemonade backend failure or free-space override, +/// the simulated OOM launch, and the scripted engine startup death — each +/// compiled in only with a test feature and armed by an environment variable. +/// Under any of them the gate can open while the bail cannot fire. That is +/// deliberate — the hook stands in for the GPU the lane does not have, so the +/// reuse path it exercises must still behave as it would on a real GPU-less +/// host — and it costs nothing in a shipped binary, where none of the terms can +/// be true. /// /// `host_has_no_usable_gpu` is therefore the bail's own predicate, computed once /// by the caller and handed to both, so the two cannot drift apart. They did: @@ -767,7 +770,12 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { || selection.env_id.is_some() || engine_manages_own_runtime(&selected_engine); if can_resolve_model { - if engine_manages_own_runtime(&selected_engine) { + // Not under the scripted startup death, as for the same call below: + // that scenario asserts the launch, and a real runtime download is + // exactly what its ungated lane cannot do. + if engine_manages_own_runtime(&selected_engine) + && !scripted_managed_engine_startup_failure() + { ensure_self_managed_engine_ready(&paths, &mut config, &selected_engine)?; } let probe = engine_request::<_, ResolveModelResponse>( @@ -880,13 +888,13 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // The scripted startup-death scenario bypasses the same host precondition, for // the same reason: it asserts what happens *after* a spawn, so it must reach // one without real GPU hardware. - if !cpu_only - && !scripted_backend_failure - && !e2e_simulate_oom_launch() - && !scripted_managed_engine_startup_failure() - && !reuse_existing - && host_has_no_usable_gpu - { + // + // Every test hook that waives this bail is named here once, so the bail and + // `gpu_refusal_waived_for_reuse` below cannot disagree about which ones do. + let gpu_bail_waived_by_test_hook = scripted_backend_failure + || e2e_simulate_oom_launch() + || scripted_managed_engine_startup_failure(); + if !cpu_only && !gpu_bail_waived_by_test_hook && !reuse_existing && host_has_no_usable_gpu { bail!("{}", no_usable_gpu_refusal(&device_policy)); } // The Docker probe comes after every refusal that needs no daemon (the AIM @@ -895,11 +903,11 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { ensure_engine_allowed_on_platform(&selected_engine, GatedEngineAction::Serve)?; // Whether the refusal above was waived *only* because a live service was // expected to be reused. `reuse_existing` already implies `!cpu_only` and - // `host_has_no_usable_gpu` (the pre-gate requires both), so the scripted - // backend failure is the one remaining term. Handed to the spawn, which + // `host_has_no_usable_gpu` (the pre-gate requires both), so the test-hook + // waiver is the one remaining term. Handed to the spawn, which // re-checks reuse authoritatively under the launch lock and raises the // refusal itself if the service is gone by then. - let gpu_refusal_waived_for_reuse = reuse_existing && !scripted_backend_failure; + let gpu_refusal_waived_for_reuse = reuse_existing && !gpu_bail_waived_by_test_hook; // `--gpu` selects by the amd-smi `gpu` ordinal but is exported via // `HIP_VISIBLE_DEVICES`; those orderings can diverge when // `ROCR_VISIBLE_DEVICES`/partitioning is in play, so warn at serve time. From 16578bfc24b284804a384095d76e7819f29ca6d8 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Fri, 13 Nov 2026 12:34:15 +0200 Subject: [PATCH 6/9] fix(serve): keep a restart's endpoint key when cleaning up a failed launch A failed launch's cleanup skips its `stopped` mark when a restart has taken the record over, but it still cleared the endpoint key, and a restart stores its key under the same service id. Clear the key only when the mark applied or the record is gone. update_managed_record_on_disk and record_wait_outcome checked exists() before reading the record, so a removal between the two came back as a read error. They now read once and treat the typed ManagedServiceNotFound as the record being gone. The publish copies only the launch's own fields over the record on disk, because a stop writes its marker before `stopped`. Say so where it happens, and pin which fields those are with a test in which a stop marker lands during the spawn. Also: - the exited-while-starting and died-during-wait messages take their `rocm logs --service` hint from service_hints, so the id is quoted; - the doc of abandon_failed_managed_launch no longer says that a record with no engine pid means nothing ran, and the claims about start-time tokens now hold on Linux as well as Windows; - a seam test pins that an engine seen dead at startup is never handed to the termination; - manual-testing.md and testing.md say the stop-during-start result is the usual one, not a certain one, and testing.md lists the new tests by unambiguous names. Signed-off-by: Roman Inflianskas --- apps/rocm/src/service_hints.rs | 25 +++ .../src/service_launch/launch_transaction.rs | 151 +++++++++++++++--- apps/rocm/src/service_launch/readiness.rs | 52 ++++-- apps/rocm/src/service_launch/spawn.rs | 117 +++++++++++++- docs/architecture.md | 2 +- docs/manual-testing.md | 8 +- docs/testing.md | 13 +- 7 files changed, 326 insertions(+), 42 deletions(-) diff --git a/apps/rocm/src/service_hints.rs b/apps/rocm/src/service_hints.rs index 6556c2e70a..00131232f0 100644 --- a/apps/rocm/src/service_hints.rs +++ b/apps/rocm/src/service_hints.rs @@ -48,6 +48,13 @@ pub(crate) fn service_forced_remove_hint(service_id: &str) -> String { ) } +/// The directly-runnable `rocm logs --service ` hint, which prints a local +/// managed service's own log, for a message that sends the user there to see +/// why the server exited. +pub(crate) fn service_logs_hint(service_id: &str) -> String { + format!("rocm logs --service {}", shell_quote(service_id)) +} + /// The directly-runnable `rocm services stop --yes` hint. pub(crate) fn service_stop_hint(service_id: &str) -> String { service_action_hint("stop", service_id) @@ -145,5 +152,23 @@ mod tests { ); } } + + // Mutation: printing `service_id` bare in `service_logs_hint` fails this + // for any id carrying whitespace or a quote. + #[test] + fn the_logs_hint_denotes_the_argv_it_names(service_id in hostile_arg()) { + let hint = service_logs_hint(&service_id); + prop_assert_eq!( + shlex::split(&hint), + Some(vec![ + "rocm".to_owned(), + "logs".to_owned(), + "--service".to_owned(), + service_id, + ]), + "{}", + hint + ); + } } } diff --git a/apps/rocm/src/service_launch/launch_transaction.rs b/apps/rocm/src/service_launch/launch_transaction.rs index 208a2d4705..3c5d64e65c 100644 --- a/apps/rocm/src/service_launch/launch_transaction.rs +++ b/apps/rocm/src/service_launch/launch_transaction.rs @@ -134,26 +134,36 @@ const fn launch_cleanup_refusal( /// Order matters: a spawned child is terminated *first*, so it is not still /// holding the port or writing to the log by the time cleanup runs. The /// termination outcome is audited but not acted on: the record is marked -/// abandoned (`stopped`, releasing the port) even when death was not confirmed. If no child was ever spawned (`record.engine_pid` is still `None`), -/// every artifact this attempt created is removed: there is nothing for -/// `rocm services`/`rocm logs` to show. If a child did run, its record and log -/// are kept instead (marked `status = "stopped"`, see -/// [`ManagedServiceRecord::mark_launch_abandoned`]), since the error that led here -/// already told the user to go inspect that log; only the endpoint key, which a -/// failed attempt has no use for, is cleared. +/// abandoned (`stopped`, releasing the port) even when death was not confirmed. +/// +/// If no child ran — none was spawned, so `record.engine_pid` is still `None` +/// and `child_known_dead` is unset — every artifact this attempt created is +/// removed: there is nothing for `rocm services`/`rocm logs` to show. If a +/// child did run, its record and log are kept instead (marked +/// `status = "stopped"`, see [`ManagedServiceRecord::mark_launch_abandoned`]), +/// since the error that led here already told the user to go inspect that log; +/// only the endpoint key, which a failed attempt has no use for, is cleared. A +/// child ran in two shapes: its pid is on record, or `child_known_dead` says it +/// died before its pid could be recorded (below). /// Every step is best-effort and idempotent; cleanup must never mask the /// original failure with a second one. /// /// `child_known_dead` is set when the startup check saw the child exit /// (`EngineDiedAtStartup` in `spawn`). Such a child is not signalled at all: it is -/// gone, and its pid may already belong to an unrelated process. No start-time -/// token was captured for it before it died, so nothing else would stop the kill -/// from landing on that process. Its record and log are still kept, since the -/// error names that log. +/// gone, and its pid may already belong to an unrelated process. On Windows the +/// check runs before the pid is recorded, so no start-time token was captured +/// for it and nothing else would stop the kill from landing on that process; on +/// Linux the token taken at spawn would refuse such a process, but there is +/// nothing left to signal either way. Its record and log are still kept, since +/// the error names that log. /// /// The `stopped` mark is a guarded write ([`update_managed_record_on_disk`]): /// a stop that settled the record, a restart that took it over, or a removal -/// is left as it is rather than overwritten or re-created. +/// is left as it is rather than overwritten or re-created. The endpoint key +/// follows the same guard: it is cleared only when the mark applied or the +/// record is gone. A record that is still there but no longer this launch's +/// keeps it — a restart that took the record over stores its key under the same +/// service id, and a stop that settled it does its own key cleanup. pub(super) fn abandon_failed_managed_launch( paths: &AppPaths, record: &ManagedServiceRecord, @@ -182,13 +192,17 @@ fn abandon_failed_managed_launch_with( if let Some(pid) = spawned_pid.filter(|_| !child_known_dead) { terminate_abandoned_launch_child(paths, record, pid, terminate); } - let _ = update_managed_record_on_disk( + let marked = update_managed_record_on_disk( paths, &record.service_id, |current| launch_still_owns(current, record), ManagedServiceRecord::mark_launch_abandoned, ); - endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); + let owned_by_someone_else = + matches!(marked, Ok(None)) && paths.service_manifest_path(&record.service_id).exists(); + if !owned_by_someone_else { + endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); + } return; } for path in [ @@ -273,10 +287,19 @@ pub(super) fn update_managed_record_on_disk( still_owned: impl FnOnce(&ManagedServiceRecord) -> bool, update: impl FnOnce(&mut ManagedServiceRecord), ) -> Result> { - if !paths.service_manifest_path(service_id).exists() { - return Ok(None); - } - let mut current = ManagedServiceRecord::read_manifest(paths, service_id)?; + // One read, not an `exists()` check before it: a removal landing between + // the two would turn "the record is gone" into a read error. + let mut current = match ManagedServiceRecord::read_manifest(paths, service_id) { + Ok(current) => current, + Err(error) + if error + .downcast_ref::() + .is_some() => + { + return Ok(None); + } + Err(error) => return Err(error), + }; if !still_owned(¤t) { return Ok(None); } @@ -1184,8 +1207,9 @@ mod tests { /// A sleeper the test owns, standing in for an unrelated process that now /// holds a pid the launch once recorded. Spawned with `std`'s `Command` - /// directly, so the test runs on Windows too, where this matters most: no - /// start-time token guards a kill there. + /// directly, so the test runs on Windows too, where this matters most: the + /// startup check there runs before the dead engine's pid is recorded, so no + /// start-time token was captured that could guard a kill. fn spawn_bystander() -> std::process::Child { std::process::Command::new(std::env::current_exe().expect("current test exe")) .arg("--exact") @@ -1305,6 +1329,93 @@ mod tests { Ok(()) } + /// Through the seam, with a token captured and so on every host: a child the + /// startup check saw die is never handed to the termination at all, while + /// the same record with the child not known dead is. + #[test] + fn an_engine_seen_dead_at_startup_is_never_handed_to_the_termination() -> Result<()> { + use std::cell::Cell; + for (child_known_dead, expect_call) in [(true, false), (false, true)] { + let (root, paths) = test_paths("abandon-known-dead-seam"); + paths.ensure()?; + let mut record = write_live_service_record( + &paths, + "lemonade-abandon-known-dead-seam", + "abandon-known-dead-seam-model", + 11_493, + "starting", + )?; + record.engine_pid = Some(4243); + record.engine_start_ticks = Some(78); + let asked = Cell::new(false); + + abandon_failed_managed_launch_with(&paths, &record, child_known_dead, |_| { + asked.set(true); + rocm_core::TerminationOutcome::Forced + }); + + let persisted = ManagedServiceRecord::read_manifest(&paths, &record.service_id); + let _ = fs::remove_dir_all(root); + assert_eq!( + asked.get(), + expect_call, + "child_known_dead {child_known_dead}" + ); + assert_eq!( + persisted?.status, "stopped", + "child_known_dead {child_known_dead}" + ); + } + Ok(()) + } + + /// A restart that took the record over during the failed launch stored its + /// own endpoint key under the same service id. The cleanup leaves that + /// record alone, and must leave its key alone too; the same cleanup on a + /// record that is still the launch's own clears the key. + #[test] + fn abandoning_a_launch_keeps_the_key_of_a_record_a_restart_took_over() -> Result<()> { + for taken_over in [true, false] { + let (root, paths) = test_paths("abandon-keeps-restart-key"); + paths.ensure()?; + let launched = write_live_service_record( + &paths, + "lemonade-abandon-restart-key", + "abandon-restart-key-model", + 11_492, + "starting", + )?; + if taken_over { + let mut taken = ManagedServiceRecord::read_manifest(&paths, &launched.service_id)?; + taken.supervisor_pid = launched.supervisor_pid.wrapping_add(7); + taken.write()?; + } + endpoint_keys::store_endpoint_api_key(&paths, &launched.service_id, "restart-key")?; + + abandon_failed_managed_launch(&paths, &launched, true); + + let key = endpoint_keys::endpoint_api_key(&paths, &launched.service_id); + let persisted = ManagedServiceRecord::read_manifest(&paths, &launched.service_id); + let _ = fs::remove_dir_all(root); + let persisted = persisted?; + if taken_over { + assert_eq!( + key.as_deref(), + Some("restart-key"), + "the restart's key was cleared" + ); + assert_eq!( + persisted.status, "starting", + "the restart's record was overwritten" + ); + } else { + assert_eq!(key, None, "a failed launch's own key must be cleared"); + assert_eq!(persisted.status, "stopped"); + } + } + Ok(()) + } + #[test] fn an_engine_seen_dead_before_it_was_recorded_keeps_its_record_and_log() -> Result<()> { // The Windows shape: the startup check saw the engine exit before diff --git a/apps/rocm/src/service_launch/readiness.rs b/apps/rocm/src/service_launch/readiness.rs index 82dd7bced0..8c1c93b11e 100644 --- a/apps/rocm/src/service_launch/readiness.rs +++ b/apps/rocm/src/service_launch/readiness.rs @@ -19,6 +19,7 @@ use super::spawn::ManagedLaunchReport; use crate::device_policy_name; use crate::endpoint_keys; use crate::engine_runtime_select::wait_for_service_http_ready_with_progress; +use crate::service_hints; use crate::status_for_readiness; #[cfg(feature = "e2e-oom-fault-injection")] use anyhow::Context; @@ -130,7 +131,8 @@ pub(super) fn died_during_wait( } settle_exited_service(paths, service_id); let mut message = format!( - "the server exited before it became ready (vLLM and AIM give up after 5 minutes by default; `ROCM_CLI_VLLM_READY_TIMEOUT_SECS` / `ROCM_CLI_AIM_READY_TIMEOUT_SECS` allow longer); see `rocm logs --service {service_id}` for its output" + "the server exited before it became ready (vLLM and AIM give up after 5 minutes by default; `ROCM_CLI_VLLM_READY_TIMEOUT_SECS` / `ROCM_CLI_AIM_READY_TIMEOUT_SECS` allow longer); see `{}` for its output", + service_hints::service_logs_hint(service_id) ); // A plain `serve` adds the memory guidance for a vLLM launch that ran out of // VRAM; the wait's failure is the same launch, so it gets the same note. @@ -184,10 +186,27 @@ pub(super) fn record_wait_outcome( } }, )?; - *held = match written { - Some(latest) => latest, - None if paths.service_manifest_path(&held.service_id).exists() => { - let settled = ManagedServiceRecord::read_manifest(paths, &held.service_id)?; + // Read once, not after an `exists()` check: a removal landing between the + // two would surface as a bare read error instead of the "removed" failure. + let settled = match written { + Some(latest) => { + *held = latest; + return Ok(()); + } + None => match ManagedServiceRecord::read_manifest(paths, &held.service_id) { + Ok(settled) => Some(settled), + Err(error) + if error + .downcast_ref::() + .is_some() => + { + None + } + Err(error) => return Err(error), + }, + }; + match settled { + Some(settled) => { *held = settled; // A stop leaves its marker; anything else that settled the record // (the liveness refresh demoting a dead engine, or the engine's own @@ -204,10 +223,10 @@ pub(super) fn record_wait_outcome( } bail!( "managed service `{}` exited while it was starting (its record now reads `{}`); \ - see `rocm logs --service {}` for its output", + see `{}` for its output", held.service_id, held.status, - held.service_id + service_hints::service_logs_hint(&held.service_id) ); } None => bail!( @@ -215,8 +234,7 @@ pub(super) fn record_wait_outcome( no record of it left to report", held.service_id ), - }; - Ok(()) + } } /// Test-only fault-injection switch, off (`const false`) in shipped binaries: @@ -615,8 +633,14 @@ mod tests { message.contains("exited before it became ready"), "{message}" ); + // The hinted command parses with these flags: + // `advised_commands::every_advised_command_parses` reads the literal in + // `service_hints::service_logs_hint`. assert!( - message.contains(&format!("rocm logs --service {id}")), + message.contains(&format!( + "see `{}` for its output", + service_hints::service_logs_hint(&id) + )), "{message}" ); assert_eq!(status(), "stopped"); @@ -723,8 +747,14 @@ mod tests { "{message}" ); assert!(!message.contains("was stopped"), "{message}"); + // The hinted command parses with these flags: + // `advised_commands::every_advised_command_parses` reads the literal + // in `service_hints::service_logs_hint`. assert!( - message.contains(&format!("rocm logs --service {}", held.service_id)), + message.contains(&format!( + "see `{}` for its output", + service_hints::service_logs_hint(&held.service_id) + )), "{message}" ); assert_eq!( diff --git a/apps/rocm/src/service_launch/spawn.rs b/apps/rocm/src/service_launch/spawn.rs index 589a676d63..17d50259b0 100644 --- a/apps/rocm/src/service_launch/spawn.rs +++ b/apps/rocm/src/service_launch/spawn.rs @@ -652,6 +652,16 @@ fn launch_managed_engine_child_locked( // the launch lock, so it can settle the claim while the child is spawned. // Writing the held copy back would revive that service as `running`. // + // Only the fields this launch learned after its claim are applied: the engine + // and supervisor identities and the `running` status. Everything else is + // taken from the record on disk, which is the claim unless somebody wrote + // since — and a write that left the claim this launch's, such as the stop + // marker a stop records before its `stopped`, must survive the publish. So + // a field the launch sets on `record` after the claim write has to be added + // here, or it is dropped when `record` is replaced by what was published; + // `the_publish_applies_the_launch_s_own_fields_over_the_record_on_disk` + // pins which fields those are. + // // On any error here cleanup is the caller's job: `record.engine_pid` (set // above) is enough for `abandon_failed_managed_launch` to terminate this // child, which nothing on record names when the claim was taken away. @@ -1478,19 +1488,27 @@ mod tests { Ok(()) } - #[test] + /// A launch's claim written to disk, with the engine answer and port lease + /// `launch_managed_engine_child_locked` takes, for tests driving it through + /// the injected spawn. #[cfg(not(windows))] - fn a_stop_settled_while_the_engine_spawns_is_kept_and_the_engine_terminated() -> Result<()> { - let (root, paths) = test_paths("launch-stopped-during-spawn"); + fn claimed_launch( + paths: &AppPaths, + service_id: &str, + ) -> Result<( + ManagedServiceRecord, + ResolveModelResponse, + rocm_core::LoopbackPortLease, + )> { paths.ensure()?; fs::create_dir_all(paths.services_dir())?; let port = std::net::TcpListener::bind(("127.0.0.1", 0))? .local_addr()? .port(); let lease = rocm_core::lease_loopback_port(port)?; - let mut record = ManagedServiceRecord::new( - &paths, - "lemonade-stopped-during-spawn", + let record = ManagedServiceRecord::new( + paths, + service_id, "lemonade", "qwen", "qwen-canonical", @@ -1519,6 +1537,93 @@ mod tests { engine_recipe: None, warnings: Vec::new(), }; + Ok((record, resolve, lease)) + } + + /// The sleeper child the injected spawn starts in place of an engine. + #[cfg(not(windows))] + fn spawn_sleeper() -> io::Result { + ProcessCommand::new(std::env::current_exe().expect("test binary")) + .arg(sleeper_test_name()) + .arg("--exact") + .arg("--ignored") + .env("ROCM_TEST_SLEEPER_CHILD", "1") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + } + + /// The publish writes the launch's own fields onto the record as it is on + /// disk, not the copy the launch holds: a write made since the claim that + /// left it the launch's (here the stop marker a stop records before its + /// `stopped`) survives, and the published record differs from the claim in + /// exactly the engine and supervisor identities and the status. The held + /// copy is replaced by what was published, so the two agree. + #[test] + #[cfg(not(windows))] + fn the_publish_applies_the_launch_s_own_fields_over_the_record_on_disk() -> Result<()> { + let (root, paths) = test_paths("launch-publish-fields"); + let (mut record, resolve, lease) = claimed_launch(&paths, "lemonade-publish-fields")?; + let manifest_paths = paths.clone(); + let service_id = record.service_id.clone(); + let spawn_override: &dyn Fn(&mut ProcessCommand) -> io::Result = + &|_command| { + let mut marked = ManagedServiceRecord::read_manifest(&manifest_paths, &service_id) + .expect("claim on disk"); + marked.stop_requested_unix_ms = Some(4_243); + marked.write().expect("write the stop marker"); + spawn_sleeper() + }; + let claim = serde_json::to_value(&record)?; + + let result = launch_managed_engine_child_locked( + &paths, + &mut record, + lease, + "lemonade", + "lemonade-publish-fields", + &resolve, + "127.0.0.1", + &resolve.device_policy, + &[], + None, + None, + None, + Some(spawn_override), + ); + if let Some(pid) = record.engine_pid { + let _ = rocm_core::terminate_process(pid); + } + let on_disk = ManagedServiceRecord::read_manifest(&paths, &record.service_id); + let _ = fs::remove_dir_all(root); + let child_pid = result?; + let on_disk = serde_json::to_value(on_disk?)?; + + let mut expected = claim; + expected["stop_requested_unix_ms"] = serde_json::json!(4_243); + expected["engine_pid"] = serde_json::json!(child_pid); + expected["supervisor_pid"] = serde_json::json!(child_pid); + expected["engine_start_ticks"] = on_disk["engine_start_ticks"].clone(); + expected["supervisor_start_ticks"] = on_disk["supervisor_start_ticks"].clone(); + expected["status"] = serde_json::json!("running"); + assert_eq!( + on_disk, expected, + "the publish changed a field it does not own" + ); + assert_eq!( + serde_json::to_value(&record)?, + on_disk, + "the held copy disagrees" + ); + Ok(()) + } + + #[test] + #[cfg(not(windows))] + fn a_stop_settled_while_the_engine_spawns_is_kept_and_the_engine_terminated() -> Result<()> { + let (root, paths) = test_paths("launch-stopped-during-spawn"); + let (mut record, resolve, lease) = claimed_launch(&paths, "lemonade-stopped-during-spawn")?; let manifest_paths = paths.clone(); let service_id = record.service_id.clone(); let spawn_override: &dyn Fn(&mut ProcessCommand) -> io::Result = diff --git a/docs/architecture.md b/docs/architecture.md index 63a6778f76..7dd8ed93aa 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -119,7 +119,7 @@ The `rocm` binary: the clap command tree, a handler module for most commands, an | [`service_launch/attach.rs`](../apps/rocm/src/service_launch/attach.rs) | The attached serve path (`--verbose`/`--foreground`): spawn the managed child and stream its log in the terminal until the user detaches or stops it. | relocation | | [`service_launch/endpoint_auth.rs`](../apps/rocm/src/service_launch/endpoint_auth.rs) | Bind-host validation and the endpoint API key checks that gate a public bind (`validate_bind_host`, `resolve_endpoint_auth`), including on a respawn. | relocation | | [`service_launch/helper_autostart.rs`](../apps/rocm/src/service_launch/helper_autostart.rs) | Starting the background automation helper when a watcher needs it, with the autostart claim that keeps concurrent callers from spawning two. | relocation | -| [`service_launch/launch_transaction.rs`](../apps/rocm/src/service_launch/launch_transaction.rs) | The locked launch decision: port allocation against live records and the OS, failed-launch cleanup, and whether a live service may answer a `serve` request. The cleanup terminates a spawned child through the same `rocm_core::terminate_recorded_pid_within` routine a stop uses, so on Windows its descendants are taken down too. A spawned child whose start token was not captured, on a host that records tokens (Linux, and Windows, which records the process creation time), is not signalled (`launch_cleanup_refusal`): it was gone before it could be captured and its pid may belong to another process. Nor is an engine the startup check saw die: its pid is free, but its record and log are kept. `update_managed_record_on_disk` is the guarded read-modify-write every launch write after the claim goes through (the `running` record, the cleanup's `stopped` mark, the readiness verdict), applied only while `launch_still_owns` accepts the record on disk, so a `rocm services stop` or `remove`, which do not take the launch lock, is not overwritten or undone. | relocation | +| [`service_launch/launch_transaction.rs`](../apps/rocm/src/service_launch/launch_transaction.rs) | The locked launch decision: port allocation against live records and the OS, failed-launch cleanup, and whether a live service may answer a `serve` request. The cleanup terminates a spawned child through the same `rocm_core::terminate_recorded_pid_within` routine a stop uses, so on Windows its descendants are taken down too. A spawned child whose start token was not captured, on a host that records tokens (Linux, and Windows, which records the process creation time), is not signalled (`launch_cleanup_refusal`): it was gone before it could be captured and its pid may belong to another process. Nor is an engine the startup check saw die: its pid is free, but its record and log are kept. `update_managed_record_on_disk` is the guarded read-modify-write every launch write after the claim goes through (the `running` record, the cleanup's `stopped` mark, the readiness verdict), applied only while `launch_still_owns` (or, for the readiness verdict, the record still reading as live) accepts the record on disk, so a `rocm services stop` or `remove`, which do not take the launch lock, is not overwritten or undone. The cleanup clears the endpoint key only when its `stopped` mark applied or the record is gone, so a restart that took the record over keeps its key. | relocation | | [`service_launch/readiness.rs`](../apps/rocm/src/service_launch/readiness.rs) | Waiting on a launched or live managed service: the readiness budget, process liveness, how a wait ends, and the OOM fault-injection seam for end-to-end tests. A record a stop settled during the wait fails the serve saying it was stopped while starting, one settled without a stop marker fails it saying the server exited, and a record removed during it fails the serve rather than being re-created. | relocation | | [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs) | Spawning the detached managed engine child both serve paths share: process and stdio plumbing and the serialized spawn transaction (`spawn_managed_engine_child`). The startup check (`fail_if_engine_died_at_startup`, settle window `MANAGED_ENGINE_STARTUP_SETTLE`) fails a launch whose engine exited inside the window with the typed `EngineDiedAtStartup`; on Windows it goes through `adopt_watched_engine_spawn`, which records the engine's pid only after the check passes, and reads the exit code through the process handle (`rocm_core::spawn_detached_no_inherit_watching_startup`) rather than re-checking a pid Windows may have reused. `scripted_managed_engine_startup_failure` is the `e2e-test-hooks` switch that scripts that death. | relocation | | [`service_launch/start.rs`](../apps/rocm/src/service_launch/start.rs) | The background launch path (`start_managed_service`) and the plain line-by-line rendering of its launch report. | relocation | diff --git a/docs/manual-testing.md b/docs/manual-testing.md index b85330c3de..dc3adc01c3 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -662,7 +662,7 @@ the scripted death, which makes the spawned engine reject its arguments: ```powershell cargo build -p rocm --features e2e-test-hooks -$env:ROCM_E2E_MANAGED_ENGINE_STARTUP_FAILURE = "1" # bash: export ROCM_E2E_MANAGED_ENGINE_STARTUP_FAILURE=1 +$env:ROCM_E2E_MANAGED_ENGINE_STARTUP_FAILURE = "1" ./target/debug/rocm serve qwen --engine lemonade --managed ./target/debug/rocm services list --all ``` @@ -688,7 +688,11 @@ To see the stop-during-start case, run `rocm serve --managed` in one terminal and, while it is still waiting for the server, `rocm services stop --yes` from another. The serve exits non-zero with an error saying the managed service "was stopped while it was starting", and -`rocm services list --all` shows the service as `stopped`. +`rocm services list --all` shows the service as `stopped`. This is the usual +result, not a certain one: the launch reads the record and writes its readiness +verdict as two separate steps, and a stop takes no lock on the record, so a stop +that lands between the two is overwritten by the verdict and the serve reports +the server as it found it. #### Windows host check diff --git a/docs/testing.md b/docs/testing.md index 4e993bdee0..72a504ea15 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2021,10 +2021,14 @@ Managed-launch startup-death and record-write focused tests: ```bash cargo test -p rocm --bin rocm service_launch::spawn::tests::a_child_that_exited_during_the_startup_window cargo test -p rocm --bin rocm service_launch::launch_transaction::tests::an_engine_seen_dead_at_startup_is_never_signalled +cargo test -p rocm --bin rocm service_launch::launch_transaction::tests::an_engine_seen_dead_at_startup_is_never_handed +cargo test -p rocm --bin rocm service_launch::launch_transaction::tests::abandoning_a_launch_keeps_the_key cargo test -p rocm --bin rocm service_launch::launch_transaction::tests::abandoning_a_launch_does_not_recreate_a_removed_record cargo test -p rocm --bin rocm service_launch::spawn::tests::a_stop_settled_while_the_engine_spawns cargo test -p rocm --bin rocm service_launch::readiness::tests::a_stop_settled_during_the_wait -cargo test -p rocm --bin rocm service_launch::readiness::tests::record_wait_outcome +cargo test -p rocm --bin rocm service_launch::readiness::tests::record_wait_outcome_keeps_what_changed +cargo test -p rocm --bin rocm service_launch::readiness::tests::record_wait_outcome_does_not_recreate +cargo test -p rocm --bin rocm service_launch::spawn::tests::the_publish_applies cargo test -p rocm --bin rocm service_launch::launch_transaction::tests::the_launch_does_not_publish_over cargo test -p rocm --bin rocm service_launch::launch_transaction::tests::an_engine_seen_dead_before cargo test -p rocm --bin rocm service_launch::spawn::tests::an_engine_that_dies_at_startup_keeps @@ -2044,7 +2048,12 @@ stop marker, and a `rocm services remove` is not undone by re-creating the recor A serve whose server is stopped during its readiness wait fails (non-zero exit) saying the service was stopped while it was starting, instead of reporting a launch; one whose server exited on its own during the wait (a record settled without a stop marker) fails -saying it exited and naming `rocm logs --service `. +saying it exited and naming `rocm logs --service `. Each of those writes reads +the record and then writes it, with no lock against a stop, so a stop landing +between the two is still overwritten: the tests pin the narrowed race, not a +closed one. The publish applies only the launch's own fields (engine and +supervisor identities, `running`) over the record on disk, and a failed launch's +cleanup leaves the endpoint key of a record a restart took over. The `watched_detached_spawn_*` tests run on the required Windows lane. `serve-55` (`@id:serve-managed-engine-dies-at-startup`) is the black-box half: an From 2169ddb45990870c09ad22cff1fd652fa3afba12 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Fri, 13 Nov 2026 12:34:15 +0200 Subject: [PATCH 7/9] test(e2e): check the retry's record in serve-55 The retry step checked only the exit code and the message. It now also reads the records on disk and expects two `stopped` claims that name no engine. The comments about start-time tokens now say that only Windows captures none for a dead engine. Signed-off-by: Roman Inflianskas --- .../features/model_serving.feature | 7 ++- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 55 ++++++++++++------- 2 files changed, 38 insertions(+), 24 deletions(-) diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index d9cc965053..4162f8b780 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -865,9 +865,10 @@ Feature: Model serving # terminal for it to report to: the child's own service log is the only account # of why. The launch fails naming that log, and leaves its record `stopped` # (not `failed`, which `rocmd`'s recovery would respawn) without naming the - # dead child: its pid is free, and no start-time token was captured for it - # that would stop a later cleanup from signalling whatever process reuses it. Serving the same - # model again then reaches the engine launch again. The engine is scripted to + # dead child: its pid is free, and a later stop or cleanup must not signal + # whatever process reuses it (on Windows no start-time token was captured for + # the dead engine that could tell the two apart). Serving the same model + # again then reaches the engine launch again. The engine is scripted to # die at startup in test builds (`rocm/e2e-test-hooks`), which also waives the # no-GPU pre-flight and engine preparation so this reaches a real spawn without # GPU hardware or a runtime download — the death itself is real, only its diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 2180147cde..07452ab49b 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -3547,32 +3547,43 @@ async fn assert_startup_death_names_log(world: &mut E2eWorld) { // detached spawn does not redirect, and the scripted child never receives // `--log`, so the log stays empty and only the no-tail branch runs. Naming // the path is the claim both branches share, and it is the one pinned here. - let services = world - .isolated_root - .as_ref() - .expect("scenario has no isolated root") - .path() - .join("data") - .join("services"); - let records: Vec = std::fs::read_dir(&services) - .unwrap_or_else(|e| panic!("no services dir at {}: {e}", services.display())) - .filter_map(|entry| std::fs::read_to_string(entry.ok()?.path()).ok()) - .filter_map(|json| serde_json::from_str::(&json).ok()) - .filter(|record| record["engine"] == "lemonade") - .collect(); + let records = lemonade_service_records(world); let log_paths: Vec<&str> = records .iter() .filter_map(|record| record["log_path"].as_str()) .collect(); assert!( !log_paths.is_empty(), - "the failed launch left no lemonade service record in {}", - services.display() + "the failed launch left no lemonade service record" ); assert!( log_paths.iter().any(|path| output.contains(path)), "expected the failure to name the engine's own service log {log_paths:?}:\n{output}" ); + assert_failed_launches_left_their_claims(&records, 1); +} + +/// Every lemonade service record in the scenario's isolated data folder, read +/// raw from disk. +fn lemonade_service_records(world: &E2eWorld) -> Vec { + let services = world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .join("data") + .join("services"); + std::fs::read_dir(&services) + .unwrap_or_else(|e| panic!("no services dir at {}: {e}", services.display())) + .filter_map(|entry| std::fs::read_to_string(entry.ok()?.path()).ok()) + .filter_map(|json| serde_json::from_str::(&json).ok()) + .filter(|record| record["engine"] == "lemonade") + .collect() +} + +/// `launches` failed launches of an engine that died at startup each left one +/// record: `stopped`, and still the claim as it was written. +fn assert_failed_launches_left_their_claims(records: &[serde_json::Value], launches: usize) { // Read raw from disk, not through `rocm services`, whose liveness refresh // would rewrite the very fields pinned here. `stopped`, not `failed`: // `rocmd`'s recovery restarts a managed `failed` record that carries no stop @@ -3583,15 +3594,15 @@ async fn assert_startup_death_names_log(world: &mut E2eWorld) { .collect(); assert_eq!( statuses, - ["stopped"], - "the failed launch must leave its record `stopped`, not claiming the engine \ + vec!["stopped"; launches], + "each failed launch must leave its record `stopped`, not claiming the engine \ and model" ); // The startup check saw the child die, so the record never names it: the // pid is free, and a later stop or cleanup must not signal whatever process - // reuses it. No start-time token was captured for the dead engine, so - // nothing else would tell that process from it. Both platforms leave the - // claim as it was written, before any child was put on record. + // reuses it. On Windows no start-time token was captured for the dead + // engine, so nothing else would tell that process from it. Both platforms + // leave the claim as it was written, before any child was put on record. let pids: Vec<(serde_json::Value, serde_json::Value)> = records .iter() .map(|record| { @@ -3603,7 +3614,7 @@ async fn assert_startup_death_names_log(world: &mut E2eWorld) { .collect(); assert_eq!( pids, - [(serde_json::json!(0), serde_json::Value::Null)], + vec![(serde_json::json!(0), serde_json::Value::Null); launches], "a child the startup check saw die must not be named by the record" ); } @@ -3636,4 +3647,6 @@ async fn assert_retry_reaches_the_engine_launch(world: &mut E2eWorld) { "the retry must reach the engine launch again rather than being turned \ away by the previous failed launch:\n{output}" ); + // The retry's own cleanup left its record the same way, beside the first. + assert_failed_launches_left_their_claims(&lemonade_service_records(world), 2); } From 74c468fb70346da94f3757e8e1f1d55a6cd3287e Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 17 Nov 2026 17:58:41 +0200 Subject: [PATCH 8/9] fix(serve): keep the key of an unreadable record and test the publish on Windows update_managed_record_on_disk now says what it found: Written, NotOwned (with the record it refused) or Gone. The failed-launch cleanup used to read an error the same as "no longer ours" and cleared the endpoint key of a record it could not read. It now keeps the key, leaves the record alone and records an audit event. Its second exists() check is gone too. record_wait_outcome builds its message from the record the guarded write refused, without reading it a second time, so nothing can change between the decision and the report. The step after the spawn is now publish_spawned_engine, shared by both platforms. A new test drives it with a real child on every lane, the Windows gate included: a stop settled meanwhile fails the publish, keeps its stopped and stop marker, and the cleanup ends the child. The docs no longer read the guarded write as certain: a stop that lands between its read and its write is still overwritten, since nothing that writes a service record locks it. Signed-off-by: Roman Inflianskas --- .../src/service_launch/launch_transaction.rs | 102 ++++++++++++--- apps/rocm/src/service_launch/readiness.rs | 34 ++--- apps/rocm/src/service_launch/spawn.rs | 119 +++++++++++++++++- docs/architecture.md | 6 +- docs/testing.md | 7 +- 5 files changed, 218 insertions(+), 50 deletions(-) diff --git a/apps/rocm/src/service_launch/launch_transaction.rs b/apps/rocm/src/service_launch/launch_transaction.rs index 3c5d64e65c..abf789fec0 100644 --- a/apps/rocm/src/service_launch/launch_transaction.rs +++ b/apps/rocm/src/service_launch/launch_transaction.rs @@ -192,16 +192,32 @@ fn abandon_failed_managed_launch_with( if let Some(pid) = spawned_pid.filter(|_| !child_known_dead) { terminate_abandoned_launch_child(paths, record, pid, terminate); } - let marked = update_managed_record_on_disk( + match update_managed_record_on_disk( paths, &record.service_id, |current| launch_still_owns(current, record), ManagedServiceRecord::mark_launch_abandoned, - ); - let owned_by_someone_else = - matches!(marked, Ok(None)) && paths.service_manifest_path(&record.service_id).exists(); - if !owned_by_someone_else { - endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); + ) { + Ok(RecordUpdate::Written(_) | RecordUpdate::Gone) => { + endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); + } + // A restart that took the record over stores its key under the same + // service id, and a stop that settled it does its own key cleanup. + Ok(RecordUpdate::NotOwned(_)) => {} + // The record is there but could not be read, so whose it is now is + // unknown: keep the key rather than guess, and leave a trace, since + // cleanup must not fail the launch a second time. + Err(error) => record_cli_audit_event( + paths, + "service", + "managed_service_launch_cleanup", + "warn", + format!( + "could not mark the failed launch stopped; its record and endpoint key \ + were left as they are: {error:#}" + ), + Some(&record.service_id), + ), } return; } @@ -276,17 +292,18 @@ fn terminate_abandoned_launch_child( /// bringing back as live a service the stop just reported stopped, and with it /// the marker that keeps `rocmd`'s recovery off it. /// -/// Returns `Ok(None)`, writing nothing, when the record is gone or -/// `still_owned` refuses the record as it is now: whoever wrote since holds a -/// truer view. This narrows the lost-update window; it does not close it, since -/// the read and the write are separate steps and a stop takes no lock on the -/// manifest. +/// Writes nothing when the record is gone or `still_owned` refuses the record +/// as it is now — whoever wrote since holds a truer view — and says which, so a +/// caller can tell the two apart without reading the record again. This narrows +/// the lost-update window; it does not close it, since the read and the write +/// are separate steps and nothing that writes a service record (a stop, a +/// removal, `rocmd`, the liveness refresh) takes a lock on it. pub(super) fn update_managed_record_on_disk( paths: &AppPaths, service_id: &str, still_owned: impl FnOnce(&ManagedServiceRecord) -> bool, update: impl FnOnce(&mut ManagedServiceRecord), -) -> Result> { +) -> Result { // One read, not an `exists()` check before it: a removal landing between // the two would turn "the record is gone" into a read error. let mut current = match ManagedServiceRecord::read_manifest(paths, service_id) { @@ -296,16 +313,28 @@ pub(super) fn update_managed_record_on_disk( .downcast_ref::() .is_some() => { - return Ok(None); + return Ok(RecordUpdate::Gone); } Err(error) => return Err(error), }; if !still_owned(¤t) { - return Ok(None); + return Ok(RecordUpdate::NotOwned(Box::new(current))); } update(&mut current); current.write()?; - Ok(Some(current)) + Ok(RecordUpdate::Written(Box::new(current))) +} + +/// What [`update_managed_record_on_disk`] did with the record on disk. +#[derive(Debug)] +pub(super) enum RecordUpdate { + /// The change was applied; this is the record as written. + Written(Box), + /// The record is there but `still_owned` refused it; this is the record as + /// read, left unchanged. + NotOwned(Box), + /// There is no record for the service. + Gone, } /// Whether the launch that wrote `launched` may still write `current`, the same @@ -1314,8 +1343,8 @@ mod tests { let after = ManagedServiceRecord::read_manifest(&paths, &launched.service_id)?; let _ = fs::remove_dir_all(root); assert!( - written.is_none(), - "the launch wrote over a restart's record" + matches!(written, RecordUpdate::NotOwned(_)), + "the launch wrote over a restart's record: {written:?}" ); assert_eq!(after.status, "starting"); assert_eq!(after.supervisor_pid, taken.supervisor_pid); @@ -1416,6 +1445,45 @@ mod tests { Ok(()) } + /// A record that is there but cannot be read says nothing about whose it is + /// now, so the cleanup keeps the endpoint key and leaves the record's bytes + /// alone, and audits that it did so instead of failing the launch twice. + #[test] + fn abandoning_a_launch_over_an_unreadable_record_keeps_the_key_and_says_so() -> Result<()> { + let (root, paths) = test_paths("abandon-unreadable-record"); + paths.ensure()?; + let launched = write_live_service_record( + &paths, + "lemonade-abandon-unreadable", + "abandon-unreadable-model", + 11_488, + "starting", + )?; + endpoint_keys::store_endpoint_api_key(&paths, &launched.service_id, "unknown-owner-key")?; + fs::write(&launched.manifest_path, b"{ not a service record")?; + + abandon_failed_managed_launch(&paths, &launched, true); + + let key = endpoint_keys::endpoint_api_key(&paths, &launched.service_id); + let bytes = fs::read(&launched.manifest_path)?; + let audit = fs::read_to_string(paths.audit_events_path()).unwrap_or_default(); + let _ = fs::remove_dir_all(root); + assert_eq!( + key.as_deref(), + Some("unknown-owner-key"), + "the key was cleared" + ); + assert_eq!( + bytes, b"{ not a service record", + "the unreadable record was rewritten" + ); + assert!( + audit.contains("could not mark the failed launch stopped"), + "the skipped cleanup left no trace: {audit}" + ); + Ok(()) + } + #[test] fn an_engine_seen_dead_before_it_was_recorded_keeps_its_record_and_log() -> Result<()> { // The Windows shape: the startup check saw the engine exit before diff --git a/apps/rocm/src/service_launch/readiness.rs b/apps/rocm/src/service_launch/readiness.rs index d7018aa609..8db8908a90 100644 --- a/apps/rocm/src/service_launch/readiness.rs +++ b/apps/rocm/src/service_launch/readiness.rs @@ -14,7 +14,7 @@ // would be unused in some build, and a gate on it would have to be kept in step // with every use site. Keep the full paths. The tests module imports `fs` for // itself because its tests use it on every platform. -use super::launch_transaction::update_managed_record_on_disk; +use super::launch_transaction::{RecordUpdate, update_managed_record_on_disk}; use super::spawn::ManagedLaunchReport; #[cfg(feature = "e2e-oom-fault-injection")] use crate::device_policy_name; @@ -187,28 +187,16 @@ pub(super) fn record_wait_outcome( } }, )?; - // Read once, not after an `exists()` check: a removal landing between the - // two would surface as a bare read error instead of the "removed" failure. - let settled = match written { - Some(latest) => { - *held = latest; - return Ok(()); + // The record the guarded write refused is the one the message describes: + // it is not read a second time, so nothing can change between the decision + // and what is reported. + match written { + RecordUpdate::Written(latest) => { + *held = *latest; + Ok(()) } - None => match ManagedServiceRecord::read_manifest(paths, &held.service_id) { - Ok(settled) => Some(settled), - Err(error) - if error - .downcast_ref::() - .is_some() => - { - None - } - Err(error) => return Err(error), - }, - }; - match settled { - Some(settled) => { - *held = settled; + RecordUpdate::NotOwned(settled) => { + *held = *settled; // A stop leaves its marker; anything else that settled the record // (the liveness refresh demoting a dead engine, or the engine's own // `failed`) means the server exited on its own, and the user needs @@ -230,7 +218,7 @@ pub(super) fn record_wait_outcome( service_hints::service_logs_hint(&held.service_id) ); } - None => bail!( + RecordUpdate::Gone => bail!( "managed service `{}` was removed while `rocm serve` waited for it, so there is \ no record of it left to report", held.service_id diff --git a/apps/rocm/src/service_launch/spawn.rs b/apps/rocm/src/service_launch/spawn.rs index 17d50259b0..5fb48badae 100644 --- a/apps/rocm/src/service_launch/spawn.rs +++ b/apps/rocm/src/service_launch/spawn.rs @@ -8,9 +8,9 @@ use super::endpoint_auth::ensure_public_service_has_endpoint_key; use super::launch_transaction::{ - ManagedPortDecision, ManagedReuseRefusal, abandon_failed_managed_launch, launch_still_owns, - managed_service_reuse_refusal, no_usable_gpu_refusal, resolve_managed_port_in_transaction, - update_managed_record_on_disk, + ManagedPortDecision, ManagedReuseRefusal, RecordUpdate, abandon_failed_managed_launch, + launch_still_owns, managed_service_reuse_refusal, no_usable_gpu_refusal, + resolve_managed_port_in_transaction, update_managed_record_on_disk, }; use crate::endpoint_keys; use crate::engine_ipc::builtin_engine_serve_http_args; @@ -639,6 +639,23 @@ fn launch_managed_engine_child_locked( fail_if_engine_died_at_startup(startup_exit, &record.log_path)?; child_pid }; + publish_spawned_engine(paths, service_id, record, child_pid) +} + +/// Put a spawned engine on the launch's record as `running`, as the last step of +/// the launch, or fail it when a stop or removal has settled the claim since. +/// +/// A failure leaves `record.engine_pid` naming the child, which is what +/// `abandon_failed_managed_launch` terminates; that is how the failure's own +/// "the failed launch terminates it" comes true. Shared by both platforms and +/// kept apart from the spawn so a test can drive it with a real child on every +/// lane, Windows included. +fn publish_spawned_engine( + paths: &AppPaths, + service_id: &str, + record: &mut ManagedServiceRecord, + child_pid: u32, +) -> Result { // The child is both roles until the engine state file names a separate // server process. Each role gets its own token, captured while the child is // alive: a stop pairs every recorded PID with its own token, and the two @@ -678,14 +695,14 @@ fn launch_managed_engine_child_locked( }, ) .context("failed to publish the managed service record")?; - let Some(published) = published else { + let RecordUpdate::Written(published) = published else { bail!( "managed service `{service_id}` was stopped or removed while it was launching, so \ the engine it started (pid {child_pid}) is not recorded on it; the failed launch \ terminates it" ); }; - *record = published; + *record = *published; Ok(child_pid) } @@ -695,7 +712,6 @@ mod tests { use super::*; use crate::managed_service_status::load_managed_services; use crate::service_launch::launch_transaction::live_managed_services; - #[cfg(not(windows))] use crate::service_launch::test_support::sleeper_test_name; use crate::service_launch::test_support::write_live_service_record; use crate::tests::test_paths; @@ -1554,6 +1570,97 @@ mod tests { .spawn() } + /// The step after the spawn, on every platform and so on the Windows gate: + /// a stop that settled the claim while the engine spawned fails the publish, + /// keeps its `stopped` and stop marker, and the cleanup the failure promises + /// ("the failed launch terminates it") terminates the real child through the + /// same routine a stop uses — a tree kill on Windows. + #[test] + fn a_publish_refused_by_a_stop_leaves_the_stop_and_the_cleanup_ends_the_engine() -> Result<()> { + let (root, paths) = test_paths("publish-refused-by-stop"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-publish-refused", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11_489, + "managed", + 0, + None, + None, + None, + ); + record.write()?; + // `std::process` paths spelled out: this module imports `Command` and + // `Stdio` only off Windows, and this test runs on Windows too. + let mut child = std::process::Command::new(std::env::current_exe()?) + .arg("--exact") + .arg(sleeper_test_name()) + .arg("--ignored") + .arg("--test-threads=1") + .env("ROCM_TEST_SLEEPER_CHILD", "1") + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn()?; + let child_pid = child.id(); + // As both platforms do before the publish: the engine is on the held + // record with its token, so cleanup knows what to terminate. + record.record_engine_identity(child_pid); + // `rocm services stop` settles the claim meanwhile; it takes no launch lock. + let mut stopped = ManagedServiceRecord::read_manifest(&paths, &record.service_id)?; + stopped.stop_requested_unix_ms = Some(4_244); + stopped.status = "stopped".to_owned(); + stopped.write()?; + + let result = + publish_spawned_engine(&paths, &record.service_id.clone(), &mut record, child_pid); + let error = result.as_ref().err().map(|error| format!("{error:#}")); + abandon_failed_managed_launch(&paths, &record, false); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + let mut exited = false; + while std::time::Instant::now() < deadline { + if child.try_wait()?.is_some() { + exited = true; + break; + } + thread::sleep(Duration::from_millis(50)); + } + if !exited { + let _ = child.kill(); + let _ = child.wait(); + } + let after = ManagedServiceRecord::read_manifest(&paths, &record.service_id); + let _ = fs::remove_dir_all(root); + let after = after?; + let error = error.expect("a publish over a settled stop must fail"); + assert!( + error.contains("was stopped or removed while it was launching") + && error.contains(&format!("(pid {child_pid})")) + && error.contains("the failed launch terminates it"), + "{error}" + ); + assert!( + exited, + "the engine the failure says is terminated is still running" + ); + assert_eq!(after.status, "stopped", "the stop was overwritten"); + assert_eq!( + after.stop_requested_unix_ms, + Some(4_244), + "the stop marker was erased" + ); + assert_eq!( + after.engine_pid, None, + "the refused engine was put on record" + ); + Ok(()) + } + /// The publish writes the launch's own fields onto the record as it is on /// disk, not the copy the launch holds: a write made since the claim that /// left it the launch's (here the stop marker a stop records before its diff --git a/docs/architecture.md b/docs/architecture.md index 19345fd467..74ce52ec73 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -114,12 +114,12 @@ The `rocm` binary: the clap command tree, a handler module for most commands, an | [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs) | The pure transforms `serve()` applies to an engine's launch recipe hint: the catalog mapping, then `--tool-call-parser`, `--gpu-memory-utilization`, generation defaults and `--engine-env`. | relocation | | [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs) | The engine-agnostic `rocm serve` deployment summary: a status indicator while the server starts, an inference smoke test, and the endpoint and timing table. | domain | | [`service_control.rs`](../apps/rocm/src/service_control.rs) | Stopping and restarting a managed local server (`rocm services stop` and `restart`, runtime moves) and the internal sandbox-tool runner (`run_internal_sandbox_tool`). Its stop path, `stop_recorded_processes`, hands the recorded PIDs to the routine [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) shares with `rocmd`. | relocation | -| [`service_hints.rs`](../apps/rocm/src/service_hints.rs) | The shared builder of the hint text that tells a user how to stop, restart or remove a service with `--yes`, plus the forced-removal hint (`service_forced_remove_hint`, `rocm services remove --force --yes`) and the command word, verb and past tense for a service tool. | flat | +| [`service_hints.rs`](../apps/rocm/src/service_hints.rs) | The shared builder of the hint text that tells a user how to stop, restart or remove a service with `--yes`, plus the forced-removal hint (`service_forced_remove_hint`, `rocm services remove --force --yes`), the `rocm logs --service ` hints with and without `--follow` (`service_logs_hint`, `service_logs_follow_hint`), and the command word, verb and past tense for a service tool. | flat | | [`service_launch.rs`](../apps/rocm/src/service_launch.rs) | Front of the `service_launch/` children: their declarations and the `pub(crate) use` re-exports other modules call, with no code of its own. | relocation | | [`service_launch/attach.rs`](../apps/rocm/src/service_launch/attach.rs) | The attached serve path (`--verbose`/`--foreground`): spawn the managed child and stream its log in the terminal until the user detaches or stops it. | relocation | | [`service_launch/endpoint_auth.rs`](../apps/rocm/src/service_launch/endpoint_auth.rs) | Bind-host validation and the endpoint API key checks that gate a public bind (`validate_bind_host`, `resolve_endpoint_auth`), including on a respawn. | relocation | | [`service_launch/helper_autostart.rs`](../apps/rocm/src/service_launch/helper_autostart.rs) | Starting the background automation helper when a watcher needs it, with the autostart claim that keeps concurrent callers from spawning two. | relocation | -| [`service_launch/launch_transaction.rs`](../apps/rocm/src/service_launch/launch_transaction.rs) | The locked launch decision: port allocation against live records and the OS, failed-launch cleanup, and whether a live service may answer a `serve` request. The cleanup terminates a spawned child through the same `rocm_core::terminate_recorded_pid_within` routine a stop uses, so on Windows its descendants are taken down too. A spawned child whose start token was not captured, on a host that records tokens (Linux, and Windows, which records the process creation time), is not signalled (`launch_cleanup_refusal`): it was gone before it could be captured and its pid may belong to another process. Nor is an engine the startup check saw die: its pid is free, but its record and log are kept. `update_managed_record_on_disk` is the guarded read-modify-write every launch write after the claim goes through (the `running` record, the cleanup's `stopped` mark, the readiness verdict), applied only while `launch_still_owns` (or, for the readiness verdict, the record still reading as live) accepts the record on disk, so a `rocm services stop` or `remove`, which do not take the launch lock, is not overwritten or undone. The cleanup clears the endpoint key only when its `stopped` mark applied or the record is gone, so a restart that took the record over keeps its key. | relocation | +| [`service_launch/launch_transaction.rs`](../apps/rocm/src/service_launch/launch_transaction.rs) | The locked launch decision: port allocation against live records and the OS, failed-launch cleanup, and whether a live service may answer a `serve` request. The cleanup terminates a spawned child through the same `rocm_core::terminate_recorded_pid_within` routine a stop uses, so on Windows its descendants are taken down too. A spawned child whose start token was not captured, on a host that records tokens (Linux, and Windows, which records the process creation time), is not signalled (`launch_cleanup_refusal`): it was gone before it could be captured and its pid may belong to another process. Nor is an engine the startup check saw die: its pid is free, but its record and log are kept. `update_managed_record_on_disk` is the guarded read-modify-write every launch write after the claim goes through (the `running` record, the cleanup's `stopped` mark, the readiness verdict), applied only while `launch_still_owns` (or, for the readiness verdict, the record still reading as live) accepts the record on disk, so a `rocm services stop` or `remove`, which do not take the launch lock, is not overwritten or undone unless it lands between that read and the write (nothing that writes a service record locks it). It reports whether it wrote, found the record no longer the launch's, or found it gone (`RecordUpdate`). The cleanup clears the endpoint key only when its `stopped` mark applied or the record is gone, so a restart that took the record over keeps its key. | relocation | | [`service_launch/readiness.rs`](../apps/rocm/src/service_launch/readiness.rs) | Waiting on a launched or live managed service: the readiness budget, process liveness, how a wait ends, and the fault-injection seam (an OOM launch, or a ready launch) for end-to-end tests. A record a stop settled during the wait fails the serve saying it was stopped while starting, one settled without a stop marker fails it saying the server exited, and a record removed during it fails the serve rather than being re-created. | relocation | | [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs) | Spawning the detached managed engine child both serve paths share: process and stdio plumbing and the serialized spawn transaction (`spawn_managed_engine_child`). The startup check (`fail_if_engine_died_at_startup`, settle window `MANAGED_ENGINE_STARTUP_SETTLE`) fails a launch whose engine exited inside the window with the typed `EngineDiedAtStartup`; on Windows it goes through `adopt_watched_engine_spawn`, which records the engine's pid only after the check passes, and reads the exit code through the process handle (`rocm_core::spawn_detached_no_inherit_watching_startup`) rather than re-checking a pid Windows may have reused. `scripted_managed_engine_startup_failure` is the `e2e-test-hooks` switch that scripts that death. | relocation | | [`service_launch/start.rs`](../apps/rocm/src/service_launch/start.rs) | The background launch path (`start_managed_service`) and the plain line-by-line rendering of its launch report. | relocation | @@ -194,7 +194,7 @@ Boundaries: **Shared sources and contracts** -- [`service_hints.rs`](../apps/rocm/src/service_hints.rs) is the shared builder of the `rocm services stop|restart|remove --yes` hint text, so the callers that use it cannot drift apart. Those callers are [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs), [`serve_port.rs`](../apps/rocm/src/serve_port.rs), [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs), [`service_control.rs`](../apps/rocm/src/service_control.rs), [`service_records.rs`](../apps/rocm/src/service_records.rs), [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs), [`services_text.rs`](../apps/rocm/src/services_text.rs) and, under `service_launch/`, [`attach.rs`](../apps/rocm/src/service_launch/attach.rs) and [`spawn.rs`](../apps/rocm/src/service_launch/spawn.rs); the parent [`service_launch.rs`](../apps/rocm/src/service_launch.rs) does not call it. +- [`service_hints.rs`](../apps/rocm/src/service_hints.rs) is the shared builder of the `rocm services stop|restart|remove --yes` and `rocm logs --service ` hint text, so the callers that use it cannot drift apart. Those callers are [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs), [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`serve_port.rs`](../apps/rocm/src/serve_port.rs), [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs), [`service_control.rs`](../apps/rocm/src/service_control.rs), [`service_records.rs`](../apps/rocm/src/service_records.rs), [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs), [`services_text.rs`](../apps/rocm/src/services_text.rs) and, under `service_launch/`, [`attach.rs`](../apps/rocm/src/service_launch/attach.rs), [`readiness.rs`](../apps/rocm/src/service_launch/readiness.rs) and [`spawn.rs`](../apps/rocm/src/service_launch/spawn.rs); the parent [`service_launch.rs`](../apps/rocm/src/service_launch.rs) does not call it. - `DEFAULT_ENGINE_MARKER` in [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs) is shared by both engine-inventory renderers and their legend line, so the rendered marker and the legend cannot drift apart. - [`shell_quote.rs`](../apps/rocm/src/shell_quote.rs) is the CLI's quoter; `rocm-dash-tui` keeps a copy (`quote_display_arg`) because it may not depend on `rocm-core`, and a property test here holds the two equal. - The local-model thinking rule [`providers/wire.rs`](../apps/rocm/src/providers/wire.rs) applies lives in `rocm-dash-core`'s [`chat_template.rs`](../crates/rocm-dash-core/src/chat_template.rs), because the dashboard's chat client must decide it identically. diff --git a/docs/testing.md b/docs/testing.md index 2c9b605590..b4fd687290 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2450,8 +2450,10 @@ cargo test -p rocm --bin rocm service_launch::spawn::tests::a_child_that_exited_ cargo test -p rocm --bin rocm service_launch::launch_transaction::tests::an_engine_seen_dead_at_startup_is_never_signalled cargo test -p rocm --bin rocm service_launch::launch_transaction::tests::an_engine_seen_dead_at_startup_is_never_handed cargo test -p rocm --bin rocm service_launch::launch_transaction::tests::abandoning_a_launch_keeps_the_key +cargo test -p rocm --bin rocm service_launch::launch_transaction::tests::abandoning_a_launch_over_an_unreadable cargo test -p rocm --bin rocm service_launch::launch_transaction::tests::abandoning_a_launch_does_not_recreate_a_removed_record cargo test -p rocm --bin rocm service_launch::spawn::tests::a_stop_settled_while_the_engine_spawns +cargo test -p rocm --bin rocm service_launch::spawn::tests::a_publish_refused_by_a_stop cargo test -p rocm --bin rocm service_launch::readiness::tests::a_stop_settled_during_the_wait cargo test -p rocm --bin rocm service_launch::readiness::tests::record_wait_outcome_keeps_what_changed cargo test -p rocm --bin rocm service_launch::readiness::tests::record_wait_outcome_does_not_recreate @@ -2480,7 +2482,10 @@ the record and then writes it, with no lock against a stop, so a stop landing between the two is still overwritten: the tests pin the narrowed race, not a closed one. The publish applies only the launch's own fields (engine and supervisor identities, `running`) over the record on disk, and a failed launch's -cleanup leaves the endpoint key of a record a restart took over. +cleanup leaves the endpoint key of a record a restart took over, or of one it +cannot read, which it audits. A publish refused by a stop, and the cleanup +terminating the engine it then names, run on the required Windows lane too +(`a_publish_refused_by_a_stop_*`, with a real child and a tree kill there). The `watched_detached_spawn_*` tests run on the required Windows lane. `serve-61` (`@id:serve-managed-engine-dies-at-startup`) is the black-box half: an From a4360d179bc0a5e4182f5416c2c6bb85aa793502 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 17 Nov 2026 17:58:41 +0200 Subject: [PATCH 9/9] fix(serve): quote the service id in every rocm logs hint The deployment summary's logs row and notes, and serve's "Waiting for ..." line, built `rocm logs --service ` by hand. They now take it from service_hints, with a new follow variant spelled out in full so the advised-command contract test still checks --follow. A property test covers both builders with hostile ids. Output for an ordinary id is unchanged. Signed-off-by: Roman Inflianskas --- apps/rocm/src/serve_cmd.rs | 3 ++- apps/rocm/src/serve_summary.rs | 10 +++++----- apps/rocm/src/service_hints.rs | 35 +++++++++++++++++++++++----------- 3 files changed, 31 insertions(+), 17 deletions(-) diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index d6b5c57a63..08b650d277 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -1181,7 +1181,8 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { spinner.borrow_mut().clear(); eprintln!( "Waiting for {waiting_on}; Ctrl-C stops waiting, the server keeps running. \ - Follow: rocm logs --service {waiting_on} --follow" + Follow: {}", + crate::service_hints::service_logs_follow_hint(waiting_on) ); if let Some(config) = wait_begin_client_config( waiting_on, diff --git a/apps/rocm/src/serve_summary.rs b/apps/rocm/src/serve_summary.rs index c30bcc654a..182b5e25e2 100644 --- a/apps/rocm/src/serve_summary.rs +++ b/apps/rocm/src/serve_summary.rs @@ -229,7 +229,7 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { ), ( "logs", - format!("rocm logs --service {}", summary.service_id), + crate::service_hints::service_logs_hint(&summary.service_id), ), ], ] @@ -266,8 +266,8 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { }; let _ = writeln!( out, - " note: {explanation} - follow `rocm logs --service {} --follow` or check `rocm services list`", - summary.service_id + " note: {explanation} - follow `{}` or check `rocm services list`", + crate::service_hints::service_logs_follow_hint(&summary.service_id) ); } if let (true, Some(reason)) = (smoke_failed, summary.smoke_failure.as_deref()) { @@ -276,8 +276,8 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { // empty, hedged: the reason may be rocm's own plumbing, not the engine. let _ = writeln!( out, - " note: the smoke test could not complete, so no metrics were measured ({reason}); the server may not be able to serve requests - check `rocm logs --service {id}`", - id = summary.service_id + " note: the smoke test could not complete, so no metrics were measured ({reason}); the server may not be able to serve requests - check `{logs}`", + logs = crate::service_hints::service_logs_hint(&summary.service_id) ); } for note in &summary.notes { diff --git a/apps/rocm/src/service_hints.rs b/apps/rocm/src/service_hints.rs index 00131232f0..90bbc975b7 100644 --- a/apps/rocm/src/service_hints.rs +++ b/apps/rocm/src/service_hints.rs @@ -2,7 +2,8 @@ // // SPDX-License-Identifier: MIT -//! The one place every `rocm services --yes` hint is built. +//! The one place every `rocm services --yes` hint is built, and +//! the `rocm logs --service ` hints beside them. //! //! `main.rs`, `remote/mod.rs`, and `serve_summary.rs` each used to format this //! string by hand, and it drifted: `serve_summary.rs`'s stop row printed the @@ -55,6 +56,14 @@ pub(crate) fn service_logs_hint(service_id: &str) -> String { format!("rocm logs --service {}", shell_quote(service_id)) } +/// The directly-runnable `rocm logs --service --follow` hint, which keeps +/// printing a local managed service's log as it grows. +pub(crate) fn service_logs_follow_hint(service_id: &str) -> String { + // Spelled out rather than built on `service_logs_hint`, so the advised-command + // contract test reads `--follow` too. + format!("rocm logs --service {} --follow", shell_quote(service_id)) +} + /// The directly-runnable `rocm services stop --yes` hint. pub(crate) fn service_stop_hint(service_id: &str) -> String { service_action_hint("stop", service_id) @@ -153,22 +162,26 @@ mod tests { } } - // Mutation: printing `service_id` bare in `service_logs_hint` fails this + // Mutation: printing `service_id` bare in either logs builder fails this // for any id carrying whitespace or a quote. #[test] fn the_logs_hint_denotes_the_argv_it_names(service_id in hostile_arg()) { - let hint = service_logs_hint(&service_id); - prop_assert_eq!( - shlex::split(&hint), - Some(vec![ + let argv = |follow: bool| { + let mut argv = vec![ "rocm".to_owned(), "logs".to_owned(), "--service".to_owned(), - service_id, - ]), - "{}", - hint - ); + service_id.clone(), + ]; + if follow { + argv.push("--follow".to_owned()); + } + Some(argv) + }; + let hint = service_logs_hint(&service_id); + prop_assert_eq!(shlex::split(&hint), argv(false), "{}", hint); + let hint = service_logs_follow_hint(&service_id); + prop_assert_eq!(shlex::split(&hint), argv(true), "{}", hint); } } }