diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index f359699db..08b650d27 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -79,8 +79,8 @@ use crate::service_launch::{ drop_orphaned_endpoint_key_on_already_running, e2e_simulate_oom_launch, ensure_background_helper_running_quiet, ensure_public_bind_engine_supported, managed_service_reuse_refusal, no_usable_gpu_refusal, print_managed_launch_plain, - resolve_endpoint_auth, run_attached_service, start_managed_service, validate_bind_host, - wait_begin_client_config, + resolve_endpoint_auth, run_attached_service, scripted_managed_engine_startup_failure, + start_managed_service, validate_bind_host, wait_begin_client_config, }; use crate::{ GatedEngineAction, SandboxToolArg, cli_progress, device_policy_name, @@ -421,12 +421,15 @@ fn aim_board_note( /// AMD GPU. /// /// Not "exactly when the bail fires", and the difference is only in test-hook -/// builds: the bail also requires `!scripted_backend_failure` (see the `if` -/// below it), which is `cfg!(feature = "e2e-test-hooks")`/env-gated. Under that -/// scenario the gate can open while the bail cannot fire. That is deliberate — -/// the hook stands in for the GPU the lane does not have, so the reuse path it -/// exercises must still behave as it would on a real GPU-less host — and it -/// costs nothing in a shipped binary, where the term can never be true. +/// builds: the bail is also waived by `gpu_bail_waived_by_test_hook` in +/// `serve()` — the scripted Lemonade backend failure or free-space override, +/// the simulated OOM launch, and the scripted engine startup death — each +/// compiled in only with a test feature and armed by an environment variable. +/// Under any of them the gate can open while the bail cannot fire. That is +/// deliberate — the hook stands in for the GPU the lane does not have, so the +/// reuse path it exercises must still behave as it would on a real GPU-less +/// host — and it costs nothing in a shipped binary, where none of the terms can +/// be true. /// /// `host_has_no_usable_gpu` is therefore the bail's own predicate, computed once /// by the caller and handed to both, so the two cannot drift apart. They did: @@ -767,7 +770,12 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { || selection.env_id.is_some() || engine_manages_own_runtime(&selected_engine); if can_resolve_model { - if engine_manages_own_runtime(&selected_engine) { + // Not under the scripted startup death, as for the same call below: + // that scenario asserts the launch, and a real runtime download is + // exactly what its ungated lane cannot do. + if engine_manages_own_runtime(&selected_engine) + && !scripted_managed_engine_startup_failure() + { ensure_self_managed_engine_ready(&paths, &mut config, &selected_engine)?; } let probe = engine_request::<_, ResolveModelResponse>( @@ -877,12 +885,16 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { ] .iter() .any(|variable| std::env::var_os(variable).is_some()); - if !cpu_only - && !scripted_backend_failure - && !e2e_simulate_oom_launch() - && !reuse_existing - && host_has_no_usable_gpu - { + // The scripted startup-death scenario bypasses the same host precondition, for + // the same reason: it asserts what happens *after* a spawn, so it must reach + // one without real GPU hardware. + // + // Every test hook that waives this bail is named here once, so the bail and + // `gpu_refusal_waived_for_reuse` below cannot disagree about which ones do. + let gpu_bail_waived_by_test_hook = scripted_backend_failure + || e2e_simulate_oom_launch() + || scripted_managed_engine_startup_failure(); + if !cpu_only && !gpu_bail_waived_by_test_hook && !reuse_existing && host_has_no_usable_gpu { bail!("{}", no_usable_gpu_refusal(&device_policy)); } // The Docker probe comes after every refusal that needs no daemon (the AIM @@ -891,11 +903,11 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { ensure_engine_allowed_on_platform(&selected_engine, GatedEngineAction::Serve)?; // Whether the refusal above was waived *only* because a live service was // expected to be reused. `reuse_existing` already implies `!cpu_only` and - // `host_has_no_usable_gpu` (the pre-gate requires both), so the scripted - // backend failure is the one remaining term. Handed to the spawn, which + // `host_has_no_usable_gpu` (the pre-gate requires both), so the test-hook + // waiver is the one remaining term. Handed to the spawn, which // re-checks reuse authoritatively under the launch lock and raises the // refusal itself if the service is gone by then. - let gpu_refusal_waived_for_reuse = reuse_existing && !scripted_backend_failure; + let gpu_refusal_waived_for_reuse = reuse_existing && !gpu_bail_waived_by_test_hook; // `--gpu` selects by the amd-smi `gpu` ordinal but is exported via // `HIP_VISIBLE_DEVICES`; those orderings can diverge when // `ROCR_VISIBLE_DEVICES`/partitioning is in play, so warn at serve time. @@ -996,9 +1008,13 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { device_policy_name(&device_policy) ); } + // Preparation is skipped for the scripted startup-death scenario too: it + // asserts the launch, not the install, and a real runtime download is exactly + // what the ungated lane cannot do. if !matches!(device_policy, DevicePolicy::CpuOnly) && engine_manages_own_runtime(&selected_engine) && resolved_model.is_none() + && !scripted_managed_engine_startup_failure() { ensure_self_managed_engine_ready(&paths, &mut config, &selected_engine)?; } @@ -1165,7 +1181,8 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { spinner.borrow_mut().clear(); eprintln!( "Waiting for {waiting_on}; Ctrl-C stops waiting, the server keeps running. \ - Follow: rocm logs --service {waiting_on} --follow" + Follow: {}", + crate::service_hints::service_logs_follow_hint(waiting_on) ); if let Some(config) = wait_begin_client_config( waiting_on, diff --git a/apps/rocm/src/serve_summary.rs b/apps/rocm/src/serve_summary.rs index c30bcc654..182b5e25e 100644 --- a/apps/rocm/src/serve_summary.rs +++ b/apps/rocm/src/serve_summary.rs @@ -229,7 +229,7 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { ), ( "logs", - format!("rocm logs --service {}", summary.service_id), + crate::service_hints::service_logs_hint(&summary.service_id), ), ], ] @@ -266,8 +266,8 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { }; let _ = writeln!( out, - " note: {explanation} - follow `rocm logs --service {} --follow` or check `rocm services list`", - summary.service_id + " note: {explanation} - follow `{}` or check `rocm services list`", + crate::service_hints::service_logs_follow_hint(&summary.service_id) ); } if let (true, Some(reason)) = (smoke_failed, summary.smoke_failure.as_deref()) { @@ -276,8 +276,8 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { // empty, hedged: the reason may be rocm's own plumbing, not the engine. let _ = writeln!( out, - " note: the smoke test could not complete, so no metrics were measured ({reason}); the server may not be able to serve requests - check `rocm logs --service {id}`", - id = summary.service_id + " note: the smoke test could not complete, so no metrics were measured ({reason}); the server may not be able to serve requests - check `{logs}`", + logs = crate::service_hints::service_logs_hint(&summary.service_id) ); } for note in &summary.notes { diff --git a/apps/rocm/src/service_hints.rs b/apps/rocm/src/service_hints.rs index 6556c2e70..90bbc975b 100644 --- a/apps/rocm/src/service_hints.rs +++ b/apps/rocm/src/service_hints.rs @@ -2,7 +2,8 @@ // // SPDX-License-Identifier: MIT -//! The one place every `rocm services --yes` hint is built. +//! The one place every `rocm services --yes` hint is built, and +//! the `rocm logs --service ` hints beside them. //! //! `main.rs`, `remote/mod.rs`, and `serve_summary.rs` each used to format this //! string by hand, and it drifted: `serve_summary.rs`'s stop row printed the @@ -48,6 +49,21 @@ pub(crate) fn service_forced_remove_hint(service_id: &str) -> String { ) } +/// The directly-runnable `rocm logs --service ` hint, which prints a local +/// managed service's own log, for a message that sends the user there to see +/// why the server exited. +pub(crate) fn service_logs_hint(service_id: &str) -> String { + format!("rocm logs --service {}", shell_quote(service_id)) +} + +/// The directly-runnable `rocm logs --service --follow` hint, which keeps +/// printing a local managed service's log as it grows. +pub(crate) fn service_logs_follow_hint(service_id: &str) -> String { + // Spelled out rather than built on `service_logs_hint`, so the advised-command + // contract test reads `--follow` too. + format!("rocm logs --service {} --follow", shell_quote(service_id)) +} + /// The directly-runnable `rocm services stop --yes` hint. pub(crate) fn service_stop_hint(service_id: &str) -> String { service_action_hint("stop", service_id) @@ -145,5 +161,27 @@ mod tests { ); } } + + // Mutation: printing `service_id` bare in either logs builder fails this + // for any id carrying whitespace or a quote. + #[test] + fn the_logs_hint_denotes_the_argv_it_names(service_id in hostile_arg()) { + let argv = |follow: bool| { + let mut argv = vec![ + "rocm".to_owned(), + "logs".to_owned(), + "--service".to_owned(), + service_id.clone(), + ]; + if follow { + argv.push("--follow".to_owned()); + } + Some(argv) + }; + let hint = service_logs_hint(&service_id); + prop_assert_eq!(shlex::split(&hint), argv(false), "{}", hint); + let hint = service_logs_follow_hint(&service_id); + prop_assert_eq!(shlex::split(&hint), argv(true), "{}", hint); + } } } diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 7d0e49106..59241d487 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -39,8 +39,12 @@ //! //! - `endpoint_auth`: bind-host validation and the endpoint API key checks. //! - `launch_transaction`: the locked port allocation decision, the cleanup of a -//! failed launch, and the reuse refusals. -//! - `spawn`: the detached engine child, its process and stdio plumbing, and +//! failed launch, the reuse refusals, and the guarded read-modify-write +//! (`update_managed_record_on_disk`) every launch write after the claim goes +//! through, so a stop or removal made meanwhile is not overwritten or undone. +//! - `spawn`: the detached engine child, its process and stdio plumbing, the +//! startup check that fails a launch whose engine exits inside the settle +//! window (on Windows through the spawn's process handle), and //! `ManagedLaunchReport`. //! - `readiness`: waiting on a service, its liveness, and the fault-injection //! seam (an OOM launch, or a ready launch). @@ -69,6 +73,7 @@ pub(crate) use helper_autostart::{ }; pub(crate) use launch_transaction::{managed_service_reuse_refusal, no_usable_gpu_refusal}; pub(crate) use readiness::e2e_simulate_oom_launch; +pub(crate) use spawn::scripted_managed_engine_startup_failure; #[cfg(not(windows))] pub(crate) use spawn::{ attach_background_stdio, detach_background_command, managed_engine_startup_failure_detail, diff --git a/apps/rocm/src/service_launch/launch_transaction.rs b/apps/rocm/src/service_launch/launch_transaction.rs index 4fb530a5b..abf789fec 100644 --- a/apps/rocm/src/service_launch/launch_transaction.rs +++ b/apps/rocm/src/service_launch/launch_transaction.rs @@ -134,21 +134,46 @@ const fn launch_cleanup_refusal( /// Order matters: a spawned child is terminated *first*, so it is not still /// holding the port or writing to the log by the time cleanup runs. The /// termination outcome is audited but not acted on: the record is marked -/// abandoned (`stopped`, releasing the port) even when death was not confirmed. If no child was ever spawned (`record.engine_pid` is still `None`), -/// every artifact this attempt created is removed: there is nothing for -/// `rocm services`/`rocm logs` to show. If a child did run, its record and log -/// are kept instead (marked `status = "stopped"`, see -/// [`ManagedServiceRecord::mark_launch_abandoned`]), since the error that led here -/// already told the user to go inspect that log; only the endpoint key, which a -/// failed attempt has no use for, is cleared. +/// abandoned (`stopped`, releasing the port) even when death was not confirmed. +/// +/// If no child ran — none was spawned, so `record.engine_pid` is still `None` +/// and `child_known_dead` is unset — every artifact this attempt created is +/// removed: there is nothing for `rocm services`/`rocm logs` to show. If a +/// child did run, its record and log are kept instead (marked +/// `status = "stopped"`, see [`ManagedServiceRecord::mark_launch_abandoned`]), +/// since the error that led here already told the user to go inspect that log; +/// only the endpoint key, which a failed attempt has no use for, is cleared. A +/// child ran in two shapes: its pid is on record, or `child_known_dead` says it +/// died before its pid could be recorded (below). /// Every step is best-effort and idempotent; cleanup must never mask the /// original failure with a second one. -pub(super) fn abandon_failed_managed_launch(paths: &AppPaths, record: &ManagedServiceRecord) { +/// +/// `child_known_dead` is set when the startup check saw the child exit +/// (`EngineDiedAtStartup` in `spawn`). Such a child is not signalled at all: it is +/// gone, and its pid may already belong to an unrelated process. On Windows the +/// check runs before the pid is recorded, so no start-time token was captured +/// for it and nothing else would stop the kill from landing on that process; on +/// Linux the token taken at spawn would refuse such a process, but there is +/// nothing left to signal either way. Its record and log are still kept, since +/// the error names that log. +/// +/// The `stopped` mark is a guarded write ([`update_managed_record_on_disk`]): +/// a stop that settled the record, a restart that took it over, or a removal +/// is left as it is rather than overwritten or re-created. The endpoint key +/// follows the same guard: it is cleared only when the mark applied or the +/// record is gone. A record that is still there but no longer this launch's +/// keeps it — a restart that took the record over stores its key under the same +/// service id, and a stop that settled it does its own key cleanup. +pub(super) fn abandon_failed_managed_launch( + paths: &AppPaths, + record: &ManagedServiceRecord, + child_known_dead: bool, +) { // The shared routine `rocm services stop` and `rocmd` use, not // `terminate_verified` alone: on Windows only the routine takes the // processes under the child down, so a failed launch cannot leave the engine // one level below the supervisor running. - abandon_failed_managed_launch_with(paths, record, |identity| { + abandon_failed_managed_launch_with(paths, record, child_known_dead, |identity| { rocm_core::terminate_recorded_pid_within(identity, MANAGED_LAUNCH_CLEANUP_GRACE) }); } @@ -159,55 +184,41 @@ pub(super) fn abandon_failed_managed_launch(paths: &AppPaths, record: &ManagedSe fn abandon_failed_managed_launch_with( paths: &AppPaths, record: &ManagedServiceRecord, + child_known_dead: bool, terminate: impl Fn(&rocm_core::ProcessIdentity) -> rocm_core::TerminationOutcome, ) { - if let Some(pid) = record.engine_pid.filter(|pid| *pid != 0) { - // The token captured at spawn, not a fresh `capture`: when the child - // already exited, a fresh one would describe a recycled pid's new owner - // and the forced tree kill would hit it. Where start tokens exist - // (Linux, and Windows, which records the process creation time), a - // missing one means the child was gone before it could be captured, so - // the pid is unverifiable and nothing is signalled (`terminate_verified` - // alone would "best-effort proceed"). A token that was captured is - // verified by the termination on both. - // - // The tradeoff on Windows, chosen deliberately: a child whose creation - // time could not be read at capture is not killed, so if `OpenProcess` - // failed for a child that is in fact alive (it is this account's own - // fresh child, so that is not expected), a failed launch leaves its - // engine running and the audit event says `unverified`. The alternative - // is a forced tree kill of a pid that may by now belong to another - // process, which cannot be undone. The spawn primitive returns only the - // pid and closes its handle, so a handle-based identity would mean - // changing it; that is not done here. The refusal decision - // (`launch_cleanup_refusal`) and what the cleanup asks the termination to - // do are under tests that run on Windows; the cleanup of a real Windows - // child is run by no test. - let outcome = if let Some(refusal) = - launch_cleanup_refusal(record.engine_start_ticks, host_records_start_tokens()) - { - refusal - } else { - terminate(&rocm_core::ProcessIdentity::new( - pid, - record.engine_start_ticks, - )) - }; - record_cli_audit_event( + let spawned_pid = record.engine_pid.filter(|pid| *pid != 0); + if child_known_dead || spawned_pid.is_some() { + if let Some(pid) = spawned_pid.filter(|_| !child_known_dead) { + terminate_abandoned_launch_child(paths, record, pid, terminate); + } + match update_managed_record_on_disk( paths, - "service", - "managed_service_launch_cleanup", - "warn", - format!( - "terminated engine child pid={pid} after a failed launch publication outcome={}", - outcome.as_str() + &record.service_id, + |current| launch_still_owns(current, record), + ManagedServiceRecord::mark_launch_abandoned, + ) { + Ok(RecordUpdate::Written(_) | RecordUpdate::Gone) => { + endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); + } + // A restart that took the record over stores its key under the same + // service id, and a stop that settled it does its own key cleanup. + Ok(RecordUpdate::NotOwned(_)) => {} + // The record is there but could not be read, so whose it is now is + // unknown: keep the key rather than guess, and leave a trace, since + // cleanup must not fail the launch a second time. + Err(error) => record_cli_audit_event( + paths, + "service", + "managed_service_launch_cleanup", + "warn", + format!( + "could not mark the failed launch stopped; its record and endpoint key \ + were left as they are: {error:#}" + ), + Some(&record.service_id), ), - Some(&record.service_id), - ); - let mut abandoned = record.clone(); - abandoned.mark_launch_abandoned(); - let _ = abandoned.write(); - endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); + } return; } for path in [ @@ -220,6 +231,126 @@ fn abandon_failed_managed_launch_with( endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); } +/// Terminate the child of an abandoned launch whose state is unknown, and audit +/// the outcome. +fn terminate_abandoned_launch_child( + paths: &AppPaths, + record: &ManagedServiceRecord, + pid: u32, + terminate: impl Fn(&rocm_core::ProcessIdentity) -> rocm_core::TerminationOutcome, +) { + // The token captured at spawn, not a fresh `capture`: when the child + // already exited, a fresh one would describe a recycled pid's new owner + // and the forced tree kill would hit it. Where start tokens exist + // (Linux, and Windows, which records the process creation time), a + // missing one means the child was gone before it could be captured, so + // the pid is unverifiable and nothing is signalled (`terminate_verified` + // alone would "best-effort proceed"). A token that was captured is + // verified by the termination on both. + // + // The tradeoff on Windows, chosen deliberately: a child whose creation + // time could not be read at capture is not killed, so if `OpenProcess` + // failed for a child that is in fact alive (it is this account's own + // fresh child, so that is not expected), a failed launch leaves its + // engine running and the audit event says `unverified`. The alternative + // is a forced tree kill of a pid that may by now belong to another + // process, which cannot be undone. The spawn primitive returns only the + // pid and closes its handle, so a handle-based identity would mean + // changing it; that is not done here. The refusal decision + // (`launch_cleanup_refusal`) and what the cleanup asks the termination to + // do are under tests that run on Windows; the cleanup of a real Windows + // child is run by no test. + let outcome = if let Some(refusal) = + launch_cleanup_refusal(record.engine_start_ticks, host_records_start_tokens()) + { + refusal + } else { + terminate(&rocm_core::ProcessIdentity::new( + pid, + record.engine_start_ticks, + )) + }; + record_cli_audit_event( + paths, + "service", + "managed_service_launch_cleanup", + "warn", + format!( + "terminated engine child pid={pid} after a failed launch publication outcome={}", + outcome.as_str() + ), + Some(&record.service_id), + ); +} + +/// Apply one change to a managed service's record as it is on disk now, rather +/// than writing back a copy the caller holds. +/// +/// `ManagedServiceRecord::write` replaces the whole file, and a stop records +/// `stop_requested_unix_ms` and then `"stopped"` without taking the launch lock. +/// A copy read before the stop and written back after it would erase both, +/// bringing back as live a service the stop just reported stopped, and with it +/// the marker that keeps `rocmd`'s recovery off it. +/// +/// Writes nothing when the record is gone or `still_owned` refuses the record +/// as it is now — whoever wrote since holds a truer view — and says which, so a +/// caller can tell the two apart without reading the record again. This narrows +/// the lost-update window; it does not close it, since the read and the write +/// are separate steps and nothing that writes a service record (a stop, a +/// removal, `rocmd`, the liveness refresh) takes a lock on it. +pub(super) fn update_managed_record_on_disk( + paths: &AppPaths, + service_id: &str, + still_owned: impl FnOnce(&ManagedServiceRecord) -> bool, + update: impl FnOnce(&mut ManagedServiceRecord), +) -> Result { + // One read, not an `exists()` check before it: a removal landing between + // the two would turn "the record is gone" into a read error. + let mut current = match ManagedServiceRecord::read_manifest(paths, service_id) { + Ok(current) => current, + Err(error) + if error + .downcast_ref::() + .is_some() => + { + return Ok(RecordUpdate::Gone); + } + Err(error) => return Err(error), + }; + if !still_owned(¤t) { + return Ok(RecordUpdate::NotOwned(Box::new(current))); + } + update(&mut current); + current.write()?; + Ok(RecordUpdate::Written(Box::new(current))) +} + +/// What [`update_managed_record_on_disk`] did with the record on disk. +#[derive(Debug)] +pub(super) enum RecordUpdate { + /// The change was applied; this is the record as written. + Written(Box), + /// The record is there but `still_owned` refused it; this is the record as + /// read, left unchanged. + NotOwned(Box), + /// There is no record for the service. + Gone, +} + +/// Whether the launch that wrote `launched` may still write `current`, the same +/// service's record as it is on disk now: it still reads as live, and names the +/// supervisor the launch last recorded — 0 for the claim, before any child is +/// on record. A confirmed stop writes `"stopped"`; a restart records its own +/// supervisor. Either way the record is no longer the launch's to write. +pub(super) fn launch_still_owns( + current: &ManagedServiceRecord, + launched: &ManagedServiceRecord, +) -> bool { + crate::managed_service_status::managed_service_is_live(current) + && current.supervisor_pid == launched.supervisor_pid + && current.supervisor_start_ticks == launched.supervisor_start_ticks +} + /// The ways a live managed service that already matches on liveness, engine and /// canonical model id can still be unusable by this invocation. /// @@ -353,7 +484,6 @@ pub(crate) fn managed_service_reuse_refusal( mod tests { use super::*; use crate::service_hints; - #[cfg(not(windows))] use crate::service_launch::test_support::sleeper_test_name; use crate::service_launch::test_support::{reusable_record, write_live_service_record}; use crate::tests::test_paths; @@ -939,7 +1069,7 @@ mod tests { // Simulate a pre-spawn failure: nothing ever ran. record.engine_pid = None; - abandon_failed_managed_launch(&paths, &record); + abandon_failed_managed_launch(&paths, &record, false); assert!(!record.manifest_path.exists()); assert!(!record.log_path.exists()); @@ -980,7 +1110,7 @@ mod tests { record.engine_start_ticks = token; let asked = RefCell::new(Vec::new()); - abandon_failed_managed_launch_with(&paths, &record, |identity| { + abandon_failed_managed_launch_with(&paths, &record, false, |identity| { asked.borrow_mut().push(*identity); rocm_core::TerminationOutcome::Forced }); @@ -1029,13 +1159,13 @@ mod tests { record.record_engine_identity(bystander.id()); record.engine_start_ticks = record.engine_start_ticks.map(|ticks| ticks + 1); - abandon_failed_managed_launch(&paths, &record); + abandon_failed_managed_launch(&paths, &record, false); let mismatched_left_alone = bystander.try_wait().expect("poll bystander").is_none(); // A child that was gone before its token could be captured leaves none: // that must not fall back to "best-effort" signalling either. record.engine_start_ticks = None; - abandon_failed_managed_launch(&paths, &record); + abandon_failed_managed_launch(&paths, &record, false); let still_running = mismatched_left_alone && bystander.try_wait().expect("poll bystander").is_none(); let _ = bystander.kill(); @@ -1073,7 +1203,7 @@ mod tests { let child_pid = child.id(); record.record_engine_identity(child_pid); - abandon_failed_managed_launch(&paths, &record); + abandon_failed_managed_launch(&paths, &record, false); let wait_result = child.wait(); @@ -1104,6 +1234,286 @@ mod tests { Ok(()) } + /// A sleeper the test owns, standing in for an unrelated process that now + /// holds a pid the launch once recorded. Spawned with `std`'s `Command` + /// directly, so the test runs on Windows too, where this matters most: the + /// startup check there runs before the dead engine's pid is recorded, so no + /// start-time token was captured that could guard a kill. + fn spawn_bystander() -> std::process::Child { + std::process::Command::new(std::env::current_exe().expect("current test exe")) + .arg("--exact") + .arg(sleeper_test_name()) + .arg("--ignored") + .arg("--test-threads=1") + .env("ROCM_TEST_SLEEPER_CHILD", "1") + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn() + .expect("spawn bystander") + } + + #[test] + fn an_engine_seen_dead_at_startup_is_never_signalled() -> Result<()> { + // The startup check saw the engine exit, so its pid is free. Here a + // bystander holds it. Cleanup must not signal it — not even through a + // start-time token, which on Windows would be read only after the + // engine had died and so would describe the bystander — and must + // still keep the record and log of the failed launch for the error + // that names them. + let (root, paths) = test_paths("abandon-known-dead-engine"); + paths.ensure()?; + let mut record = write_live_service_record( + &paths, + "lemonade-abandon-known-dead", + "abandon-known-dead-model", + 11_495, + "starting", + )?; + fs::write(&record.log_path, "engine: bad flag\n")?; + let mut bystander = spawn_bystander(); + record.record_engine_identity(bystander.id()); + + abandon_failed_managed_launch(&paths, &record, true); + + let left_alone = bystander.try_wait().expect("poll bystander").is_none(); + let _ = bystander.kill(); + let _ = bystander.wait(); + let persisted = ManagedServiceRecord::read_manifest(&paths, &record.service_id); + let log_kept = record.log_path.exists(); + let _ = fs::remove_dir_all(root); + assert!( + left_alone, + "a pid the dead engine freed must not be signalled" + ); + assert_eq!(persisted?.status, "stopped"); + assert!(log_kept, "the log the error names must be kept"); + Ok(()) + } + + #[test] + fn abandoning_a_launch_does_not_recreate_a_removed_record() -> Result<()> { + let (root, paths) = test_paths("abandon-removed-record"); + paths.ensure()?; + let record = write_live_service_record( + &paths, + "lemonade-abandon-removed", + "abandon-removed-model", + 11_496, + "starting", + )?; + fs::remove_file(&record.manifest_path)?; + + abandon_failed_managed_launch(&paths, &record, true); + + let recreated = record.manifest_path.exists(); + let _ = fs::remove_dir_all(root); + assert!(!recreated, "cleanup must not bring back a removed record"); + Ok(()) + } + + #[test] + fn the_launch_does_not_publish_over_a_record_a_restart_took_over() -> Result<()> { + // A restart records its own supervisor on a record that still reads as + // live. The launch's claim names supervisor 0, so it is no longer the + // launch's to write. + let (root, paths) = test_paths("launch-owns-supervisor"); + paths.ensure()?; + let launched = write_live_service_record( + &paths, + "lemonade-owns-supervisor", + "owns-supervisor-model", + 11_497, + "starting", + )?; + let mut taken = ManagedServiceRecord::read_manifest(&paths, &launched.service_id)?; + taken.supervisor_pid = launched.supervisor_pid.wrapping_add(7); + taken.write()?; + assert!(crate::managed_service_status::managed_service_is_live( + &taken + )); + + assert!(!launch_still_owns(&taken, &launched)); + let written = update_managed_record_on_disk( + &paths, + &launched.service_id, + |current| launch_still_owns(current, &launched), + |current| current.status = "running".to_owned(), + )?; + let after = ManagedServiceRecord::read_manifest(&paths, &launched.service_id)?; + let _ = fs::remove_dir_all(root); + assert!( + matches!(written, RecordUpdate::NotOwned(_)), + "the launch wrote over a restart's record: {written:?}" + ); + assert_eq!(after.status, "starting"); + assert_eq!(after.supervisor_pid, taken.supervisor_pid); + // Same supervisor pid but a different start token: a restart whose new + // supervisor reused the pid. Not the launch's either. + let mut reused = launched.clone(); + reused.supervisor_start_ticks = Some(launched.supervisor_start_ticks.unwrap_or(0) + 1); + assert!(!launch_still_owns(&reused, &launched)); + // Same supervisor, same start token, still live: the launch owns it. + assert!(launch_still_owns(&launched, &launched)); + Ok(()) + } + + /// Through the seam, with a token captured and so on every host: a child the + /// startup check saw die is never handed to the termination at all, while + /// the same record with the child not known dead is. + #[test] + fn an_engine_seen_dead_at_startup_is_never_handed_to_the_termination() -> Result<()> { + use std::cell::Cell; + for (child_known_dead, expect_call) in [(true, false), (false, true)] { + let (root, paths) = test_paths("abandon-known-dead-seam"); + paths.ensure()?; + let mut record = write_live_service_record( + &paths, + "lemonade-abandon-known-dead-seam", + "abandon-known-dead-seam-model", + 11_493, + "starting", + )?; + record.engine_pid = Some(4243); + record.engine_start_ticks = Some(78); + let asked = Cell::new(false); + + abandon_failed_managed_launch_with(&paths, &record, child_known_dead, |_| { + asked.set(true); + rocm_core::TerminationOutcome::Forced + }); + + let persisted = ManagedServiceRecord::read_manifest(&paths, &record.service_id); + let _ = fs::remove_dir_all(root); + assert_eq!( + asked.get(), + expect_call, + "child_known_dead {child_known_dead}" + ); + assert_eq!( + persisted?.status, "stopped", + "child_known_dead {child_known_dead}" + ); + } + Ok(()) + } + + /// A restart that took the record over during the failed launch stored its + /// own endpoint key under the same service id. The cleanup leaves that + /// record alone, and must leave its key alone too; the same cleanup on a + /// record that is still the launch's own clears the key. + #[test] + fn abandoning_a_launch_keeps_the_key_of_a_record_a_restart_took_over() -> Result<()> { + for taken_over in [true, false] { + let (root, paths) = test_paths("abandon-keeps-restart-key"); + paths.ensure()?; + let launched = write_live_service_record( + &paths, + "lemonade-abandon-restart-key", + "abandon-restart-key-model", + 11_492, + "starting", + )?; + if taken_over { + let mut taken = ManagedServiceRecord::read_manifest(&paths, &launched.service_id)?; + taken.supervisor_pid = launched.supervisor_pid.wrapping_add(7); + taken.write()?; + } + endpoint_keys::store_endpoint_api_key(&paths, &launched.service_id, "restart-key")?; + + abandon_failed_managed_launch(&paths, &launched, true); + + let key = endpoint_keys::endpoint_api_key(&paths, &launched.service_id); + let persisted = ManagedServiceRecord::read_manifest(&paths, &launched.service_id); + let _ = fs::remove_dir_all(root); + let persisted = persisted?; + if taken_over { + assert_eq!( + key.as_deref(), + Some("restart-key"), + "the restart's key was cleared" + ); + assert_eq!( + persisted.status, "starting", + "the restart's record was overwritten" + ); + } else { + assert_eq!(key, None, "a failed launch's own key must be cleared"); + assert_eq!(persisted.status, "stopped"); + } + } + Ok(()) + } + + /// A record that is there but cannot be read says nothing about whose it is + /// now, so the cleanup keeps the endpoint key and leaves the record's bytes + /// alone, and audits that it did so instead of failing the launch twice. + #[test] + fn abandoning_a_launch_over_an_unreadable_record_keeps_the_key_and_says_so() -> Result<()> { + let (root, paths) = test_paths("abandon-unreadable-record"); + paths.ensure()?; + let launched = write_live_service_record( + &paths, + "lemonade-abandon-unreadable", + "abandon-unreadable-model", + 11_488, + "starting", + )?; + endpoint_keys::store_endpoint_api_key(&paths, &launched.service_id, "unknown-owner-key")?; + fs::write(&launched.manifest_path, b"{ not a service record")?; + + abandon_failed_managed_launch(&paths, &launched, true); + + let key = endpoint_keys::endpoint_api_key(&paths, &launched.service_id); + let bytes = fs::read(&launched.manifest_path)?; + let audit = fs::read_to_string(paths.audit_events_path()).unwrap_or_default(); + let _ = fs::remove_dir_all(root); + assert_eq!( + key.as_deref(), + Some("unknown-owner-key"), + "the key was cleared" + ); + assert_eq!( + bytes, b"{ not a service record", + "the unreadable record was rewritten" + ); + assert!( + audit.contains("could not mark the failed launch stopped"), + "the skipped cleanup left no trace: {audit}" + ); + Ok(()) + } + + #[test] + fn an_engine_seen_dead_before_it_was_recorded_keeps_its_record_and_log() -> Result<()> { + // The Windows shape: the startup check saw the engine exit before + // `record_engine_identity` ran, so no engine pid is on record. Nothing + // is spawned-and-recorded, yet the failure names the log, so the record + // and log must survive, marked `stopped`. + let (root, paths) = test_paths("abandon-dead-engine-never-recorded"); + paths.ensure()?; + let mut record = write_live_service_record( + &paths, + "lemonade-abandon-never-recorded", + "abandon-never-recorded-model", + 11_498, + "starting", + )?; + record.engine_pid = None; + record.supervisor_pid = 0; + fs::write(&record.log_path, "engine: bad flag\n")?; + record.write()?; + + abandon_failed_managed_launch(&paths, &record, true); + + let persisted = ManagedServiceRecord::read_manifest(&paths, &record.service_id); + let log_kept = record.log_path.exists(); + let _ = fs::remove_dir_all(root); + assert_eq!(persisted?.status, "stopped"); + assert!(log_kept, "the log the error names must be kept"); + Ok(()) + } + /// A failed launch's cleanup refuses to signal a child whose token was not /// captured on any host that records tokens. Windows is one of them, since /// it records the creation time: a missing token there means the child was diff --git a/apps/rocm/src/service_launch/readiness.rs b/apps/rocm/src/service_launch/readiness.rs index c10a29eea..8db8908a9 100644 --- a/apps/rocm/src/service_launch/readiness.rs +++ b/apps/rocm/src/service_launch/readiness.rs @@ -14,15 +14,17 @@ // would be unused in some build, and a gate on it would have to be kept in step // with every use site. Keep the full paths. The tests module imports `fs` for // itself because its tests use it on every platform. +use super::launch_transaction::{RecordUpdate, update_managed_record_on_disk}; use super::spawn::ManagedLaunchReport; #[cfg(feature = "e2e-oom-fault-injection")] use crate::device_policy_name; use crate::endpoint_keys; use crate::engine_runtime_select::wait_for_service_http_ready_with_progress; +use crate::service_hints; use crate::status_for_readiness; #[cfg(feature = "e2e-oom-fault-injection")] use anyhow::Context; -use anyhow::{Result, anyhow}; +use anyhow::{Result, anyhow, bail}; #[cfg(feature = "e2e-oom-fault-injection")] use rocm_core::format_http_base_url; use rocm_core::{AppPaths, EndpointReadiness, ManagedServiceRecord}; @@ -130,7 +132,8 @@ pub(super) fn died_during_wait( } settle_exited_service(paths, service_id); let mut message = format!( - "the server exited before it became ready (vLLM and AIM give up after 5 minutes by default; `ROCM_CLI_VLLM_READY_TIMEOUT_SECS` / `ROCM_CLI_AIM_READY_TIMEOUT_SECS` allow longer); see `rocm logs --service {service_id}` for its output" + "the server exited before it became ready (vLLM and AIM give up after 5 minutes by default; `ROCM_CLI_VLLM_READY_TIMEOUT_SECS` / `ROCM_CLI_AIM_READY_TIMEOUT_SECS` allow longer); see `{}` for its output", + service_hints::service_logs_hint(service_id) ); // A plain `serve` adds the memory guidance for a vLLM launch that ran out of // VRAM; the wait's failure is the same launch, so it gets the same note. @@ -154,23 +157,73 @@ pub(super) fn died_during_wait( /// only what the wait learned. The copy read at spawn time can be long stale: /// `rocmd` may have recorded a restart or a new supervisor pid, and `rocm logs` /// refreshes the record, so writing it back whole would revert those. +/// +/// The verdict is written only while the record still reads as live. A +/// `rocm services stop` that settled it during the wait keeps its `stopped` +/// and the wait fails saying so: reporting a readiness verdict (and exiting 0) +/// for a server somebody stopped would be false. A record settled without a +/// stop marker means the server exited on its own (without `--wait`, +/// `died_during_wait` does not run, so this is where that is noticed), and the +/// failure points at its log instead. A record removed during the +/// wait (`rocm services remove`) is not re-created from the stale copy; that +/// fails too, since there is no service left to report. pub(super) fn record_wait_outcome( paths: &AppPaths, held: &mut ManagedServiceRecord, status: &str, verified: bool, ) -> Result<()> { - let mut latest = ManagedServiceRecord::read_manifest(paths, &held.service_id) - .unwrap_or_else(|_| held.clone()); - status.clone_into(&mut latest.status); - if verified { - // Latch the verification the wait just performed, so the readiness checks - // behind `services list` and chat read it instead of re-probing. - latest.inference_verified_at_unix_ms = Some(rocm_core::unix_time_millis() as u64); + let written = update_managed_record_on_disk( + paths, + &held.service_id, + crate::managed_service_status::managed_service_is_live, + |latest| { + status.clone_into(&mut latest.status); + if verified { + // Latch the verification the wait just performed, so the readiness + // checks behind `services list` and chat read it instead of + // re-probing. + latest.inference_verified_at_unix_ms = Some(rocm_core::unix_time_millis() as u64); + } + }, + )?; + // The record the guarded write refused is the one the message describes: + // it is not read a second time, so nothing can change between the decision + // and what is reported. + match written { + RecordUpdate::Written(latest) => { + *held = *latest; + Ok(()) + } + RecordUpdate::NotOwned(settled) => { + *held = *settled; + // A stop leaves its marker; anything else that settled the record + // (the liveness refresh demoting a dead engine, or the engine's own + // `failed`) means the server exited on its own, and the user needs + // its log rather than to hear somebody stopped it. + if held.stop_requested_unix_ms.is_some() { + bail!( + "managed service `{}` was stopped while it was starting (its record now \ + reads `{}`), so no readiness is reported for it; start it again with \ + `rocm serve`", + held.service_id, + held.status + ); + } + bail!( + "managed service `{}` exited while it was starting (its record now reads `{}`); \ + see `{}` for its output", + held.service_id, + held.status, + service_hints::service_logs_hint(&held.service_id) + ); + } + RecordUpdate::Gone => bail!( + "managed service `{}` was removed while `rocm serve` waited for it, so there is \ + no record of it left to report", + held.service_id + ), } - latest.write()?; - *held = latest; - Ok(()) } /// Test-only fault-injection switch, off (`const false`) in shipped binaries: @@ -607,8 +660,14 @@ mod tests { message.contains("exited before it became ready"), "{message}" ); + // The hinted command parses with these flags: + // `advised_commands::every_advised_command_parses` reads the literal in + // `service_hints::service_logs_hint`. assert!( - message.contains(&format!("rocm logs --service {id}")), + message.contains(&format!( + "see `{}` for its output", + service_hints::service_logs_hint(&id) + )), "{message}" ); assert_eq!(status(), "stopped"); @@ -670,4 +729,97 @@ mod tests { assert_eq!(after.status, "starting"); let _ = fs::remove_dir_all(root); } + + #[test] + fn record_wait_outcome_does_not_recreate_a_record_removed_during_the_wait() { + let (root, paths) = test_paths("wait-outcome-removed"); + paths.ensure().expect("paths"); + let mut held = record_tracking_pid(&paths, 111, "starting"); + held.write().expect("write"); + // `rocm services remove` ran while `rocm serve` waited. + fs::remove_file(&held.manifest_path).expect("remove manifest"); + + let error = record_wait_outcome(&paths, &mut held, "ready", true) + .expect_err("a removed service has nothing left to report"); + let recreated = held.manifest_path.exists(); + let _ = fs::remove_dir_all(root); + assert!( + error.to_string().contains("was removed while"), + "the failure must say what happened: {error:#}" + ); + assert!(!recreated, "the stale copy must not re-create the record"); + } + + #[test] + fn an_engine_that_exited_during_the_wait_is_reported_as_exited_not_stopped() { + // Without `--wait`, `died_during_wait` does not run, so a record the + // liveness refresh demoted (or the engine marked `failed`) reaches the + // verdict write. No stop marker: nobody stopped it, it exited. + for settled_status in ["stopped", "failed"] { + let (root, paths) = test_paths(&format!("wait-outcome-exited-{settled_status}")); + paths.ensure().expect("paths"); + let mut held = record_tracking_pid(&paths, 111, "starting"); + held.write().expect("write"); + let mut exited = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); + exited.status = settled_status.to_owned(); + exited.write().expect("write"); + + let error = record_wait_outcome(&paths, &mut held, "starting", false) + .expect_err("a server that exited must not be reported as launched"); + let after = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); + let _ = fs::remove_dir_all(root); + let message = error.to_string(); + assert!( + message.contains("exited while it was starting"), + "{message}" + ); + assert!(!message.contains("was stopped"), "{message}"); + // The hinted command parses with these flags: + // `advised_commands::every_advised_command_parses` reads the literal + // in `service_hints::service_logs_hint`. + assert!( + message.contains(&format!( + "see `{}` for its output", + service_hints::service_logs_hint(&held.service_id) + )), + "{message}" + ); + assert_eq!( + after.status, settled_status, + "the settled record was overwritten" + ); + assert_eq!(after.stop_requested_unix_ms, None); + } + } + + #[test] + fn a_stop_settled_during_the_wait_fails_the_serve_and_keeps_the_stop() { + let (root, paths) = test_paths("wait-outcome-stopped"); + paths.ensure().expect("paths"); + let mut held = record_tracking_pid(&paths, 111, "starting"); + held.write().expect("write"); + // `rocm services stop` settled the record while `rocm serve` waited. + let mut stopped = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); + stopped.stop_requested_unix_ms = Some(1_234); + stopped.status = "stopped".to_owned(); + stopped.write().expect("write"); + + // An `Err` here is what `rocm serve` turns into a non-zero exit: the + // launch path propagates it with `?` and prints nothing else. + let error = record_wait_outcome(&paths, &mut held, "ready", true) + .expect_err("a server stopped during its wait must not be reported as launched"); + let after = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); + let _ = fs::remove_dir_all(root); + let message = error.to_string(); + assert!( + message.contains("was stopped while it was starting"), + "{message}" + ); + assert!(message.contains(&held.service_id), "{message}"); + // The message and the state it describes, together: the record really + // does read stopped, with its marker, and carries no readiness verdict. + assert_eq!(after.status, "stopped", "the stop was overwritten"); + assert_eq!(after.stop_requested_unix_ms, Some(1_234)); + assert_eq!(after.inference_verified_at_unix_ms, None); + } } diff --git a/apps/rocm/src/service_launch/spawn.rs b/apps/rocm/src/service_launch/spawn.rs index 9b3252efc..5fb48bada 100644 --- a/apps/rocm/src/service_launch/spawn.rs +++ b/apps/rocm/src/service_launch/spawn.rs @@ -8,8 +8,9 @@ use super::endpoint_auth::ensure_public_service_has_endpoint_key; use super::launch_transaction::{ - ManagedPortDecision, ManagedReuseRefusal, abandon_failed_managed_launch, - managed_service_reuse_refusal, no_usable_gpu_refusal, resolve_managed_port_in_transaction, + ManagedPortDecision, ManagedReuseRefusal, RecordUpdate, abandon_failed_managed_launch, + launch_still_owns, managed_service_reuse_refusal, no_usable_gpu_refusal, + resolve_managed_port_in_transaction, update_managed_record_on_disk, }; use crate::endpoint_keys; use crate::engine_ipc::builtin_engine_serve_http_args; @@ -22,14 +23,13 @@ use crate::service_hints; use crate::services_text::read_optional_tail_lines; use crate::{device_policy_name, record_cli_audit_event}; use anyhow::{Context, Result, bail}; -#[cfg(windows)] -use rocm_core::process_is_running; use rocm_core::{AppPaths, ManagedServiceRecord}; use rocm_engine_protocol::{DevicePolicy, EngineRecipeHint, ResolveModelResponse}; use std::fs; #[cfg(not(windows))] use std::io; use std::path::{Path, PathBuf}; +use std::process::ExitStatus; #[cfg(not(windows))] use std::process::{Command as ProcessCommand, Stdio}; use std::thread; @@ -60,32 +60,128 @@ pub(crate) fn attach_background_stdio( Ok(()) } -/// Windows spawn-liveness check: wait out the startup window, then fail with the -/// startup log when the child is already gone. The wait and the liveness probe -/// are injected so a unit test runs the exit-detected path on every platform, -/// including the Windows lane that actually calls it. +/// How long a freshly spawned managed engine is watched before it is treated as +/// started. Long enough to catch an engine that dies on the way up — arguments it +/// rejects, a runtime it cannot load — and short enough to be invisible next to +/// the readiness wait that follows. Both platforms use the same budget. /// -/// # Errors -/// Fails with the startup-failure detail when `is_running` reports the child dead. -#[cfg_attr(not(windows), allow(dead_code))] -fn ensure_spawned_child_survives_startup( - child_pid: u32, - log_path: &Path, - settle: impl FnOnce(), - is_running: impl FnOnce(u32) -> bool, -) -> Result<()> { - settle(); - if is_running(child_pid) { - Ok(()) - } else { - bail!("{}", managed_engine_startup_failure_detail("", log_path)) +/// A window, not a guarantee: engines bind their port and load the model later +/// than this, so a death there passes the check and the launch is reported as +/// still starting. +pub(crate) const MANAGED_ENGINE_STARTUP_SETTLE: Duration = Duration::from_millis(200); + +/// The error a launch fails with when its engine was seen dead inside +/// [`MANAGED_ENGINE_STARTUP_SETTLE`]. +/// +/// A type of its own so the launch's cleanup can tell a child *known* to be dead +/// from one whose state is unknown: a known-dead child is never signalled, since +/// its pid is free and may already name an unrelated process. +#[derive(Debug)] +struct EngineDiedAtStartup(String); + +impl std::fmt::Display for EngineDiedAtStartup { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) } } +impl std::error::Error for EngineDiedAtStartup {} + +/// Fail the launch when the startup check saw the engine exit, naming the +/// engine's own log and carrying a tail of it. +/// +/// Both platforms report through here — Unix from `Child::try_wait`, Windows +/// from the exit code the watched detached spawn read through the process +/// handle `CreateProcessW` returned — so a death at startup reads the same on +/// each, exit status included. `None` means the engine was still running, or +/// (on Windows) that its state could not be read, which is treated as running. +fn fail_if_engine_died_at_startup(startup_exit: Option, log_path: &Path) -> Result<()> { + let Some(status) = startup_exit else { + return Ok(()); + }; + Err(EngineDiedAtStartup(managed_engine_startup_failure_detail( + &format!(" with status {status}"), + log_path, + )) + .into()) +} + +/// Put a watched spawn's engine on `record`, unless the watch saw it exit. +/// +/// The order is the point: the startup check runs *before* the pid is recorded, +/// so an engine seen dead is never named by the record. Its pid is free, and a +/// creation time read only after it died would describe whatever process reuses +/// that pid, so nothing else would stop a later cleanup or stop from signalling +/// it. A live engine is recorded before anything else can fail, so cleanup knows +/// to terminate it. +/// +/// Called on Windows, where the spawn reports the exit itself; kept free of any +/// platform API so its unit test runs on every lane, the Windows one included. +#[cfg_attr(not(windows), allow(dead_code))] +fn adopt_watched_engine_spawn( + record: &mut ManagedServiceRecord, + pid: u32, + startup_exit: Option, +) -> Result { + fail_if_engine_died_at_startup(startup_exit, &record.log_path)?; + record.record_engine_identity(pid); + Ok(pid) +} + +/// Whether `error` is a launch failure for an engine seen dead at startup. +fn engine_died_at_startup(error: &anyhow::Error) -> bool { + error.downcast_ref::().is_some() +} + +/// Read a watched detached spawn's early exit as an `ExitStatus`. +/// +/// `GetExitCodeProcess` yields a bare `u32`, which is what +/// `ExitStatusExt::from_raw` takes on Windows. +#[cfg(windows)] +fn managed_startup_exit(spawn: &rocm_core::DetachedSpawn) -> Option { + use std::os::windows::process::ExitStatusExt as _; + + spawn.early_exit_code.map(ExitStatus::from_raw) +} + +/// Whether the E2E-only switch that makes the managed engine die during startup +/// is armed, so the black-box suite can assert what a user sees when it does. +/// +/// Read in the parent, which spawns the engine, so nothing test-only reaches +/// the child's own code path. The child is the real `rocm` binary, really +/// spawned and really dead — only its arguments are swapped for ones the CLI +/// rejects, so the launch takes the path a broken engine takes. +/// +/// Armed by [`rocm_core::MANAGED_ENGINE_STARTUP_FAILURE_TEST_ENV`], which the +/// E2E suite names too. +#[cfg(feature = "e2e-test-hooks")] +pub(crate) fn scripted_managed_engine_startup_failure() -> bool { + std::env::var_os(rocm_core::MANAGED_ENGINE_STARTUP_FAILURE_TEST_ENV).is_some() +} + +/// Release builds do not enable `e2e-test-hooks`, so the switch does not exist +/// there and no environment variable can arm it. +#[cfg(not(feature = "e2e-test-hooks"))] +pub(crate) const fn scripted_managed_engine_startup_failure() -> bool { + false +} + +/// Arguments that make the spawned `rocm` exit at once with a non-zero status. +/// Used only under [`scripted_managed_engine_startup_failure`]. +/// +/// An unrecognised *flag*, deliberately: clap rejects it with exit code 2 before +/// any work. An unrecognised subcommand would not do — the CLI falls back to +/// natural-language request parsing for those and exits 0. +#[cfg(feature = "e2e-test-hooks")] +fn managed_engine_startup_failure_args() -> Vec { + vec!["--e2e-managed-engine-startup-failure".to_owned()] +} + /// `status_clause` is a pre-formatted fragment inserted right after "exited -/// immediately" (e.g. `" with status exit status: 1"` on Unix, where a real -/// `ExitStatus` is available; empty on Windows, where the liveness check -/// below only has a yes/no answer, not an exit code). +/// immediately", e.g. `" with status exit status: 1"`. The launch passes one on +/// both platforms: the exit status comes from `Child::try_wait` on Unix and from +/// the exit code the watched Windows spawn read through the process handle. It +/// may be empty for a caller that has no status to name. pub(crate) fn managed_engine_startup_failure_detail( status_clause: &str, log_path: &Path, @@ -372,7 +468,8 @@ pub(super) fn spawn_managed_engine_child( // record's first write: `record.engine_pid` tells it whether a child // ever ran, so a pre-spawn failure deletes everything and a post-spawn // one keeps the record and log (marked `status = "stopped"`) instead. - abandon_failed_managed_launch(paths, &record); + // An engine seen dead at startup is not signalled: its pid is free. + abandon_failed_managed_launch(paths, &record, engine_died_at_startup(&error)); Err(error) } } @@ -444,6 +541,14 @@ fn launch_managed_engine_child_locked( &record.engine_state_path, Some(&record.log_path), )?; + // E2E-only: swap in arguments the child rejects at once, so the launch below + // observes a real engine that really died during startup. + #[cfg(feature = "e2e-test-hooks")] + let serve_args = if scripted_managed_engine_startup_failure() { + managed_engine_startup_failure_args() + } else { + serve_args + }; let engine_envs_root = env_root_for_service(paths, engine, runtime_id, env_id)?; // Hand the child the *path* to the endpoint key file (public bind only) via the // environment. A path — not the secret value — is what the detached-spawn @@ -486,21 +591,17 @@ fn launch_managed_engine_child_locked( if let Some(key_file) = endpoint_key_file.as_deref() { env_refs.push((rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file)); } - let child_pid = rocm_core::spawn_detached_no_inherit(¤t_exe, &serve_args, &env_refs) - .context("failed to launch managed engine process")?; - // Set before the immediate-exit check below, so a bail! here still leaves - // `record.engine_pid` telling cleanup that a child was spawned. The - // start-time token is captured here too, while the child is certainly - // ours: cleanup binds its kill to it rather than to whatever owns the - // pid by then. - record.record_engine_identity(child_pid); - ensure_spawned_child_survives_startup( - child_pid, - &record.log_path, - || thread::sleep(Duration::from_millis(200)), - process_is_running, - )?; - child_pid + // Watched through the process handle `CreateProcessW` returned, before + // it is closed: re-checking the pid afterwards could find a different + // process that reused it, and report a dead engine as running. + let spawn = rocm_core::spawn_detached_no_inherit_watching_startup( + ¤t_exe, + &serve_args, + &env_refs, + MANAGED_ENGINE_STARTUP_SETTLE, + ) + .context("failed to launch managed engine process")?; + adopt_watched_engine_spawn(record, spawn.pid, managed_startup_exit(&spawn))? }; #[cfg(not(windows))] let child_pid = { @@ -531,21 +632,30 @@ fn launch_managed_engine_child_locked( // ours: cleanup binds its kill to it rather than to whatever owns the // pid by then. record.record_engine_identity(child_pid); - thread::sleep(Duration::from_millis(200)); - if let Some(status) = child + thread::sleep(MANAGED_ENGINE_STARTUP_SETTLE); + let startup_exit = child .try_wait() - .context("failed to check managed engine startup state")? - { - bail!( - "{}", - managed_engine_startup_failure_detail( - &format!(" with status {status}"), - &record.log_path - ) - ); - } + .context("failed to check managed engine startup state")?; + fail_if_engine_died_at_startup(startup_exit, &record.log_path)?; child_pid }; + publish_spawned_engine(paths, service_id, record, child_pid) +} + +/// Put a spawned engine on the launch's record as `running`, as the last step of +/// the launch, or fail it when a stop or removal has settled the claim since. +/// +/// A failure leaves `record.engine_pid` naming the child, which is what +/// `abandon_failed_managed_launch` terminates; that is how the failure's own +/// "the failed launch terminates it" comes true. Shared by both platforms and +/// kept apart from the spawn so a test can drive it with a real child on every +/// lane, Windows included. +fn publish_spawned_engine( + paths: &AppPaths, + service_id: &str, + record: &mut ManagedServiceRecord, + child_pid: u32, +) -> Result { // The child is both roles until the engine state file names a separate // server process. Each role gets its own token, captured while the child is // alive: a stop pairs every recorded PID with its own token, and the two @@ -553,15 +663,46 @@ fn launch_managed_engine_child_locked( // engine PID recorded without one could only be signalled blind. // The engine's token was already captured right after the spawn (so cleanup // can bind to it); re-capturing here could replace it with a later owner's. - record.record_supervisor_identity(child_pid); - record.status = "running".to_owned(); - if let Err(error) = record.write() { - // Cleanup is the caller's job now: `record.engine_pid` (set above) is - // already enough for `abandon_failed_managed_launch` to terminate this - // child, so there is nothing left for this inner path to do beyond - // surfacing the error. - return Err(error.context("failed to publish the managed service record")); - } + // + // Applied to the record as it is on disk, and only while it is still the + // claim this launch wrote: a `rocm services stop` or `remove` does not take + // the launch lock, so it can settle the claim while the child is spawned. + // Writing the held copy back would revive that service as `running`. + // + // Only the fields this launch learned after its claim are applied: the engine + // and supervisor identities and the `running` status. Everything else is + // taken from the record on disk, which is the claim unless somebody wrote + // since — and a write that left the claim this launch's, such as the stop + // marker a stop records before its `stopped`, must survive the publish. So + // a field the launch sets on `record` after the claim write has to be added + // here, or it is dropped when `record` is replaced by what was published; + // `the_publish_applies_the_launch_s_own_fields_over_the_record_on_disk` + // pins which fields those are. + // + // On any error here cleanup is the caller's job: `record.engine_pid` (set + // above) is enough for `abandon_failed_managed_launch` to terminate this + // child, which nothing on record names when the claim was taken away. + let (engine_pid, engine_start_ticks) = (record.engine_pid, record.engine_start_ticks); + let published = update_managed_record_on_disk( + paths, + service_id, + |current| launch_still_owns(current, record), + |current| { + current.engine_pid = engine_pid; + current.engine_start_ticks = engine_start_ticks; + current.record_supervisor_identity(child_pid); + current.status = "running".to_owned(); + }, + ) + .context("failed to publish the managed service record")?; + let RecordUpdate::Written(published) = published else { + bail!( + "managed service `{service_id}` was stopped or removed while it was launching, so \ + the engine it started (pid {child_pid}) is not recorded on it; the failed launch \ + terminates it" + ); + }; + *record = *published; Ok(child_pid) } @@ -571,32 +712,46 @@ mod tests { use super::*; use crate::managed_service_status::load_managed_services; use crate::service_launch::launch_transaction::live_managed_services; - #[cfg(not(windows))] use crate::service_launch::test_support::sleeper_test_name; use crate::service_launch::test_support::write_live_service_record; use crate::tests::test_paths; use rocm_core::DEFAULT_LOCAL_HOST; + /// An `ExitStatus` for `code`, built the way each platform's startup + /// check gets one: `try_wait` on Unix, `GetExitCodeProcess` on Windows. + fn exit_status(code: u8) -> ExitStatus { + #[cfg(unix)] + { + use std::os::unix::process::ExitStatusExt as _; + ExitStatus::from_raw(i32::from(code) << 8) + } + #[cfg(windows)] + { + use std::os::windows::process::ExitStatusExt as _; + ExitStatus::from_raw(u32::from(code)) + } + } + #[test] fn a_child_that_exited_during_the_startup_window_is_reported_with_its_log() -> Result<()> { let dir = tempfile::tempdir()?; let log = dir.path().join("engine.log"); fs::write(&log, "boom: engine could not start\n")?; - let settled = std::cell::Cell::new(false); - let error = ensure_spawned_child_survives_startup( - 4242, - &log, - || settled.set(true), - |pid| { - assert_eq!(pid, 4242); - assert!(settled.get(), "liveness must be probed after the wait"); - false - }, - ) - .expect_err("a dead child must fail the launch"); - assert!(settled.get(), "the startup window must elapse first"); - assert!(format!("{error:#}").contains("boom"), "{error:#}"); - ensure_spawned_child_survives_startup(4242, &log, || {}, |_| true)?; + let error = fail_if_engine_died_at_startup(Some(exit_status(2)), &log) + .expect_err("a dead child must fail the launch"); + let message = format!("{error:#}"); + assert!(message.contains("boom"), "{message}"); + assert!(message.contains(&log.display().to_string()), "{message}"); + // The exit status is named on both platforms, not only where a `Child` + // is at hand. + let status_clause = format!(" with status {}", exit_status(2)); + assert!(message.contains(&status_clause), "{message}"); + // Typed, so cleanup knows not to signal a pid that is already free. + assert!(engine_died_at_startup(&error)); + fail_if_engine_died_at_startup(None, &log)?; + assert!(!engine_died_at_startup(&anyhow::anyhow!( + "some other launch failure" + ))); Ok(()) } @@ -982,19 +1137,20 @@ mod tests { } #[test] - fn a_failed_spawn_past_the_first_write_terminates_the_child_and_keeps_an_abandoned_record() + fn an_engine_that_dies_at_startup_keeps_an_abandoned_record_and_is_never_signalled() -> Result<()> { // Drives the real call site's *post*-spawn cleanup branch, which // neither `abandoned_launch_*` test reaches (they call // `abandon_failed_managed_launch` directly). An engine name this // binary does not build in makes the real spawned child (a second // `rocm` process) exit almost immediately via - // `run_builtin_engine_serve_http`'s `other => bail!(...)` arm, after - // `launch_managed_engine_child_locked` has already set - // `record.engine_pid` — so cleanup must terminate (already-dead) the - // child, keep the record and log for inspection, mark the record - // `stopped` (a `failed` one would be respawned by rocmd recovery), and - // clear only the endpoint key. + // `run_builtin_engine_serve_http`'s `other => bail!(...)` arm. The + // startup check sees it exit (`try_wait` on Unix, the watched spawn's + // exit code on Windows), so cleanup must keep the record and log for + // inspection, mark the record `stopped` (a `failed` one would be + // respawned by rocmd recovery), clear only the endpoint key, and never + // signal the engine: its pid is free. This drives the real mapping from + // the startup check's error to that cleanup, on every platform. // // The child always dies near-instantly, but `spawn_managed_engine_child` // only reports that within a fixed startup-check window. On a loaded or @@ -1097,7 +1253,53 @@ mod tests { live.is_empty(), "an abandoned record must not still reserve its port" ); + assert_eq!( + persisted.engine_pid, None, + "the record must not name an engine seen dead: its pid is free" + ); + let audit = fs::read_to_string(paths.audit_events_path()).unwrap_or_default(); let _ = fs::remove_dir_all(root); + assert!( + !audit.contains("managed_service_launch_cleanup"), + "cleanup must not signal an engine the startup check saw die:\n{audit}" + ); + Ok(()) + } + + #[test] + fn a_watched_spawn_seen_dead_is_never_put_on_record() -> Result<()> { + // The Windows ordering, on every lane: the startup check runs before + // the pid is recorded, so an engine seen dead is never named. + let dir = tempfile::tempdir()?; + let (_root, paths) = test_paths("adopt-watched-spawn"); + let mut record = write_live_service_record( + &paths, + "lemonade-adopt-watched", + "adopt-watched-model", + 11_499, + "starting", + )?; + record.engine_pid = None; + record.log_path = dir.path().join("engine.log"); + fs::write(&record.log_path, "engine: bad flag\n")?; + + let error = adopt_watched_engine_spawn(&mut record, 4242, Some(exit_status(2))) + .expect_err("an engine seen dead must fail the launch"); + assert!(engine_died_at_startup(&error), "{error:#}"); + assert_eq!( + record.engine_pid, None, + "a dead engine's pid was put on record" + ); + + assert_eq!( + adopt_watched_engine_spawn(&mut record, std::process::id(), None)?, + std::process::id() + ); + assert_eq!( + record.engine_pid, + Some(std::process::id()), + "a live engine must be on record before anything else can fail" + ); Ok(()) } @@ -1301,4 +1503,315 @@ mod tests { result?; Ok(()) } + + /// A launch's claim written to disk, with the engine answer and port lease + /// `launch_managed_engine_child_locked` takes, for tests driving it through + /// the injected spawn. + #[cfg(not(windows))] + fn claimed_launch( + paths: &AppPaths, + service_id: &str, + ) -> Result<( + ManagedServiceRecord, + ResolveModelResponse, + rocm_core::LoopbackPortLease, + )> { + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + let port = std::net::TcpListener::bind(("127.0.0.1", 0))? + .local_addr()? + .port(); + let lease = rocm_core::lease_loopback_port(port)?; + let record = ManagedServiceRecord::new( + paths, + service_id, + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + port, + "managed", + 0, + None, + None, + None, + ); + record.write()?; + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + Ok((record, resolve, lease)) + } + + /// The sleeper child the injected spawn starts in place of an engine. + #[cfg(not(windows))] + fn spawn_sleeper() -> io::Result { + ProcessCommand::new(std::env::current_exe().expect("test binary")) + .arg(sleeper_test_name()) + .arg("--exact") + .arg("--ignored") + .env("ROCM_TEST_SLEEPER_CHILD", "1") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + } + + /// The step after the spawn, on every platform and so on the Windows gate: + /// a stop that settled the claim while the engine spawned fails the publish, + /// keeps its `stopped` and stop marker, and the cleanup the failure promises + /// ("the failed launch terminates it") terminates the real child through the + /// same routine a stop uses — a tree kill on Windows. + #[test] + fn a_publish_refused_by_a_stop_leaves_the_stop_and_the_cleanup_ends_the_engine() -> Result<()> { + let (root, paths) = test_paths("publish-refused-by-stop"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-publish-refused", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11_489, + "managed", + 0, + None, + None, + None, + ); + record.write()?; + // `std::process` paths spelled out: this module imports `Command` and + // `Stdio` only off Windows, and this test runs on Windows too. + let mut child = std::process::Command::new(std::env::current_exe()?) + .arg("--exact") + .arg(sleeper_test_name()) + .arg("--ignored") + .arg("--test-threads=1") + .env("ROCM_TEST_SLEEPER_CHILD", "1") + .stdin(std::process::Stdio::null()) + .stdout(std::process::Stdio::null()) + .stderr(std::process::Stdio::null()) + .spawn()?; + let child_pid = child.id(); + // As both platforms do before the publish: the engine is on the held + // record with its token, so cleanup knows what to terminate. + record.record_engine_identity(child_pid); + // `rocm services stop` settles the claim meanwhile; it takes no launch lock. + let mut stopped = ManagedServiceRecord::read_manifest(&paths, &record.service_id)?; + stopped.stop_requested_unix_ms = Some(4_244); + stopped.status = "stopped".to_owned(); + stopped.write()?; + + let result = + publish_spawned_engine(&paths, &record.service_id.clone(), &mut record, child_pid); + let error = result.as_ref().err().map(|error| format!("{error:#}")); + abandon_failed_managed_launch(&paths, &record, false); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + let mut exited = false; + while std::time::Instant::now() < deadline { + if child.try_wait()?.is_some() { + exited = true; + break; + } + thread::sleep(Duration::from_millis(50)); + } + if !exited { + let _ = child.kill(); + let _ = child.wait(); + } + let after = ManagedServiceRecord::read_manifest(&paths, &record.service_id); + let _ = fs::remove_dir_all(root); + let after = after?; + let error = error.expect("a publish over a settled stop must fail"); + assert!( + error.contains("was stopped or removed while it was launching") + && error.contains(&format!("(pid {child_pid})")) + && error.contains("the failed launch terminates it"), + "{error}" + ); + assert!( + exited, + "the engine the failure says is terminated is still running" + ); + assert_eq!(after.status, "stopped", "the stop was overwritten"); + assert_eq!( + after.stop_requested_unix_ms, + Some(4_244), + "the stop marker was erased" + ); + assert_eq!( + after.engine_pid, None, + "the refused engine was put on record" + ); + Ok(()) + } + + /// The publish writes the launch's own fields onto the record as it is on + /// disk, not the copy the launch holds: a write made since the claim that + /// left it the launch's (here the stop marker a stop records before its + /// `stopped`) survives, and the published record differs from the claim in + /// exactly the engine and supervisor identities and the status. The held + /// copy is replaced by what was published, so the two agree. + #[test] + #[cfg(not(windows))] + fn the_publish_applies_the_launch_s_own_fields_over_the_record_on_disk() -> Result<()> { + let (root, paths) = test_paths("launch-publish-fields"); + let (mut record, resolve, lease) = claimed_launch(&paths, "lemonade-publish-fields")?; + let manifest_paths = paths.clone(); + let service_id = record.service_id.clone(); + let spawn_override: &dyn Fn(&mut ProcessCommand) -> io::Result = + &|_command| { + let mut marked = ManagedServiceRecord::read_manifest(&manifest_paths, &service_id) + .expect("claim on disk"); + marked.stop_requested_unix_ms = Some(4_243); + marked.write().expect("write the stop marker"); + spawn_sleeper() + }; + let claim = serde_json::to_value(&record)?; + + let result = launch_managed_engine_child_locked( + &paths, + &mut record, + lease, + "lemonade", + "lemonade-publish-fields", + &resolve, + "127.0.0.1", + &resolve.device_policy, + &[], + None, + None, + None, + Some(spawn_override), + ); + if let Some(pid) = record.engine_pid { + let _ = rocm_core::terminate_process(pid); + } + let on_disk = ManagedServiceRecord::read_manifest(&paths, &record.service_id); + let _ = fs::remove_dir_all(root); + let child_pid = result?; + let on_disk = serde_json::to_value(on_disk?)?; + + let mut expected = claim; + expected["stop_requested_unix_ms"] = serde_json::json!(4_243); + expected["engine_pid"] = serde_json::json!(child_pid); + expected["supervisor_pid"] = serde_json::json!(child_pid); + expected["engine_start_ticks"] = on_disk["engine_start_ticks"].clone(); + expected["supervisor_start_ticks"] = on_disk["supervisor_start_ticks"].clone(); + expected["status"] = serde_json::json!("running"); + assert_eq!( + on_disk, expected, + "the publish changed a field it does not own" + ); + assert_eq!( + serde_json::to_value(&record)?, + on_disk, + "the held copy disagrees" + ); + Ok(()) + } + + #[test] + #[cfg(not(windows))] + fn a_stop_settled_while_the_engine_spawns_is_kept_and_the_engine_terminated() -> Result<()> { + let (root, paths) = test_paths("launch-stopped-during-spawn"); + let (mut record, resolve, lease) = claimed_launch(&paths, "lemonade-stopped-during-spawn")?; + let manifest_paths = paths.clone(); + let service_id = record.service_id.clone(); + let spawn_override: &dyn Fn(&mut ProcessCommand) -> io::Result = + &|_command| { + // `rocm services stop` settles the claim while the child is + // being spawned; it takes no launch lock. + let mut stopped = ManagedServiceRecord::read_manifest(&manifest_paths, &service_id) + .expect("claim on disk"); + stopped.stop_requested_unix_ms = Some(4_242); + stopped.status = "stopped".to_owned(); + stopped.write().expect("write stop"); + ProcessCommand::new(std::env::current_exe().expect("test binary")) + .arg(sleeper_test_name()) + .arg("--exact") + .arg("--ignored") + .env("ROCM_TEST_SLEEPER_CHILD", "1") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + }; + + let result = launch_managed_engine_child_locked( + &paths, + &mut record, + lease, + "lemonade", + "lemonade-stopped-during-spawn", + &resolve, + "127.0.0.1", + &resolve.device_policy, + &[], + None, + None, + None, + Some(spawn_override), + ); + let child_pid = record.engine_pid.expect("the child is known to cleanup"); + let error = match result { + Ok(pid) => { + let _ = rocm_core::terminate_process(pid); + let _ = fs::remove_dir_all(&root); + panic!("a launch whose record was stopped must not report success"); + } + Err(error) => error, + }; + // What `spawn_managed_engine_child` does with that error. + abandon_failed_managed_launch(&paths, &record, engine_died_at_startup(&error)); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while super::super::readiness::service_pid_is_alive(child_pid) + && std::time::Instant::now() < deadline + { + thread::sleep(Duration::from_millis(50)); + } + let child_alive = super::super::readiness::service_pid_is_alive(child_pid); + if child_alive { + let _ = rocm_core::terminate_process(child_pid); + } + let after = ManagedServiceRecord::read_manifest(&paths, &record.service_id); + let _ = fs::remove_dir_all(root); + let after = after?; + assert!( + error + .to_string() + .contains("was stopped or removed while it was launching"), + "{error:#}" + ); + assert_eq!(after.status, "stopped", "the stop was overwritten"); + assert_eq!( + after.stop_requested_unix_ms, + Some(4_242), + "the stop marker was erased" + ); + assert_eq!( + after.supervisor_pid, 0, + "the unrecorded child was put on record" + ); + assert!( + !child_alive, + "the engine nothing records must be terminated" + ); + Ok(()) + } } diff --git a/apps/rocm/src/service_launch/test_support.rs b/apps/rocm/src/service_launch/test_support.rs index 70a5504e0..5366b5f70 100644 --- a/apps/rocm/src/service_launch/test_support.rs +++ b/apps/rocm/src/service_launch/test_support.rs @@ -6,9 +6,7 @@ use anyhow::Result; use rocm_core::{AppPaths, ManagedServiceRecord}; -#[cfg(not(windows))] use std::thread; -#[cfg(not(windows))] use std::time::Duration; /// A record with every reuse-relevant field set to the "reusable" value, so @@ -64,9 +62,10 @@ pub(super) fn write_live_service_record( /// process that outlives the spawn they are checking. It is `#[ignore]`d so a /// normal run skips it, and it only sleeps when the tests that spawn it set /// `ROCM_TEST_SLEEPER_CHILD`. They select it by [`sleeper_test_name`]. +// Not gated to Unix: `an_engine_seen_dead_at_startup_is_never_signalled` in +// `launch_transaction` spawns it on Windows too. #[test] -#[cfg(not(windows))] -#[ignore = "spawned as a child process by abandoned_launch_terminates_the_spawned_child"] +#[ignore = "spawned as a child process by the abandoned-launch tests"] fn managed_launch_cleanup_sleeper_child() { if std::env::var_os("ROCM_TEST_SLEEPER_CHILD").is_some() { thread::sleep(Duration::from_mins(2)); @@ -81,7 +80,6 @@ fn managed_launch_cleanup_sleeper_child() { /// cannot be left naming a test that no longer exists. libtest names a test by /// its path inside the crate, while the item path starts with the crate name, /// which is dropped. The assertions check that against the module path as well. -#[cfg(not(windows))] pub(super) fn sleeper_test_name() -> String { let item_path = std::any::type_name_of_val(&managed_launch_cleanup_sleeper_child); let name = item_path diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 90385d208..3ff85fcee 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -198,17 +198,17 @@ pub use proc_lifecycle::{ terminate_recorded_pid, terminate_recorded_pid_within, terminate_recorded_service_pids, terminate_verified, }; +#[cfg(windows)] +pub use process::{ + DetachedSpawn, spawn_detached_no_inherit, spawn_detached_no_inherit_watching_startup, + spawn_hidden_console_no_inherit, spawn_hidden_console_with_log, wait_for_process_exit, +}; pub use process::{ FileLock, LoopbackPortLease, MANAGED_LAUNCH_LOCK_TIMEOUT, ProcessProbe, detach_command_session, is_canonical_loopback_host, launch_port_argument, lease_loopback_port, lease_loopback_port_or_bind_error, probe_process, process_is_running, terminate_process, terminate_process_tree, windows_system_tool, }; -#[cfg(windows)] -pub use process::{ - spawn_detached_no_inherit, spawn_hidden_console_no_inherit, spawn_hidden_console_with_log, - wait_for_process_exit, -}; pub use quoting::{posix_shell_quote, powershell_quote}; pub use report::{ APPROVED_ARCHITECTURES, APPROVED_ARCHITECTURES_SOURCE, GROUP_AXES, REPORT_SCHEMA_VERSION, @@ -265,6 +265,14 @@ pub use vllm_oom::{ vllm_oom_display_line, }; +/// E2E-only switch that makes `rocm serve`'s managed engine die during startup. +/// +/// Honoured only by a `rocm` binary built with the `e2e-test-hooks` feature; a +/// release build has no code that reads it. Defined here, rather than in the +/// binary that reads it, because the E2E suite sets it from another crate: one +/// constant both sides name cannot drift the way two copies of the string could. +pub const MANAGED_ENGINE_STARTUP_FAILURE_TEST_ENV: &str = "ROCM_E2E_MANAGED_ENGINE_STARTUP_FAILURE"; + /// The variable that opts a machine out of rocm-cli choosing its runtime's torch. pub const TORCH_ALIGNMENT_DISABLED_ENV: &str = "ROCM_CLI_DISABLE_TORCH_ALIGNMENT"; diff --git a/crates/rocm-core/src/process.rs b/crates/rocm-core/src/process.rs index 5751d17c7..54f937031 100644 --- a/crates/rocm-core/src/process.rs +++ b/crates/rocm-core/src/process.rs @@ -43,6 +43,27 @@ mod bounded; pub use bounded::run_with_timeout; pub(crate) use bounded::{MAX_QUOTED_OUTPUT_CHARS, run_bounded}; +/// Outcome of a detached Windows spawn that was briefly watched for an early exit. +/// +/// The observation runs while the process handle returned by `CreateProcessW` is +/// still open, so a reported exit code is always the exit code of the process that +/// was just spawned. Re-opening the process by PID after the fact could not offer +/// that guarantee — Windows recycles PIDs, so by then the PID may name an +/// unrelated process. +#[cfg(windows)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DetachedSpawn { + /// PID of the spawned process. + pub pid: u32, + /// `Some(code)` if the process exited at any point within the observation + /// window — known as soon as it exits, since the wait returns then rather + /// than at the end of the window — and `None` if it was still running when + /// the window ran out. `None` is also reported if the wait failed or the + /// exit code could not be read, which degrades to the unwatched behaviour + /// rather than inventing a failure. + pub early_exit_code: Option, +} + #[cfg(windows)] pub fn spawn_detached_no_inherit( program: &Path, @@ -56,6 +77,38 @@ pub fn spawn_detached_no_inherit( DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW | CREATE_UNICODE_ENVIRONMENT, false, None, + None, + ) + .map(|spawn| spawn.pid) +} + +/// As [`spawn_detached_no_inherit`], but watches the new process for up to +/// `settle` before returning, so the caller can tell "started" apart from +/// "started and died immediately". +/// +/// Creation flags and handle inheritance are identical to +/// [`spawn_detached_no_inherit`]: the child stays fully detached and outlives this +/// process. The wait is bounded and delays the caller by at most `settle` — less +/// when the child exits sooner; it is not a join. +/// +/// This exists because the detached spawn primitives hand back a bare PID rather +/// than a `std::process::Child`, so a caller has no equivalent of `try_wait()` to +/// notice a child that failed during startup. +#[cfg(windows)] +pub fn spawn_detached_no_inherit_watching_startup( + program: &Path, + args: &[String], + env_overrides: &[(&str, &Path)], + settle: Duration, +) -> Result { + spawn_windows_no_inherit( + program, + args, + env_overrides, + DETACHED_PROCESS | CREATE_NEW_PROCESS_GROUP | CREATE_NO_WINDOW | CREATE_UNICODE_ENVIRONMENT, + false, + None, + Some(settle), ) } @@ -72,7 +125,9 @@ pub fn spawn_hidden_console_no_inherit( CREATE_NEW_CONSOLE | CREATE_NEW_PROCESS_GROUP | CREATE_UNICODE_ENVIRONMENT, true, None, + None, ) + .map(|spawn| spawn.pid) } #[cfg(windows)] @@ -141,12 +196,13 @@ pub fn spawn_hidden_console_with_log( CREATE_NEW_CONSOLE | CREATE_NEW_PROCESS_GROUP | CREATE_UNICODE_ENVIRONMENT, true, Some((stdout_handle, stderr_handle)), + None, ); unsafe { CloseHandle(stdout_handle); CloseHandle(stderr_handle); } - result + result.map(|spawn| spawn.pid) } #[cfg(windows)] @@ -879,7 +935,8 @@ fn spawn_windows_no_inherit( windows_sys::Win32::Foundation::HANDLE, windows_sys::Win32::Foundation::HANDLE, )>, -) -> Result { + settle: Option, +) -> Result { use std::ptr::{null, null_mut}; use windows_sys::Win32::Foundation::CloseHandle; @@ -923,11 +980,63 @@ fn spawn_windows_no_inherit( std::io::Error::last_os_error() ); } + // Observe the child *before* the handle is closed. Waiting on this handle is + // race-free; re-opening the process later by PID would not be, since Windows + // recycles PIDs and the PID could by then belong to something else. + let early_exit_code = + settle.and_then(|settle| unsafe { observe_early_exit(process_info.hProcess, settle) }); unsafe { CloseHandle(process_info.hThread); CloseHandle(process_info.hProcess); } - Ok(process_info.dwProcessId) + Ok(DetachedSpawn { + pid: process_info.dwProcessId, + early_exit_code, + }) +} + +/// Wait up to `settle` for `process` to exit, reporting its exit code if it did. +/// +/// Returns `None` both when the process is still running and when the exit code +/// could not be read, so a failed query degrades to "assume it is alive" rather +/// than reporting a startup failure that may not have happened. +/// +/// # Safety +/// +/// `process` must be an open process handle that is not closed for the duration +/// of the call; this function does not close it. The only caller passes the +/// `hProcess` that `CreateProcessW` just returned, which carries full access +/// (`PROCESS_ALL_ACCESS`), so the rights the two calls need are always present: +/// `SYNCHRONIZE` for `WaitForSingleObject`, and `PROCESS_QUERY_LIMITED_INFORMATION` +/// (or the broader `PROCESS_QUERY_INFORMATION`) for `GetExitCodeProcess`. +#[cfg(windows)] +#[allow(unsafe_code)] // Win32 FFI +unsafe fn observe_early_exit( + process: windows_sys::Win32::Foundation::HANDLE, + settle: Duration, +) -> Option { + use windows_sys::Win32::Foundation::WAIT_OBJECT_0; + + if unsafe { WaitForSingleObject(process, wait_timeout_millis(settle)) } != WAIT_OBJECT_0 { + return None; + } + let mut exit_code: u32 = 0; + if unsafe { GetExitCodeProcess(process, &raw mut exit_code) } == 0 { + return None; + } + Some(exit_code) +} + +/// Clamp a wait budget to the `u32` milliseconds `WaitForSingleObject` takes, +/// never yielding `INFINITE`. +/// +/// `INFINITE` is `u32::MAX`, so a saturating conversion of a long duration would +/// silently turn a bounded startup check into one that blocks until the child +/// exits. Cap one millisecond below it instead. +#[cfg(any(windows, test))] +fn wait_timeout_millis(budget: Duration) -> u32 { + const LONGEST_FINITE_WAIT_MS: u128 = (u32::MAX - 1) as u128; + budget.as_millis().min(LONGEST_FINITE_WAIT_MS) as u32 } #[cfg(windows)] @@ -1281,6 +1390,86 @@ mod tests { ); } + #[test] + fn wait_timeout_millis_keeps_the_startup_wait_bounded() { + assert_eq!(wait_timeout_millis(Duration::from_millis(200)), 200); + assert_eq!(wait_timeout_millis(Duration::ZERO), 0); + // Sub-millisecond budgets truncate to a poll rather than rounding up. + assert_eq!(wait_timeout_millis(Duration::from_micros(900)), 0); + // A budget past the `u32` millisecond range must never land on + // `INFINITE` (`u32::MAX`), which would block until the child exits. + let huge = wait_timeout_millis(Duration::from_secs(u64::from(u32::MAX))); + assert_eq!(huge, u32::MAX - 1); + assert_ne!(huge, u32::MAX); + } + + /// Watching a detached spawn must surface a child that died during startup. + /// Only meaningful on Windows: the detached spawn primitives return a bare + /// PID there, so this wait is the only chance to notice the exit. + /// + /// It also pins what `DetachedSpawn::early_exit_code` documents: the exit is + /// reported when it happens, not when the window runs out. The window is far + /// longer than `cmd /C exit` takes, so a wait that sat it out fails the + /// elapsed-time check rather than passing slowly. + #[cfg(windows)] + #[test] + fn watched_detached_spawn_reports_a_child_that_exits_immediately() { + let window = Duration::from_secs(30); + let started = std::time::Instant::now(); + // Each token is a separate argument so the assembled command line needs no + // quoting, keeping `cmd.exe`'s quote-stripping rules out of the test. + let spawn = spawn_detached_no_inherit_watching_startup( + &test_system32_tool("cmd.exe"), + &["/C".to_owned(), "exit".to_owned(), "7".to_owned()], + &[], + window, + ) + .expect("spawn should succeed"); + let elapsed = started.elapsed(); + assert_eq!(spawn.early_exit_code, Some(7)); + assert_ne!(spawn.pid, 0); + assert!( + elapsed < window / 2, + "the exit must be reported as soon as the child exits, not at the end of \ + the {window:?} window (took {elapsed:?})" + ); + } + + /// The converse: a child that is still running must not be reported as a + /// startup failure, or every successful launch would be rejected. + #[cfg(windows)] + #[test] + fn watched_detached_spawn_does_not_report_a_child_that_keeps_running() { + // `ping` against localhost is the dependency-free Windows sleep, and ~9s + // outlasts the 200 ms observation window by well over an order of + // magnitude. Spawned directly rather than through `cmd /C` so the + // returned PID is the process the test has to clean up: on Windows + // `terminate_process_tree` is plain `terminate_process`, so an + // intermediate shell would leave `ping` orphaned. + let spawn = spawn_detached_no_inherit_watching_startup( + &test_system32_tool("ping.exe"), + &["-n".to_owned(), "10".to_owned(), "127.0.0.1".to_owned()], + &[], + Duration::from_millis(200), + ) + .expect("spawn should succeed"); + // Clean up before asserting, so a failing assertion cannot leak the child. + let early_exit_code = spawn.early_exit_code; + let was_running = process_is_running(spawn.pid); + let _ = terminate_process(spawn.pid); + assert_eq!(early_exit_code, None); + assert!(was_running); + } + + /// `CreateProcessW` is called with an explicit application name, so it does + /// no `PATH` search — the program has to be a full path. + #[cfg(windows)] + fn test_system32_tool(exe: &str) -> PathBuf { + PathBuf::from(std::env::var_os("SystemRoot").unwrap_or_else(|| r"C:\Windows".into())) + .join("System32") + .join(exe) + } + #[test] fn file_lock_creates_missing_parent_dirs_and_lock_file() { let dir = diff --git a/docs/architecture.md b/docs/architecture.md index b929c3165..74ce52ec7 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -114,14 +114,14 @@ The `rocm` binary: the clap command tree, a handler module for most commands, an | [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs) | The pure transforms `serve()` applies to an engine's launch recipe hint: the catalog mapping, then `--tool-call-parser`, `--gpu-memory-utilization`, generation defaults and `--engine-env`. | relocation | | [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs) | The engine-agnostic `rocm serve` deployment summary: a status indicator while the server starts, an inference smoke test, and the endpoint and timing table. | domain | | [`service_control.rs`](../apps/rocm/src/service_control.rs) | Stopping and restarting a managed local server (`rocm services stop` and `restart`, runtime moves) and the internal sandbox-tool runner (`run_internal_sandbox_tool`). Its stop path, `stop_recorded_processes`, hands the recorded PIDs to the routine [`proc_lifecycle.rs`](../crates/rocm-core/src/proc_lifecycle.rs) shares with `rocmd`. | relocation | -| [`service_hints.rs`](../apps/rocm/src/service_hints.rs) | The shared builder of the hint text that tells a user how to stop, restart or remove a service with `--yes`, plus the forced-removal hint (`service_forced_remove_hint`, `rocm services remove --force --yes`) and the command word, verb and past tense for a service tool. | flat | +| [`service_hints.rs`](../apps/rocm/src/service_hints.rs) | The shared builder of the hint text that tells a user how to stop, restart or remove a service with `--yes`, plus the forced-removal hint (`service_forced_remove_hint`, `rocm services remove --force --yes`), the `rocm logs --service ` hints with and without `--follow` (`service_logs_hint`, `service_logs_follow_hint`), and the command word, verb and past tense for a service tool. | flat | | [`service_launch.rs`](../apps/rocm/src/service_launch.rs) | Front of the `service_launch/` children: their declarations and the `pub(crate) use` re-exports other modules call, with no code of its own. | relocation | | [`service_launch/attach.rs`](../apps/rocm/src/service_launch/attach.rs) | The attached serve path (`--verbose`/`--foreground`): spawn the managed child and stream its log in the terminal until the user detaches or stops it. | relocation | | [`service_launch/endpoint_auth.rs`](../apps/rocm/src/service_launch/endpoint_auth.rs) | Bind-host validation and the endpoint API key checks that gate a public bind (`validate_bind_host`, `resolve_endpoint_auth`), including on a respawn. | relocation | | [`service_launch/helper_autostart.rs`](../apps/rocm/src/service_launch/helper_autostart.rs) | Starting the background automation helper when a watcher needs it, with the autostart claim that keeps concurrent callers from spawning two. | relocation | -| [`service_launch/launch_transaction.rs`](../apps/rocm/src/service_launch/launch_transaction.rs) | The locked launch decision: port allocation against live records and the OS, failed-launch cleanup, and whether a live service may answer a `serve` request. The cleanup terminates a spawned child through the same `rocm_core::terminate_recorded_pid_within` routine a stop uses, so on Windows its descendants are taken down too. A spawned child whose start token was not captured, on a host that records tokens (Linux, and Windows, which records the process creation time), is not signalled (`launch_cleanup_refusal`): it was gone before it could be captured and its pid may belong to another process. | relocation | -| [`service_launch/readiness.rs`](../apps/rocm/src/service_launch/readiness.rs) | Waiting on a launched or live managed service: the readiness budget, process liveness, how a wait ends, and the fault-injection seam (an OOM launch, or a ready launch) for end-to-end tests. | relocation | -| [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs) | Spawning the detached managed engine child both serve paths share: process and stdio plumbing and the serialized spawn transaction (`spawn_managed_engine_child`). | relocation | +| [`service_launch/launch_transaction.rs`](../apps/rocm/src/service_launch/launch_transaction.rs) | The locked launch decision: port allocation against live records and the OS, failed-launch cleanup, and whether a live service may answer a `serve` request. The cleanup terminates a spawned child through the same `rocm_core::terminate_recorded_pid_within` routine a stop uses, so on Windows its descendants are taken down too. A spawned child whose start token was not captured, on a host that records tokens (Linux, and Windows, which records the process creation time), is not signalled (`launch_cleanup_refusal`): it was gone before it could be captured and its pid may belong to another process. Nor is an engine the startup check saw die: its pid is free, but its record and log are kept. `update_managed_record_on_disk` is the guarded read-modify-write every launch write after the claim goes through (the `running` record, the cleanup's `stopped` mark, the readiness verdict), applied only while `launch_still_owns` (or, for the readiness verdict, the record still reading as live) accepts the record on disk, so a `rocm services stop` or `remove`, which do not take the launch lock, is not overwritten or undone unless it lands between that read and the write (nothing that writes a service record locks it). It reports whether it wrote, found the record no longer the launch's, or found it gone (`RecordUpdate`). The cleanup clears the endpoint key only when its `stopped` mark applied or the record is gone, so a restart that took the record over keeps its key. | relocation | +| [`service_launch/readiness.rs`](../apps/rocm/src/service_launch/readiness.rs) | Waiting on a launched or live managed service: the readiness budget, process liveness, how a wait ends, and the fault-injection seam (an OOM launch, or a ready launch) for end-to-end tests. A record a stop settled during the wait fails the serve saying it was stopped while starting, one settled without a stop marker fails it saying the server exited, and a record removed during it fails the serve rather than being re-created. | relocation | +| [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs) | Spawning the detached managed engine child both serve paths share: process and stdio plumbing and the serialized spawn transaction (`spawn_managed_engine_child`). The startup check (`fail_if_engine_died_at_startup`, settle window `MANAGED_ENGINE_STARTUP_SETTLE`) fails a launch whose engine exited inside the window with the typed `EngineDiedAtStartup`; on Windows it goes through `adopt_watched_engine_spawn`, which records the engine's pid only after the check passes, and reads the exit code through the process handle (`rocm_core::spawn_detached_no_inherit_watching_startup`) rather than re-checking a pid Windows may have reused. `scripted_managed_engine_startup_failure` is the `e2e-test-hooks` switch that scripts that death. | relocation | | [`service_launch/start.rs`](../apps/rocm/src/service_launch/start.rs) | The background launch path (`start_managed_service`) and the plain line-by-line rendering of its launch report. | relocation | | [`service_launch/test_support.rs`](../apps/rocm/src/service_launch/test_support.rs) | Test fixtures shared by more than one `service_launch/` child. | flat | | [`service_records.rs`](../apps/rocm/src/service_records.rs) | Removing a managed-service record and every file it owns — including any copy of its endpoint key an interrupted save left behind, found with one listing of the services folder per command (`staged_endpoint_keys_by_service`) — and the `rocm services prune` plan, render and apply steps with their age gate. `remove_managed_service_record_with` takes `--force` and the read of each recorded PID: `--force` removes a record only when none of its recorded processes is confirmed running, one could not be queried and a stop was requested, and reports which. | relocation | @@ -194,7 +194,7 @@ Boundaries: **Shared sources and contracts** -- [`service_hints.rs`](../apps/rocm/src/service_hints.rs) is the shared builder of the `rocm services stop|restart|remove --yes` hint text, so the callers that use it cannot drift apart. Those callers are [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs), [`serve_port.rs`](../apps/rocm/src/serve_port.rs), [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs), [`service_control.rs`](../apps/rocm/src/service_control.rs), [`service_records.rs`](../apps/rocm/src/service_records.rs), [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs), [`services_text.rs`](../apps/rocm/src/services_text.rs) and, under `service_launch/`, [`attach.rs`](../apps/rocm/src/service_launch/attach.rs) and [`spawn.rs`](../apps/rocm/src/service_launch/spawn.rs); the parent [`service_launch.rs`](../apps/rocm/src/service_launch.rs) does not call it. +- [`service_hints.rs`](../apps/rocm/src/service_hints.rs) is the shared builder of the `rocm services stop|restart|remove --yes` and `rocm logs --service ` hint text, so the callers that use it cannot drift apart. Those callers are [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs), [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`serve_port.rs`](../apps/rocm/src/serve_port.rs), [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs), [`service_control.rs`](../apps/rocm/src/service_control.rs), [`service_records.rs`](../apps/rocm/src/service_records.rs), [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs), [`services_text.rs`](../apps/rocm/src/services_text.rs) and, under `service_launch/`, [`attach.rs`](../apps/rocm/src/service_launch/attach.rs), [`readiness.rs`](../apps/rocm/src/service_launch/readiness.rs) and [`spawn.rs`](../apps/rocm/src/service_launch/spawn.rs); the parent [`service_launch.rs`](../apps/rocm/src/service_launch.rs) does not call it. - `DEFAULT_ENGINE_MARKER` in [`examine_report_text.rs`](../apps/rocm/src/examine_report_text.rs) is shared by both engine-inventory renderers and their legend line, so the rendered marker and the legend cannot drift apart. - [`shell_quote.rs`](../apps/rocm/src/shell_quote.rs) is the CLI's quoter; `rocm-dash-tui` keeps a copy (`quote_display_arg`) because it may not depend on `rocm-core`, and a property test here holds the two equal. - The local-model thinking rule [`providers/wire.rs`](../apps/rocm/src/providers/wire.rs) applies lives in `rocm-dash-core`'s [`chat_template.rs`](../crates/rocm-dash-core/src/chat_template.rs), because the dashboard's chat client must decide it identically. diff --git a/docs/manual-testing.md b/docs/manual-testing.md index d425d7cd0..eaa688a83 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -873,6 +873,46 @@ outcome for the stopping process itself; the JSON answer of the sandbox `stop_server` tool does, in `pid_outcomes`, as `self` (never signalled, counted as stopped). +**An engine that dies at startup.** On Windows and Linux alike, a managed +engine that exits within its first 200 ms fails `rocm serve --managed` +instead of reporting a server that is still starting. +To see it without a broken engine, build the CLI with its test hooks and arm +the scripted death, which makes the spawned engine reject its arguments: + +```powershell +cargo build -p rocm --features e2e-test-hooks +$env:ROCM_E2E_MANAGED_ENGINE_STARTUP_FAILURE = "1" +./target/debug/rocm serve qwen --engine lemonade --managed +./target/debug/rocm services list --all +``` + +Expected result: + +- The serve exits non-zero with `managed engine exited immediately with status + ...; inspect `, where `` is the service's own log, plus a + tail of that log when the engine wrote anything to it. On Windows the status + is the engine's exit code too. +- `rocm services list --all` shows the attempt as `stopped`. Running the same + `rocm serve` again fails again with the same `managed engine exited + immediately ...` error, rather than reporting the earlier attempt as already + running, and `rocm services list --all` then shows two `stopped` records. +- An engine that dies later than that — while loading its runtime or model — + is not caught by this check; the serve then waits for readiness and reports + the service as still starting, as before. + +Unset the variable afterwards; a build without `--features e2e-test-hooks` +ignores it. + +To see the stop-during-start case, run `rocm serve --managed` in one +terminal and, while it is still waiting for the server, `rocm services stop + --yes` from another. The serve exits non-zero with an error saying +the managed service "was stopped while it was starting", and +`rocm services list --all` shows the service as `stopped`. This is the usual +result, not a certain one: the launch reads the record and writes its readiness +verdict as two separate steps, and a stop takes no lock on the record, so a stop +that lands between the two is overwritten by the verdict and the serve reports +the server as it found it. + #### Windows host check The Windows build-and-test job runs the workspace unit tests, which include two diff --git a/docs/testing.md b/docs/testing.md index f2e113962..b4fd68729 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2443,6 +2443,64 @@ same refusal on the required Windows lane. The vLLM live GPU acceptance script should return a clean skip on native Windows. It remains a strict GPU-required test on Linux/WSL. +Managed-launch startup-death and record-write focused tests: + +```bash +cargo test -p rocm --bin rocm service_launch::spawn::tests::a_child_that_exited_during_the_startup_window +cargo test -p rocm --bin rocm service_launch::launch_transaction::tests::an_engine_seen_dead_at_startup_is_never_signalled +cargo test -p rocm --bin rocm service_launch::launch_transaction::tests::an_engine_seen_dead_at_startup_is_never_handed +cargo test -p rocm --bin rocm service_launch::launch_transaction::tests::abandoning_a_launch_keeps_the_key +cargo test -p rocm --bin rocm service_launch::launch_transaction::tests::abandoning_a_launch_over_an_unreadable +cargo test -p rocm --bin rocm service_launch::launch_transaction::tests::abandoning_a_launch_does_not_recreate_a_removed_record +cargo test -p rocm --bin rocm service_launch::spawn::tests::a_stop_settled_while_the_engine_spawns +cargo test -p rocm --bin rocm service_launch::spawn::tests::a_publish_refused_by_a_stop +cargo test -p rocm --bin rocm service_launch::readiness::tests::a_stop_settled_during_the_wait +cargo test -p rocm --bin rocm service_launch::readiness::tests::record_wait_outcome_keeps_what_changed +cargo test -p rocm --bin rocm service_launch::readiness::tests::record_wait_outcome_does_not_recreate +cargo test -p rocm --bin rocm service_launch::spawn::tests::the_publish_applies +cargo test -p rocm --bin rocm service_launch::launch_transaction::tests::the_launch_does_not_publish_over +cargo test -p rocm --bin rocm service_launch::launch_transaction::tests::an_engine_seen_dead_before +cargo test -p rocm --bin rocm service_launch::spawn::tests::an_engine_that_dies_at_startup_keeps +cargo test -p rocm --bin rocm service_launch::spawn::tests::a_watched_spawn_seen_dead +cargo test -p rocm --bin rocm service_launch::readiness::tests::an_engine_that_exited_during_the_wait +cargo test -p rocm-core watched_detached_spawn # Windows only +``` + +They pin that an engine seen dead inside the 200 ms startup window fails the +launch naming its own log, on Windows from the exit code read through the +process handle `CreateProcessW` returned (not by re-checking a pid Windows may +already have reused); that cleanup never signals such an engine, since its pid is +free; and that the launch's writes after its claim — the `running` record, the +`stopped` mark of a failed launch, the readiness verdict — go to the record as it +is on disk, so a `rocm services stop` settled meanwhile keeps its `stopped` and +stop marker, and a `rocm services remove` is not undone by re-creating the record. +A serve whose server is stopped during its readiness wait fails (non-zero exit) saying +the service was stopped while it was starting, instead of reporting a launch; one whose +server exited on its own during the wait (a record settled without a stop marker) fails +saying it exited and naming `rocm logs --service `. Each of those writes reads +the record and then writes it, with no lock against a stop, so a stop landing +between the two is still overwritten: the tests pin the narrowed race, not a +closed one. The publish applies only the launch's own fields (engine and +supervisor identities, `running`) over the record on disk, and a failed launch's +cleanup leaves the endpoint key of a record a restart took over, or of one it +cannot read, which it audits. A publish refused by a stop, and the cleanup +terminating the engine it then names, run on the required Windows lane too +(`a_publish_refused_by_a_stop_*`, with a real child and a tree kill there). +The `watched_detached_spawn_*` tests run on the required Windows lane. + +`serve-61` (`@id:serve-managed-engine-dies-at-startup`) is the black-box half: an +`e2e-test-hooks` build scripts the managed engine's death (the spawned `rocm` +gets an argument it rejects), and the scenario checks that the serve fails naming +the engine's own log, that the record left on disk reads `stopped` and names no +pid, and that serving the same model again reaches the engine launch again. It +carries no host tag, so every lane built with the hooks runs it, including the +self-hosted Windows lane; the required `windows-build-and-test` lane builds +without the hooks and does not. Run it locally with: + +```bash +cargo xtask e2e -- -n "serve-61 " +``` + Serve resolver focused tests: ```bash diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 096bd0825..070c90871 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -980,3 +980,30 @@ Feature: Model serving And the deployment summary names the logs command for that service And the launch's service record is left as ready And no background automation daemon was left behind + + # A managed engine is spawned detached, so if it dies on the way up there is no + # terminal for it to report to: the child's own service log is the only account + # of why. The launch fails naming that log, and leaves its record `stopped` + # (not `failed`, which `rocmd`'s recovery would respawn) without naming the + # dead child: its pid is free, and a later stop or cleanup must not signal + # whatever process reuses it (on Windows no start-time token was captured for + # the dead engine that could tell the two apart). Serving the same model + # again then reaches the engine launch again. The engine is scripted to + # die at startup in test builds (`rocm/e2e-test-hooks`), which also waives the + # no-GPU pre-flight and engine preparation so this reaches a real spawn without + # GPU hardware or a runtime download — the death itself is real, only its + # trigger is scripted. + # + # Carries no host tag on purpose, so it runs on every lane built with the + # hooks, the self-hosted Windows lane among them: `@requires-no-gpu` would skip + # it on any host that HAS a GPU, and the premise is a dead engine, not an + # absent GPU. The required `windows-build-and-test` lane does not build the + # hooks, so it cannot run this; the Windows watch itself is covered there by + # `rocm-core`'s `watched_detached_spawn_*` unit tests. + @id:serve-managed-engine-dies-at-startup + Scenario: serve-61 - An engine that dies at startup fails the serve and names its log + Given the managed engine dies during startup + When the user serves a model with Lemonade + Then serving fails and names the engine's own log + When the user serves the same model again + Then that serve reaches the engine launch again and fails the same way diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 990bb6b18..b02987ac4 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -4042,6 +4042,160 @@ async fn no_background_automation_daemon_was_left_behind(world: &mut E2eWorld) { ); } +/// Arms the scripted startup death. Also waives serve's no-GPU pre-flight and +/// engine preparation, so the black-box test reaches a real spawn on a host with +/// neither GPU hardware nor an installed runtime. +#[given("the managed engine dies during startup")] +async fn managed_engine_dies_during_startup(world: &mut E2eWorld) { + world.command_env.push(( + rocm_core::MANAGED_ENGINE_STARTUP_FAILURE_TEST_ENV, + "1".into(), + )); +} + +#[then("serving fails and names the engine's own log")] +async fn assert_startup_death_names_log(world: &mut E2eWorld) { + let output = serve_output(world); + // A binary built without `rocm/e2e-test-hooks` never arms the scripted death: + // on a GPU lane it launches a real engine, which this exit-code check is the + // first to notice — so it names the feature as the likely cause. + assert_ne!( + world.cli_rc, + Some(0), + "an engine that died at startup must fail the serve (if it did not die at \ + all, check the binary under test was built with `rocm/e2e-test-hooks`):\n{output}" + ); + // See the note in assert_lemonade_preparation_retry_is_bounded: the seam that + // scripts the death also waives the no-GPU pre-flight, so this refusal can + // only appear when the binary under test was built without the feature. That + // diagnosis is specific to a host with no usable GPU, where the pre-flight is + // what stops such a binary. + assert!( + !output.contains("no usable AMD GPU detected"), + "serve stopped at the no-GPU pre-flight, so the binary under test was \ + built without the `rocm/e2e-test-hooks` feature and never reached the \ + engine launch:\n{output}" + ); + assert!( + output.contains("managed engine exited immediately"), + "expected the launch to report the engine's immediate exit:\n{output}" + ); + // The child is detached, so its own log is the only account of why it died — + // the user has to be told where it is. The path is read off the record the + // launch left behind, not matched as a `.log` suffix that any incidental + // mention of a log would satisfy; and it is the path `rocm` itself recorded, + // so a Windows 8.3 short form cannot make the comparison disagree with itself. + // + // The two platforms reach this through different branches of the message. + // On Unix the parent redirects the child's stdio into the log, so the dead + // child's output is there and the message carries a tail. On Windows the + // detached spawn does not redirect, and the scripted child never receives + // `--log`, so the log stays empty and only the no-tail branch runs. Naming + // the path is the claim both branches share, and it is the one pinned here. + let records = lemonade_service_records(world); + let log_paths: Vec<&str> = records + .iter() + .filter_map(|record| record["log_path"].as_str()) + .collect(); + assert!( + !log_paths.is_empty(), + "the failed launch left no lemonade service record" + ); + assert!( + log_paths.iter().any(|path| output.contains(path)), + "expected the failure to name the engine's own service log {log_paths:?}:\n{output}" + ); + assert_failed_launches_left_their_claims(&records, 1); +} + +/// Every lemonade service record in the scenario's isolated data folder, read +/// raw from disk. +fn lemonade_service_records(world: &E2eWorld) -> Vec { + let services = world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .join("data") + .join("services"); + std::fs::read_dir(&services) + .unwrap_or_else(|e| panic!("no services dir at {}: {e}", services.display())) + .filter_map(|entry| std::fs::read_to_string(entry.ok()?.path()).ok()) + .filter_map(|json| serde_json::from_str::(&json).ok()) + .filter(|record| record["engine"] == "lemonade") + .collect() +} + +/// `launches` failed launches of an engine that died at startup each left one +/// record: `stopped`, and still the claim as it was written. +fn assert_failed_launches_left_their_claims(records: &[serde_json::Value], launches: usize) { + // Read raw from disk, not through `rocm services`, whose liveness refresh + // would rewrite the very fields pinned here. `stopped`, not `failed`: + // `rocmd`'s recovery restarts a managed `failed` record that carries no stop + // marker, which would respawn the launch this serve just reported as failed. + let statuses: Vec<&str> = records + .iter() + .filter_map(|record| record["status"].as_str()) + .collect(); + assert_eq!( + statuses, + vec!["stopped"; launches], + "each failed launch must leave its record `stopped`, not claiming the engine \ + and model" + ); + // The startup check saw the child die, so the record never names it: the + // pid is free, and a later stop or cleanup must not signal whatever process + // reuses it. On Windows no start-time token was captured for the dead + // engine, so nothing else would tell that process from it. Both platforms + // leave the claim as it was written, before any child was put on record. + let pids: Vec<(serde_json::Value, serde_json::Value)> = records + .iter() + .map(|record| { + ( + record["supervisor_pid"].clone(), + record["engine_pid"].clone(), + ) + }) + .collect(); + assert_eq!( + pids, + vec![(serde_json::json!(0), serde_json::Value::Null); launches], + "a child the startup check saw die must not be named by the record" + ); +} + +#[when("the user serves the same model again")] +async fn user_serves_the_same_model_again(world: &mut E2eWorld) { + // Re-arm the scripted death: the runner consumes the scenario env on each + // invocation, and the retry has to die the same way to be recognisable. + world.command_env.push(( + rocm_core::MANAGED_ENGINE_STARTUP_FAILURE_TEST_ENV, + "1".into(), + )); + user_serves_with_failing_lemonade_preparation(world).await; +} + +#[then("that serve reaches the engine launch again and fails the same way")] +async fn assert_retry_reaches_the_engine_launch(world: &mut E2eWorld) { + // The retry reaches the engine launch again and dies there the same way, + // rather than being answered by the first launch's record as an engine already + // starting. The `stopped` status asserted by the previous step is what frees + // the engine and model for it. + let output = serve_output(world); + assert_ne!( + world.cli_rc, + Some(0), + "the retry should fail the same way, not succeed:\n{output}" + ); + assert!( + output.contains("managed engine exited immediately"), + "the retry must reach the engine launch again rather than being turned \ + away by the previous failed launch:\n{output}" + ); + // The retry's own cleanup left its record the same way, beside the first. + assert_failed_launches_left_their_claims(&lemonade_service_records(world), 2); +} + /// Serve this host's small model on loopback with `--require-api-key`, so the /// CLI generates a key and stores it before the engine starts. The serve still /// waits a fixed 45 seconds for the engine (`READINESS_WAIT` in