diff --git a/apps/rocm/Cargo.toml b/apps/rocm/Cargo.toml index e48eb7cf8a..b55ce61fc4 100644 --- a/apps/rocm/Cargo.toml +++ b/apps/rocm/Cargo.toml @@ -22,7 +22,7 @@ e2e-test-hooks = [ # hook that simulates a managed vLLM launch which OOM'd and never became ready, # so the positive OOM-guidance summary can be verified on a GPU-less CI host. # See `e2e_simulate_oom_launch` / `simulate_oom_managed_launch` in -# `service_launch.rs`. +# `service_launch/readiness.rs`. e2e-oom-fault-injection = [] [dependencies] diff --git a/apps/rocm/src/endpoint_keys.rs b/apps/rocm/src/endpoint_keys.rs index a40ee68ab6..e85fc3d216 100644 --- a/apps/rocm/src/endpoint_keys.rs +++ b/apps/rocm/src/endpoint_keys.rs @@ -5,7 +5,7 @@ //! Secure storage for per-service inference-endpoint API keys. //! //! When `rocm serve` binds a public (non-loopback) interface it protects the -//! endpoint with an API key (see `resolve_endpoint_auth` in `service_launch.rs`). +//! endpoint with an API key (see `resolve_endpoint_auth` in `service_launch/endpoint_auth.rs`). //! The key is stored in a 0600 file under the services directory, keyed by //! service id. //! diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 7147245f2c..abe85f2b75 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -1290,7 +1290,8 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { notes, }; // Same intentional one-time key display as `print_managed_launch_plain` - // in service_launch.rs; see its rationale for why the tag below is currently inert. + // in service_launch/start.rs; see its rationale for why the tag below is + // currently inert. // codeql[rust/cleartext-logging] print!("{}", serve_summary::render_summary(&summary)); } else { diff --git a/apps/rocm/src/serve_port.rs b/apps/rocm/src/serve_port.rs index 2f501506fc..0b4a509d70 100644 --- a/apps/rocm/src/serve_port.rs +++ b/apps/rocm/src/serve_port.rs @@ -233,9 +233,9 @@ pub(crate) fn resolve_serve_port( if let Some(owner) = reserved.iter().find(|entry| entry.port == port) { // The "stop it" advice below is proven, not just worded: see // `stopping_the_reserving_record_frees_its_explicit_port_for_reuse` - // in `service_launch.rs`, which recreates the exact on-disk status - // transition `rocm services stop` performs and confirms the same - // explicit port then resolves. A full CLI-level E2E for this needs + // in `service_launch/launch_transaction.rs`, which recreates the exact + // on-disk status transition `rocm services stop` performs and confirms + // the same explicit port then resolves. A full CLI-level E2E for this needs // a real GPU, since reaching this branch requires a completed // model/engine resolution first. bail!("{}", reserved_message(DEFAULT_LOCAL_HOST, port, owner)); diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 53f9b5c41d..c9b553b531 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -10,7 +10,7 @@ //! is the managed-service-spawning tail that was deliberately deferred out of //! the Phase 6a (ROCMAI-82) split: `start_managed_service`, //! `run_attached_service`, `spawn_managed_engine_child`, `stream_attached_logs`, -//! and the rest of this file. It was entangled with crate-root launch helpers +//! and the rest of what was then a single file. It was entangled with crate-root launch helpers //! — `record_cli_audit_event`, `existing_live_managed_service_in`, and the stop //! action of the service-record-management cluster (this module reaches only //! stop, via `AttachOutcome::Stop`; restart/remove/prune are not called here) @@ -28,4907 +28,52 @@ //! here uses) to `managed_service_status.rs`. Several functions here are `pub(crate)` //! because other modules call into this cluster across the module boundary; //! `main.rs` only declares the module. +//! +//! # Layout +//! +//! This file only declares the submodules and re-exports the items that other +//! modules call, so callers keep reaching them as `crate::service_launch::name`. +//! Items a submodule shares with its siblings are `pub(super)` and reached through +//! `super::`; nothing is wider than that unless a module outside this +//! one calls it. +//! +//! - `endpoint_auth`: bind-host validation and the endpoint API key checks. +//! - `launch_transaction`: the locked port allocation decision, the cleanup of a +//! failed launch, and the reuse refusals. +//! - `spawn`: the detached engine child, its process and stdio plumbing, and +//! `ManagedLaunchReport`. +//! - `readiness`: waiting on a service, its liveness, and the OOM fault-injection +//! seam. +//! - `start`: the background launch path and its plain output. +//! - `helper_autostart`: starting the background automation helper. +//! - `attach`: the attached run and its log streaming. + +mod attach; +mod endpoint_auth; +mod helper_autostart; +mod launch_transaction; +mod readiness; +mod spawn; +mod start; +#[cfg(test)] +mod test_support; -use std::fmt::Write as _; -use std::fs; -use std::io::{self, BufRead, Write}; -use std::path::{Path, PathBuf}; -#[cfg(not(windows))] -use std::process::{Command as ProcessCommand, Stdio}; -use std::thread; -use std::time::Duration; - -use anyhow::{Context, Result, anyhow, bail}; -use rocm_core::{AppPaths, EndpointReadiness, ManagedServiceRecord}; -use rocm_core::{format_http_base_url, process_is_running}; -use rocm_engine_protocol::{DevicePolicy, EngineRecipeHint, ResolveModelResponse}; - -use crate::background_helper::background_helper_already_running; -use crate::endpoint_keys; -use crate::engine_ipc::builtin_engine_serve_http_args; -#[cfg(not(windows))] -use crate::engine_runtime_select::apply_app_path_env; -#[cfg(windows)] -use crate::engine_runtime_select::{app_path_env_var_refs, app_path_env_var_values}; -use crate::engine_runtime_select::{ - managed_service_launcher_path, wait_for_service_http_ready_with_progress, +pub(crate) use attach::run_attached_service; +pub(crate) use endpoint_auth::{ + drop_orphaned_endpoint_key_on_already_running, ensure_public_bind_engine_supported, + ensure_public_service_has_endpoint_key, is_loopback_host, resolve_endpoint_auth, + validate_bind_host, }; -use crate::engines_cmd::env_root_for_service; -use crate::managed_service_status::{ - existing_live_managed_service_in, load_managed_services, managed_service_is_live, +pub(crate) use helper_autostart::{ + ensure_background_helper_running, ensure_background_helper_running_quiet, }; -use crate::serve_port; -use crate::service_hints; -use crate::services_text::read_optional_tail_lines; -use crate::services_text::{ - render_service_action_result, render_service_action_result_with_helper_note, - service_stop_unconfirmed, -}; -use crate::{device_policy_name, record_cli_audit_event, status_for_readiness}; - -pub(crate) fn validate_bind_host(host: &str, allow_public_bind: bool) -> Result<()> { - if !is_loopback_host(host) && !allow_public_bind { - bail!( - "`rocm serve --host {host}` is not loopback; pass `--allow-public-bind` before binding a non-local interface" - ); - } - Ok(()) -} - -/// Inverse of [`rocm_engine_protocol::is_public_bind_host`], which owns the -/// policy so `rocmd` classifies a recorded `host` identically when it respawns -/// the service. -pub(crate) fn is_loopback_host(host: &str) -> bool { - !rocm_engine_protocol::is_public_bind_host(host) -} - -/// Resolve the API key that will guard this endpoint, applying the -/// loopback-vs-public policy for `rocm serve`. -/// -/// - **Loopback host** → `None`: local serving stays credential-free (the -/// unchanged default). Any key supplied for a loopback bind is ignored and -/// nothing is persisted — loopback needs no auth. -/// - **Public host** → `Some(key)`: use the user-supplied key when present, -/// otherwise generate a strong random one so a public endpoint can never come -/// up anonymous. An empty/whitespace supplied key is rejected rather than -/// silently treated as "no auth". -/// - **`required`** → treat a loopback bind as public for this purpose. -/// -/// That last case exists because "loopback" is a statement about the bind -/// address, not about who can reach the port. Publishing the port onto a -/// tailnet, proxying it, or mapping it out of a container all leave the bind -/// loopback while widening the audience — and the policy above would then hand -/// out an unauthenticated endpoint. Whoever widens the reach is responsible for -/// asking for the credential, so this is an explicit flag rather than a guess. -pub(crate) fn resolve_endpoint_auth( - host: &str, - supplied: Option<&str>, - required: bool, -) -> Result> { - if is_loopback_host(host) && !required { - return Ok(None); - } - match supplied { - Some(raw) => { - let trimmed = raw.trim(); - if trimmed.is_empty() { - bail!( - "`rocm serve --api-key` (or ROCM_SERVE_API_KEY) was empty; a public \ - endpoint must be protected by a non-empty API key" - ); - } - // The key is later interpolated verbatim into raw `Authorization: - // Bearer {key}\r\n` header lines; reject a control character (e.g. an - // embedded CR/LF) here so a crafted key cannot inject extra headers. - if rocm_core::endpoint_api_key_has_forbidden_chars(trimmed) { - bail!( - "`rocm serve --api-key` (or ROCM_SERVE_API_KEY) contained a control \ - character such as a carriage return or newline; an endpoint API key \ - must be a single line of printable characters" - ); - } - Ok(Some(trimmed.to_owned())) - } - None => Ok(Some(rocm_core::generate_endpoint_api_key())), - } -} - -/// When an equivalent managed service is already running, or the launch fails -/// before any record exists, the endpoint key serve() freshly stored for this -/// attempt is unused — drop it so it is not orphaned in storage. The already-running service keeps its own key. -/// Best-effort and idempotent; a loopback attempt (`freshly_stored == None`) -/// is a no-op. -pub(crate) fn drop_orphaned_endpoint_key_on_already_running( - paths: &AppPaths, - service_id: &str, - freshly_stored: Option<&str>, -) { - if freshly_stored.is_some() { - endpoint_keys::clear_endpoint_api_key(paths, service_id); - } -} - -/// Reject engine/platform combinations that cannot enforce a public endpoint's -/// API key, so a public bind fails closed instead of coming up unauthenticated. -/// -/// The one such case today: Windows managed Lemonade. Its server reads the -/// value-typed `LEMONADE_API_KEY` env var, but the Windows detached-spawn -/// primitive only carries path-valued env overrides, so the key never reaches it. -/// vLLM enforces auth on every platform (`VLLM_API_KEY`), and loopback binds -/// (`public_bind == false`) need no key — both pass. `is_windows` is a parameter -/// so both branches are unit-testable off-Windows. -pub(crate) fn ensure_public_bind_engine_supported( - engine: &str, - public_bind: bool, - is_windows: bool, -) -> Result<()> { - if public_bind && is_windows && engine == "lemonade" { - bail!( - "public binding with the lemonade engine is not supported on Windows: the endpoint \ - API key cannot be enforced there. Use `--engine vllm`, or bind a loopback host \ - (the default 127.0.0.1)." - ); - } - Ok(()) -} - -/// Refuse to (re)spawn a managed service recorded on a public host when its -/// endpoint API key is gone, so a respawn cannot reopen the endpoint anonymously. -/// -/// `serve()` applies the loopback-vs-public policy once, via -/// [`resolve_endpoint_auth`], and persists the resulting key. Every later spawn -/// — `rocm services restart`, and `rocmd`'s recovery supervisor — reads that key -/// file back and would otherwise treat "no key file" as "no auth wanted", -/// silently downgrading a protected public endpoint to an open one. The real -/// invariant is a property of the *host*, not of the file: a non-loopback bind -/// must always be authenticated. -/// -/// The gap is reachable through ordinary commands, because a stop deletes the -/// key file and `rocm services restart` accepts a stopped service id (its help -/// points at `rocm services list --all`). -/// -/// `key_present` is a plain `bool` rather than a path so both branches are -/// unit-testable without touching the filesystem, mirroring `is_windows` in -/// [`ensure_public_bind_engine_supported`]. -pub(crate) fn ensure_public_service_has_endpoint_key( - host: &str, - key_present: bool, - requires_api_key: bool, -) -> Result<()> { - // Two ways a service can need a key. A public bind is the obvious one. The - // other is a service that asked for auth on a loopback bind, because - // something outside this process republishes the port — a tailnet publish - // survives a reboot, let alone a restart, so "loopback" stops meaning - // "only this machine" and the bind address can no longer be trusted to - // answer the question on its own. - if requires_api_key && !key_present { - bail!( - "managed service was launched with `--require-api-key` but has no endpoint API key, \ - so restarting it would reopen it without authentication. Something outside this \ - machine may still be publishing its port. The key is dropped when a service stops \ - and cannot be recovered. Launch it again with \ - `rocm serve --require-api-key` (add `--api-key `, or set ROCM_SERVE_API_KEY, \ - to choose the key instead of generating one)." - ); - } - if rocm_engine_protocol::is_public_bind_host(host) && !key_present { - bail!( - "managed service is bound to the public host `{host}` but has no endpoint API key, \ - so restarting it would reopen it without authentication. The key is dropped when a \ - service stops and cannot be recovered. Launch it again with \ - `rocm serve --host {host} --port --allow-public-bind` (add `--api-key `, or set \ - ROCM_SERVE_API_KEY, to choose the key instead of generating one)." - ); - } - Ok(()) -} - -#[cfg(not(windows))] -pub(crate) fn detach_background_command(command: &mut ProcessCommand) { - rocm_core::detach_command_session(command); -} - -#[cfg(not(windows))] -pub(crate) fn attach_background_stdio( - command: &mut ProcessCommand, - log_path: Option<&Path>, -) -> Result<()> { - if let Some(log_path) = log_path { - let log = fs::OpenOptions::new() - .create(true) - .append(true) - .open(log_path) - .with_context(|| format!("failed to open {}", log_path.display()))?; - command - .stdout(Stdio::from(log.try_clone()?)) - .stderr(Stdio::from(log)); - } else { - command.stdout(Stdio::null()).stderr(Stdio::null()); - } - Ok(()) -} - -/// Windows spawn-liveness check: wait out the startup window, then fail with the -/// startup log when the child is already gone. The wait and the liveness probe -/// are injected so a unit test runs the exit-detected path on every platform, -/// including the Windows lane that actually calls it. -/// -/// # Errors -/// Fails with the startup-failure detail when `is_running` reports the child dead. -#[cfg_attr(not(windows), allow(dead_code))] -fn ensure_spawned_child_survives_startup( - child_pid: u32, - log_path: &Path, - settle: impl FnOnce(), - is_running: impl FnOnce(u32) -> bool, -) -> Result<()> { - settle(); - if is_running(child_pid) { - Ok(()) - } else { - bail!("{}", managed_engine_startup_failure_detail("", log_path)) - } -} - -/// `status_clause` is a pre-formatted fragment inserted right after "exited -/// immediately" (e.g. `" with status exit status: 1"` on Unix, where a real -/// `ExitStatus` is available; empty on Windows, where the liveness check -/// below only has a yes/no answer, not an exit code). -pub(crate) fn managed_engine_startup_failure_detail( - status_clause: &str, - log_path: &Path, -) -> String { - let mut recent_lines = read_optional_tail_lines(log_path, 80, "service log"); - if recent_lines.is_empty() { - for _ in 0..5 { - thread::sleep(Duration::from_millis(120)); - recent_lines = read_optional_tail_lines(log_path, 80, "service log"); - if !recent_lines.is_empty() { - break; - } - } - } - if recent_lines.is_empty() { - return format!( - "managed engine exited immediately{status_clause}; inspect {}", - log_path.display() - ); - } - format!( - "managed engine exited immediately{status_clause}; inspect {}\n\nrecent startup log output:\n{}", - log_path.display(), - recent_lines.join("\n") - ) -} -#[cfg(not(windows))] -pub(crate) fn managed_service_process_command(program: &Path, args: &[String]) -> ProcessCommand { - let mut command = ProcessCommand::new(program); - command.args(args); - command -} - -/// Engine-neutral result of a managed launch. Returned rather than printed so the -/// caller can render it either as the rich deployment summary (interactive TTY) or -/// as the plain line-by-line form (piped output, chat assistant), from one code path. -pub(crate) struct ManagedLaunchReport { - pub(crate) service_id: String, - /// `http://host:port/v1`. - pub(crate) endpoint_url: String, - /// `"ready"` (inference confirmed), `"running"` (model listed but not serving - /// yet), `"starting"` (endpoint not answering), or the existing service's - /// status when nothing was spawned. - pub(crate) status: String, - /// True when an equivalent service was already live and nothing was spawned. - pub(crate) already_running: bool, - pub(crate) child_pid: Option, - pub(crate) log_path: Option, - pub(crate) manifest_path: Option, -} - -/// Either an already-live service (nothing spawned) or a freshly spawned engine -/// child that is `running` but not yet HTTP-ready. -/// -/// Split out of [`start_managed_service`] so the attached (`--verbose` / -/// `--foreground`) serve path can spawn the very same detached child and stream -/// its log live from the first line — including startup — instead of blocking on -/// the readiness wait before any output appears. -enum ManagedSpawn { - AlreadyRunning(ManagedLaunchReport), - // `ManagedServiceRecord` is large; box it so the two variants stay a similar - // size (clippy::large_enum_variant). - Spawned { - record: Box, - child_pid: u32, - }, -} - -/// Outcome of the locked allocation step. -/// -/// `Debug` so tests can assert on `Result::expect_err`. -#[derive(Debug)] -enum ManagedPortDecision { - /// An equivalent managed service is already live; nothing may be spawned. - /// Boxed to keep the variants a similar size (clippy::large_enum_variant). - AlreadyLive(Box), - /// The concrete port this launch owns until it hands it to the child. - Lease(rocm_core::LoopbackPortLease), -} - -/// Live managed services, with liveness refreshed by [`load_managed_services`] -/// (dead PIDs demote to "stopped" and therefore neither satisfy the duplicate -/// check nor reserve a port). -/// -/// A read failure degrades to "nothing is live": for an already-`ready` -/// service the OS preflight still guards its port (the engine holds it), and -/// refusing to serve because the services directory could not be listed would -/// be a worse failure than launching. That guard does not cover a `starting` -/// record: its port is released back to the OS in -/// `launch_managed_engine_child_locked` (see `lease.release()`), right before the -/// spawn, and stays unbound until the new engine child has actually bound it, so a concurrent launch that hits this -/// read failure in that narrow window can probe the same port successfully. -/// Accepted as a known gap — narrower than the general services-dir read -/// failure case, and only reachable by a second, near-simultaneous I/O -/// failure on top of one already in flight. -fn live_managed_services(paths: &AppPaths) -> Vec { - load_managed_services(paths) - .unwrap_or_default() - .into_iter() - .filter(managed_service_is_live) - .collect() -} - -/// Ports that live managed services still own. -/// -/// Conservative by design: on the default loopback host a record reserves its -/// port regardless of the host it recorded, because a `0.0.0.0` (or `localhost`) -/// listener also blocks the loopback address a new local server would bind. -/// Over-reserving costs one candidate in a 101-port scan; under-reserving costs -/// a collision. On a custom host the recorded host is compared (see -/// `serve_port::hosts_overlap`). -fn reserved_service_ports(live: &[ManagedServiceRecord]) -> Vec { - live.iter() - .map(|record| serve_port::ReservedPort { - port: record.port, - host: record.host.clone(), - service_id: record.service_id.clone(), - status: record.status.clone(), - }) - .collect() -} - -/// The decision half of the launch transaction: refresh liveness, apply the -/// `(engine, canonical_model_id)` idempotency guard, then lease a concrete port -/// against both live records and the real OS. -/// -/// Callers MUST hold [`rocm_core::AppPaths::managed_launch_lock_path`] across -/// this call *and* the record publication that follows, or two launches can -/// pick the same candidate. -/// `probe` is injected so tests can drive deterministic bind outcomes without a -/// GPU or a spawn. -fn resolve_managed_port_in_transaction( - paths: &AppPaths, - engine: &str, - canonical_model_id: &str, - host: &str, - request: serve_port::PortRequest, - probe: &mut dyn FnMut(u16) -> std::io::Result, -) -> Result { - let live = live_managed_services(paths); - // Idempotency first: an already-live equivalent service means no port is - // needed at all, so it must be answered before any candidate is leased. - if let Some(existing) = existing_live_managed_service_in(&live, engine, canonical_model_id) { - return Ok(ManagedPortDecision::AlreadyLive(Box::new(existing.clone()))); - } - let reserved = reserved_service_ports(&live); - serve_port::resolve_serve_port(host, request, &reserved, probe).map(ManagedPortDecision::Lease) -} - -/// Grace given to a child that must be terminated because its record could not -/// be published. Short: the child is milliseconds old and has nothing to flush. -const MANAGED_LAUNCH_CLEANUP_GRACE: Duration = Duration::from_secs(2); - -/// Undo a failed managed launch attempt before the launch lock is released, -/// so no failed transaction leaves a live reservation or a stray endpoint key. -/// -/// Order matters: a spawned child is terminated *first*, so it is not still -/// holding the port or writing to the log by the time cleanup runs. The -/// termination outcome is audited but not acted on: the record is marked -/// abandoned (`stopped`, releasing the port) even when death was not confirmed. If no child was ever spawned (`record.engine_pid` is still `None`), -/// every artifact this attempt created is removed: there is nothing for -/// `rocm services`/`rocm logs` to show. If a child did run, its record and log -/// are kept instead (marked `status = "stopped"`, see -/// [`ManagedServiceRecord::mark_launch_abandoned`]), since the error that led here -/// already told the user to go inspect that log; only the endpoint key, which a -/// failed attempt has no use for, is cleared. -/// Every step is best-effort and idempotent; cleanup must never mask the -/// original failure with a second one. -fn abandon_failed_managed_launch(paths: &AppPaths, record: &ManagedServiceRecord) { - if let Some(pid) = record.engine_pid.filter(|pid| *pid != 0) { - // The token captured at spawn, not a fresh `capture`: when the child - // already exited, a fresh one would describe a recycled pid's new owner - // and the forced tree kill would hit it. Where start tokens exist - // (Linux), a missing one means the child was gone before it could be - // captured, so the pid is unverifiable and nothing is signalled - // (`terminate_verified` alone would "best-effort proceed"). Windows - // records no token at all, so there the kill proceeds as before. - let outcome = if record.engine_start_ticks.is_none() && cfg!(target_os = "linux") { - rocm_core::TerminationOutcome::Unverified - } else { - rocm_core::terminate_verified( - &rocm_core::ProcessIdentity::new(pid, record.engine_start_ticks), - rocm_core::KillScope::Tree, - MANAGED_LAUNCH_CLEANUP_GRACE, - true, - ) - }; - record_cli_audit_event( - paths, - "service", - "managed_service_launch_cleanup", - "warn", - format!( - "terminated engine child pid={pid} after a failed launch publication outcome={}", - outcome.as_str() - ), - Some(&record.service_id), - ); - let mut abandoned = record.clone(); - abandoned.mark_launch_abandoned(); - let _ = abandoned.write(); - endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); - return; - } - for path in [ - record.manifest_path.as_path(), - record.log_path.as_path(), - record.engine_state_path.as_path(), - ] { - let _ = fs::remove_file(path); - } - endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); -} - -/// The ways a live managed service that already matches on liveness, engine and -/// canonical model id can still be unusable by this invocation. -/// -/// Those three terms are necessary but not sufficient: three further terms can -/// still refuse, one variant each. [`managed_service_reuse_refusal`] is the -/// single decision procedure both callers ask, so the two cannot answer -/// differently — which they previously did; see that function for what the -/// disagreement cost. -pub(crate) enum ManagedReuseRefusal { - /// The live service was started with different serve options. - RecipeMismatch, - /// `--require-api-key` was asked for and the live service has no key. - /// - /// The engine reads its key once, at launch, from the environment the spawn - /// builds — so a server started without one keeps serving anonymously no - /// matter what is written afterwards. Upgrading the record would be worse - /// than doing nothing: it would claim auth the live process does not - /// enforce, and `ensure_public_service_has_endpoint_key` would pass on the - /// strength of a key file nothing reads. `rocm remote serve` relies on this: - /// it publishes a loopback port onto the tailnet and prints "the API key - /// above is what stops anyone else calling it", which reusing an - /// unauthenticated service silently would make false about an endpoint the - /// whole tailnet can reach. Refusing is the only answer that fails closed. - MissingApiKey, - /// The live service runs under a device policy that does not satisfy this - /// invocation's. Reusing a `cpu_only` service to answer a `gpu_required` - /// serve is the silent CPU fallback AGENTS.md section 6 forbids. - DevicePolicyTooWeak, -} - -/// Why this invocation cannot reuse `existing` — or `None` when it genuinely -/// can. See [`ManagedReuseRefusal`] for the terms this answers on. -/// -/// This exists because `serve()`'s reuse pre-gate lets a matching live service -/// skip the no-usable-AMD-GPU refusal. Deciding that bypass on a *weaker* test -/// than the spawn then uses to accept reuse produced a dead end: on a GPU-less -/// host (`HIP_VISIBLE_DEVICES=` with a server live from another shell) a serve -/// with different options bypassed the refusal, reached the spawn, and was told -/// to stop the server and run the command again — and the rerun, with nothing -/// left to reuse, hit the no-GPU refusal. The advice named a command that could -/// not clear the condition, which AGENTS.md section 3 forbids. Sharing the -/// predicate means a service that cannot actually be reused never buys the -/// bypass, so the user gets the accurate refusal first and keeps their server. -/// -/// `requested_device_policy` is taken by value, not as an `Option`, on purpose. -/// It was optional so the spawn could opt out of the device-policy term, and -/// that left two holes at once: the spawn's `DevicePolicyTooWeak` arm was -/// unreachable (an untestable bail), and `serve()` — the caller that *does* -/// gate the GPU refusal on this answer — could silently drop the term by -/// passing `None`, with the whole suite still green. Both callers have a -/// `DevicePolicy` in hand, so requiring it costs nothing and makes that -/// mutation a compile error. It widens nothing observable today either: the -/// only policy a record can carry is the one `parse_device_policy` admits -/// (see below), so `existing.device_policy == Some("cpu_only")` is not -/// reachable from any `rocm serve` this CLI can run. -/// The `rocm serve` no-usable-GPU refusal, worded once. -/// -/// Two places raise it and they must say the same thing: `serve()`'s fail-fast -/// pre-flight, and the spawn below when the reuse that waived that pre-flight -/// turns out not to be available after all. -pub(crate) fn no_usable_gpu_refusal(device_policy: &DevicePolicy) -> String { - // Keyed on the same `is_wsl_host()` the usable-GPU probe branches on, so the - // advice follows the platform whose answer produced the refusal. - no_usable_gpu_refusal_text(device_policy_name(device_policy), rocm_core::is_wsl_host()) -} - -/// The refusal text for `policy`, split on whether the probe that refused was -/// the WSL one. -/// -/// WSL reaches the GPU through `/dev/dxg` and ROCDXG, and has no `/dev/kfd`, so -/// telling a WSL user to confirm that device is present names something that can -/// never clear the condition. There the advice points at the status `rocm -/// examine` reports for the path the probe actually asked about -/// ([`rocm_core::WSL_ROCDXG_READY`]), and at `rocm diagnose`, which names what is -/// missing. -fn no_usable_gpu_refusal_text(policy: &str, on_wsl: bool) -> String { - let remedy = if on_wsl { - format!( - "WSL reaches the GPU through /dev/dxg and ROCDXG rather than /dev/kfd: run \ - `rocm examine` and check that `driver_status` is `{ready}` (`rocm diagnose` \ - names what is missing; a ROCDXG library the loader reaches only through \ - LD_LIBRARY_PATH needs that variable exported in this shell, see docs/wsl.md), and ensure HIP_VISIBLE_DEVICES / ROCR_VISIBLE_DEVICES \ - are not masking every device.", - ready = rocm_core::WSL_ROCDXG_READY - ) - } else { - "Check the driver with `rocm examine`, confirm /dev/kfd is present, and ensure \ - HIP_VISIBLE_DEVICES / ROCR_VISIBLE_DEVICES are not masking every device." - .to_owned() - }; - format!( - "no usable AMD GPU detected; `rocm serve` requires a GPU under the {policy} policy \ - and does not fall back to CPU. {remedy}" - ) -} - -pub(crate) fn managed_service_reuse_refusal( - existing: &ManagedServiceRecord, - requested_recipe_json: Option<&str>, - require_api_key: bool, - requested_device_policy: &DevicePolicy, -) -> Option { - if existing.engine_recipe_json.as_deref() != requested_recipe_json { - return Some(ManagedReuseRefusal::RecipeMismatch); - } - if require_api_key && !existing.requires_api_key { - return Some(ManagedReuseRefusal::MissingApiKey); - } - // Only `gpu_required` can be let down by the record's policy, and the - // enumeration is exhaustive rather than narrow: `parse_device_policy` - // (`main.rs`) maps `auto`, `gpu`, `gpu_required` *and* `gpu_preferred` onto - // `GpuRequired`, and rejects `cpu_only` with an error, so `GpuRequired` is - // the only policy `serve()` can hand this predicate — pinned by - // `parse_device_policy_defaults_to_gpu_required_without_cpu_fallback`. There - // is deliberately no wider match on "anything but `CpuOnly`": it would be - // identical at runtime and would read as a guard against a `gpu_preferred` - // bypass that no CLI invocation can reach. A `cpu_only` invocation would be - // satisfied by anything anyway and never reaches the bail. README.md's - // "that is the only policy pair refused" is the contract, and this matches - // it exactly. - if matches!(requested_device_policy, DevicePolicy::GpuRequired) - && existing.device_policy.as_deref() == Some("cpu_only") - { - return Some(ManagedReuseRefusal::DevicePolicyTooWeak); - } - None -} - -/// Spawn the detached engine child shared by the managed (background) and -/// attached (`--verbose`/`--foreground`) serve paths. Returns before the HTTP -/// readiness wait; callers decide whether to block on readiness -/// ([`start_managed_service`]) or start tailing the log immediately -/// ([`run_attached_service`]). -/// -/// The whole body is one serialized transaction under the shared allocation -/// lock: duplicate detection, live-reservation refresh, candidate leasing, -/// manifest/log creation, argv preparation, the bounded immediate-exit check, -/// and record publication. Readiness deliberately runs after the lock is -/// released. -#[allow(clippy::too_many_arguments)] -fn spawn_managed_engine_child( - paths: &AppPaths, - engine: &str, - service_id: &str, - requested_model: &str, - resolve: &ResolveModelResponse, - host: &str, - port_request: serve_port::PortRequest, - device_policy: &DevicePolicy, - gpu_indices: &[u32], - runtime_id: Option<&str>, - env_id: Option<&str>, - engine_recipe: Option<&EngineRecipeHint>, - require_api_key: bool, - gpu_refusal_waived_for_reuse: bool, -) -> Result { - paths.ensure()?; - fs::create_dir_all(paths.services_dir())?; - - // No lock is acquired here: every production caller - // ([`start_managed_service`], [`run_attached_service`]) already holds - // [`rocm_core::AppPaths::managed_launch_lock_path`] for the whole of this - // call, so no two transactions can choose the same port or publish - // competing records for one engine+model. A second acquisition in this - // frame would self-deadlock, since the lock is exclusive per open file - // description, not per process. - // - // Idempotency guard: if a managed service for this engine+model is already - // alive, surface it and spawn nothing. A second `serve --managed` (e.g. the - // chat assistant re-issuing the same request) is treated as satisfied, not - // an error. Keyed on engine+canonical model — the freshly generated - // `service_id` is timestamp-unique and would never match an existing one. - // Stale/dead services fall through and relaunch normally. - let requested_recipe_json = engine_recipe - .map(serde_json::to_string) - .transpose() - .context("failed to encode engine recipe hint")?; - let lease = match resolve_managed_port_in_transaction( - paths, - engine, - &resolve.canonical_model_id, - host, - port_request, - &mut serve_port::loopback_bind_probe, - )? { - ManagedPortDecision::AlreadyLive(existing) => { - let existing = *existing; - // Asked through the shared predicate rather than re-tested here, so this - // refusal and `serve()`'s reuse pre-gate cannot disagree about whether a - // service is reusable. When they disagreed, the pre-gate let a - // non-reusable service bypass the no-GPU refusal and the user was sent - // here to be told to stop a working server for nothing. - // - // The same device policy the serve path passes. This side used to pass - // `None` to keep the spawn's acceptance untouched, but that opted the - // spawn out of a term it has the input for, left its own - // `DevicePolicyTooWeak` bail unreachable, and made the serve path's - // argument droppable. Nothing observable widens: `parse_device_policy` - // is the only producer of a `DevicePolicy` on every path that reaches - // here and it rejects `cpu_only`, so no record this CLI writes can - // carry the policy this term refuses. - match managed_service_reuse_refusal( - &existing, - requested_recipe_json.as_deref(), - require_api_key, - device_policy, - ) { - Some(ManagedReuseRefusal::RecipeMismatch) => bail!( - "managed service `{}` is already running for engine `{engine}` and model `{}` with different serve options (recipe hint, tool-call parser, or generation defaults); stop it and run `rocm serve` again to apply the requested options", - existing.service_id, - resolve.canonical_model_id - ), - Some(ManagedReuseRefusal::MissingApiKey) => bail!( - "managed service `{}` is already running for engine `{engine}` and model `{}` \ - without authentication, and a running server cannot be given a key it did not \ - start with; stop it with `{}` and run the command again to \ - serve it with `--require-api-key`", - existing.service_id, - resolve.canonical_model_id, - service_hints::service_stop_hint(&existing.service_id) - ), - // Reachable only from a record whose `device_policy` says - // `cpu_only`, which no `rocm serve` can write today - // (`parse_device_policy` refuses that policy outright) — so this is - // a guard against a record written by some future path, not a - // refusal users see now. The advice still names a command that - // clears the condition: stopping the service removes the only thing - // this term can object to, and the rerun then launches normally - // (AGENTS.md section 3). - Some(ManagedReuseRefusal::DevicePolicyTooWeak) => bail!( - "managed service `{}` is already running under a device policy that cannot \ - satisfy this request; stop it with `{}` and run the command again", - existing.service_id, - service_hints::service_stop_hint(&existing.service_id) - ), - None => {} - } - // An explicit port is a demand, not a hint: reporting success on a - // different port than the one asked for would silently drop it. - if let Some(port) = port_request.explicit() - && port != existing.port - { - bail!( - "managed service `{}` is already serving engine `{engine}` and model `{}` on port {}, so port {port} would not be used; stop it with `{}` and run the command again to serve on that port", - existing.service_id, - resolve.canonical_model_id, - existing.port, - service_hints::service_stop_hint(&existing.service_id) - ); - } - record_cli_audit_event( - paths, - "service", - "managed_service_launch_skipped", - "info", - format!( - "skipped duplicate managed launch engine={engine} model={} existing_service_id={} status={}", - resolve.canonical_model_id, existing.service_id, existing.status - ), - Some(&existing.service_id), - ); - return Ok(ManagedSpawn::AlreadyRunning(ManagedLaunchReport { - service_id: existing.service_id, - endpoint_url: existing.endpoint_url, - status: existing.status, - already_running: true, - child_pid: None, - log_path: None, - manifest_path: None, - })); - } - ManagedPortDecision::Lease(lease) => lease, - }; - - // Nothing live to reuse — and if the only reason this invocation got past - // `serve()`'s no-usable-GPU pre-flight was a reusable service, that premise - // has just been falsified. `serve()` decided it from a snapshot taken before - // GPU selection and the launch lock; the check above is the authoritative - // one, so a service that died in between (an OOM, say) would otherwise fall - // through to a real spawn on a host with no usable GPU, and the user would - // get whatever the engine says about having no device instead of the - // actionable refusal the pre-flight exists to give. Raise it here, where the - // premise is known to be false. - if gpu_refusal_waived_for_reuse { - bail!("{}", no_usable_gpu_refusal(device_policy)); - } - - let mut record = ManagedServiceRecord::new( - paths, - service_id, - engine, - requested_model, - resolve.canonical_model_id.clone(), - host, - lease.port(), - "managed", - 0, - runtime_id.map(str::to_owned), - env_id.map(str::to_owned), - Some(device_policy_name(device_policy).to_owned()), - ); - record.gpu_indices = gpu_indices.to_vec(); - record.served_model_id = resolve.served_model_id.clone(); - record.engine_recipe_json = requested_recipe_json; - // The flag the user actually passed, carried through rather than re-derived. - // - // Deriving it from key-file presence looked equivalent and was not: - // `resolve_endpoint_auth` mints a key for *every* non-loopback bind whether or - // not auth was demanded, so a plain `--host 0.0.0.0 --allow-public-bind` - // recorded `true` here. The guard below tests this field before the bind - // address, so that service was then refused with a message naming a flag it - // never used and a relaunch command that drops `--allow-public-bind` — the - // public-bind branch, which carries the right command, became unreachable. - // - // This field means "the user demanded auth on a bind that would not otherwise - // require it". A public bind needs no such record; its address still says so. - record.requires_api_key = require_api_key; - // Publishing the record inside the lock is what reserves the leased port for - // every other transaction. From here on, any failure must remove it again. - record.write()?; - - match launch_managed_engine_child_locked( - paths, - &mut record, - lease, - engine, - service_id, - resolve, - host, - device_policy, - gpu_indices, - runtime_id, - env_id, - engine_recipe, - #[cfg(not(windows))] - None, - ) { - Ok(child_pid) => Ok(ManagedSpawn::Spawned { - record: Box::new(record), - child_pid, - }), - Err(error) => { - // The single cleanup choke point for every failure path below the - // record's first write: `record.engine_pid` tells it whether a child - // ever ran, so a pre-spawn failure deletes everything and a post-spawn - // one keeps the record and log (marked `status = "stopped"`) instead. - abandon_failed_managed_launch(paths, &record); - Err(error) - } - } -} - -// Test-only, thread-scoped spawn interceptor for callers that cannot take a -// `spawn_override` argument (`start_managed_service`, `run_attached_service`): -// lets a test observe state at the instant production would spawn the child. -#[cfg(all(test, not(windows)))] -thread_local! { - static TEST_SPAWN_HOOK: std::cell::RefCell>>> = - const { std::cell::RefCell::new(None) }; -} - -#[cfg(all(test, not(windows)))] -fn test_spawn_hook() -> Option>> { - TEST_SPAWN_HOOK.with(|hook| hook.borrow().clone()) -} - +pub(crate) use launch_transaction::{managed_service_reuse_refusal, no_usable_gpu_refusal}; +pub(crate) use readiness::e2e_simulate_oom_launch; #[cfg(not(windows))] -type SpawnOverride<'a> = dyn Fn(&mut ProcessCommand) -> io::Result + 'a; - -/// The side-effecting half of the launch transaction: create the log, prepare -/// concrete argv, hand the leased port to the child, spawn it, reject a child -/// that has already exited, and publish the running record. -/// -/// Split out of [`spawn_managed_engine_child`] so that *every* failure below the -/// record's first write is funnelled through one cleanup path before the -/// launch lock is released. Returns the child pid. -#[allow(clippy::too_many_arguments)] -fn launch_managed_engine_child_locked( - paths: &AppPaths, - record: &mut ManagedServiceRecord, - lease: rocm_core::LoopbackPortLease, - engine: &str, - service_id: &str, - resolve: &ResolveModelResponse, - host: &str, - device_policy: &DevicePolicy, - gpu_indices: &[u32], - runtime_id: Option<&str>, - env_id: Option<&str>, - engine_recipe: Option<&EngineRecipeHint>, - // Test-only seam: lets a test assert the leased port is actually bindable - // (i.e. released) at the exact point production would spawn the child, - // without needing to fake or intercept the real spawn mechanics. - #[cfg(not(windows))] spawn_override: Option<&SpawnOverride<'_>>, -) -> Result { - let port = record.port; - if let Some(parent) = record.engine_state_path.parent() { - fs::create_dir_all(parent) - .with_context(|| format!("failed to create {}", parent.display()))?; - } - fs::File::create(&record.log_path) - .with_context(|| format!("failed to create {}", record.log_path.display()))?; - let current_exe = managed_service_launcher_path() - .context("failed to resolve current rocm executable path")?; - let serve_args = builtin_engine_serve_http_args( - engine, - service_id, - &resolve.canonical_model_id, - host, - port, - device_policy, - gpu_indices, - runtime_id, - env_id, - engine_recipe, - &record.engine_state_path, - Some(&record.log_path), - )?; - let engine_envs_root = env_root_for_service(paths, engine, runtime_id, env_id)?; - // Hand the child the *path* to the endpoint key file (public bind only) via the - // environment. A path — not the secret value — is what the detached-spawn - // primitives accept as an env override, and it keeps the key off both the argv - // and the environment block. `serve()` wrote the file before spawning. - // Validity, not mere existence: the engine adapters resolve the key with - // `endpoint_api_key_from_file` and enforce nothing when it yields `None`, so - // an empty or malformed key file would otherwise satisfy the guard below and - // still produce an unauthenticated public listener. - let endpoint_key_file = endpoint_keys::endpoint_key_file_if_present(paths, service_id) - .filter(|path| rocm_engine_protocol::endpoint_api_key_from_file(path).is_some()); - // `serve()` already resolved and stored the key for a public bind, so the - // public-bind branch cannot fire on the fresh-launch path today. It is the - // shared choke point for managed spawns, so enforce the invariant here too - // rather than relying on every future caller having done so. - // - // `record.requires_api_key` is passed, not a literal, and the two arguments - // are deliberately different things: that field is the `--require-api-key` - // flag the caller passed, `endpoint_key_file` is whether a *usable* key is on - // disk. A present but empty or malformed key file is where they disagree, and - // is exactly what the `requires_api_key` branch exists to refuse. - // - // The field is threaded, never derived from the key file. Deriving it marked - // every public bind as having demanded auth — see the assignment above. - ensure_public_service_has_endpoint_key( - host, - endpoint_key_file.is_some(), - record.requires_api_key, - )?; - // Hand the port over: close the probe socket immediately before the spawn, - // still under the caller's launch lock, so no other ROCm transaction can - // take it in the meantime. On a custom host the lease never held a socket - // and this is a no-op — the engine owns that bind. - debug_assert_eq!(lease.port(), port); - lease.release(); - #[cfg(windows)] - let child_pid = { - let env_values = app_path_env_var_values(paths, engine_envs_root.as_deref()); - let mut env_refs = app_path_env_var_refs(&env_values); - if let Some(key_file) = endpoint_key_file.as_deref() { - env_refs.push((rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file)); - } - let child_pid = rocm_core::spawn_detached_no_inherit(¤t_exe, &serve_args, &env_refs) - .context("failed to launch managed engine process")?; - // Set before the immediate-exit check below, so a bail! here still leaves - // `record.engine_pid` telling cleanup that a child was spawned. The - // start-time token is captured here too, while the child is certainly - // ours: cleanup binds its kill to it rather than to whatever owns the - // pid by then. - record.record_engine_identity(child_pid); - ensure_spawned_child_survives_startup( - child_pid, - &record.log_path, - || thread::sleep(Duration::from_millis(200)), - process_is_running, - )?; - child_pid - }; - #[cfg(not(windows))] - let child_pid = { - let mut command = managed_service_process_command(¤t_exe, &serve_args); - command.stdin(Stdio::null()); - attach_background_stdio(&mut command, Some(&record.log_path))?; - detach_background_command(&mut command); - apply_app_path_env(&mut command, paths); - if let Some(engine_envs_root) = engine_envs_root.as_deref() { - command.env("ROCM_CLI_ENGINE_ENVS_ROOT", engine_envs_root); - } - if let Some(key_file) = endpoint_key_file.as_deref() { - command.env(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file); - } - #[cfg(all(test, not(windows)))] - let hook = test_spawn_hook(); - #[cfg(all(test, not(windows)))] - let spawn_override = spawn_override.or(hook.as_deref()); - let mut child = match spawn_override { - Some(spawn) => spawn(&mut command), - None => command.spawn(), - } - .context("failed to launch managed engine process")?; - let child_pid = child.id(); - // Set before the immediate-exit check below, so a bail! here still leaves - // `record.engine_pid` telling cleanup that a child was spawned. The - // start-time token is captured here too, while the child is certainly - // ours: cleanup binds its kill to it rather than to whatever owns the - // pid by then. - record.record_engine_identity(child_pid); - thread::sleep(Duration::from_millis(200)); - if let Some(status) = child - .try_wait() - .context("failed to check managed engine startup state")? - { - bail!( - "{}", - managed_engine_startup_failure_detail( - &format!(" with status {status}"), - &record.log_path - ) - ); - } - child_pid - }; - // The child is both roles until the engine state file names a separate - // server process. Each role gets its own token, captured while the child is - // alive: a stop pairs every recorded PID with its own token, and the two - // PIDs diverge as soon as `rocmd` restarts the service — at which point an - // engine PID recorded without one could only be signalled blind. - // The engine's token was already captured right after the spawn (so cleanup - // can bind to it); re-capturing here could replace it with a later owner's. - record.record_supervisor_identity(child_pid); - record.status = "running".to_owned(); - if let Err(error) = record.write() { - // Cleanup is the caller's job now: `record.engine_pid` (set above) is - // already enough for `abandon_failed_managed_launch` to terminate this - // child, so there is nothing left for this inner path to do beyond - // surfacing the error. - return Err(error.context("failed to publish the managed service record")); - } - - Ok(child_pid) -} - -/// How long one readiness wait polls before `rocm serve` reports `starting`. -const READINESS_WAIT: Duration = Duration::from_secs(45); - -/// `rocm serve --wait`: keep waiting past [`READINESS_WAIT`] (a first start -/// downloads the model, which can take far longer) until the server answers or -/// stops being live. A server that exited ends the wait with the last reading -/// instead of blocking forever. -fn wait_past_readiness_budget( - mut probe: impl FnMut() -> EndpointReadiness, - mut still_live: impl FnMut() -> bool, -) -> EndpointReadiness { - loop { - let readiness = probe(); - if readiness == EndpointReadiness::Serving || !still_live() { - return readiness; - } - } -} - -/// Whether `pid` is a running process that is not a zombie. The serve -/// supervisor is an unreaped child of this process, so after it dies -/// `process_is_running` alone keeps reporting it alive. -fn service_pid_is_alive(pid: u32) -> bool { - rocm_core::process_is_running(pid) && !process_is_zombie(pid) -} - -#[cfg(target_os = "linux")] -fn process_is_zombie(pid: u32) -> bool { - let Ok(stat) = fs::read_to_string(format!("/proc/{pid}/stat")) else { - return false; - }; - // `comm` may contain spaces and parentheses; the state follows the last `)`. - stat.rfind(')') - .and_then(|end| stat.get(end + 1..)) - .and_then(|rest| rest.split_whitespace().next()) - == Some("Z") -} - -#[cfg(not(target_os = "linux"))] -const fn process_is_zombie(_pid: u32) -> bool { - false -} - -/// Whether the record still describes a server that can finish starting: its -/// status is a live one *and* a tracked process is alive. The status alone is -/// not enough, since a killed engine never writes a terminal state. -fn record_still_live(record: &ManagedServiceRecord, pid_alive: impl Fn(u32) -> bool) -> bool { - if !crate::managed_service_status::managed_service_is_live(record) { - return false; - } - let pids = crate::managed_service_status::recorded_service_pids(record); - pids.is_empty() || pids.into_iter().any(pid_alive) -} - -/// [`record_still_live`] over the manifest on disk, so a `rocm services stop` -/// from another terminal (which writes only the manifest) ends the wait too. -fn service_still_live(paths: &AppPaths, service_id: &str, pid_alive: impl Fn(u32) -> bool) -> bool { - let Ok(mut latest) = ManagedServiceRecord::read_manifest(paths, service_id) else { - return false; - }; - let _ = latest.refresh_from_engine_state(); - record_still_live(&latest, pid_alive) -} - -/// Record a server that died mid-start as stopped, leaving a manifest that a -/// `services stop` already settled untouched. A `failed` state the engine -/// reported is persisted rather than overwritten, so the reason survives. -fn settle_exited_service(paths: &AppPaths, service_id: &str) { - let Ok(mut latest) = ManagedServiceRecord::read_manifest(paths, service_id) else { - return; - }; - let on_disk = latest.status.clone(); - let _ = latest.refresh_from_engine_state(); - if crate::managed_service_status::managed_service_is_live(&latest) { - latest.status = "stopped".to_owned(); - } else if latest.status == on_disk { - return; - } - let _ = latest.write(); -} - -/// The failure a `--wait` ends with when the server died before it became -/// ready, after recording the death in the manifest. `None` while the wait did -/// not apply, the endpoint is serving, or the server is still alive. -fn died_during_wait( - paths: &AppPaths, - service_id: &str, - wait_until_ready: bool, - readiness: EndpointReadiness, - pid_alive: impl Fn(u32) -> bool, -) -> Option { - if !wait_until_ready - || readiness == EndpointReadiness::Serving - || service_still_live(paths, service_id, pid_alive) - { - return None; - } - settle_exited_service(paths, service_id); - let mut message = format!( - "the server exited before it became ready (vLLM and AIM give up after 5 minutes by default; `ROCM_CLI_VLLM_READY_TIMEOUT_SECS` / `ROCM_CLI_AIM_READY_TIMEOUT_SECS` allow longer); see `rocm logs --service {service_id}` for its output" - ); - // A plain `serve` adds the memory guidance for a vLLM launch that ran out of - // VRAM; the wait's failure is the same launch, so it gets the same note. - // "starting" is the status the note keys on for "never became ready", which is - // what a `--wait` death is, whatever the settled record now says. - if let Ok(record) = ManagedServiceRecord::read_manifest(paths, service_id) { - for note in crate::serve_summary::append_oom_serve_note( - Vec::new(), - &record.engine, - false, - "starting", - Some(&record.log_path), - ) { - let _ = write!(message, "\n note: {note}"); - } - } - Some(anyhow!(message)) -} - -/// Record the outcome of a finished wait on the manifest as it is now, changing -/// only what the wait learned. The copy read at spawn time can be long stale: -/// `rocmd` may have recorded a restart or a new supervisor pid, and `rocm logs` -/// refreshes the record, so writing it back whole would revert those. -fn record_wait_outcome( - paths: &AppPaths, - held: &mut ManagedServiceRecord, - status: &str, - verified: bool, -) -> Result<()> { - let mut latest = ManagedServiceRecord::read_manifest(paths, &held.service_id) - .unwrap_or_else(|_| held.clone()); - status.clone_into(&mut latest.status); - if verified { - // Latch the verification the wait just performed, so the readiness checks - // behind `services list` and chat read it instead of re-probing. - latest.inference_verified_at_unix_ms = Some(rocm_core::unix_time_millis() as u64); - } - latest.write()?; - *held = latest; - Ok(()) -} - -/// Test-only fault-injection switch, off (`const false`) in shipped binaries: -/// the `e2e-oom-fault-injection` feature is enabled only by the `xtask e2e` -/// mock-lane build, never by the release build. When enabled and -/// `ROCM_E2E_SIMULATE_OOM_LAUNCH=1` is set on the process, `rocm serve` fabricates -/// the on-disk state of a managed vLLM launch that spawned, wrote an -/// out-of-memory traceback to the log it owns, and never became ready — so the -/// positive OOM-guidance summary path can be verified on a GPU-less CI host, -/// where a real launch cannot run. See [`simulate_oom_managed_launch`]. -#[cfg(feature = "e2e-oom-fault-injection")] -pub(crate) fn e2e_simulate_oom_launch() -> bool { - std::env::var_os("ROCM_E2E_SIMULATE_OOM_LAUNCH").is_some_and(|value| value == "1") -} - -/// Release builds carry no fault-injection switch: the seam compiles out -/// entirely, so `rocm serve` behaves identically to a build without the feature. -#[cfg(not(feature = "e2e-oom-fault-injection"))] -pub(crate) const fn e2e_simulate_oom_launch() -> bool { - false -} - -/// Test-only (see [`e2e_simulate_oom_launch`]): fabricate the on-disk state of a -/// managed vLLM launch that spawned, OOM'd, and never became ready, then return -/// the launch report the summary path would see for it — `already_running: false` -/// (this invocation launched it), `status: "starting"` (it failed to become -/// ready), and a `log_path` whose contents carry a real allocator OOM signature. -/// [`crate::serve_summary::append_oom_serve_note`] then renders the memory-knob guidance. No process is -/// spawned. Compiled out of shipped binaries. -#[cfg(feature = "e2e-oom-fault-injection")] -#[allow(clippy::too_many_arguments)] -fn simulate_oom_managed_launch( - paths: &AppPaths, - engine: &str, - service_id: &str, - requested_model: &str, - resolve: &ResolveModelResponse, - host: &str, - port: u16, - device_policy: &DevicePolicy, - gpu_indices: &[u32], - runtime_id: Option<&str>, - env_id: Option<&str>, -) -> Result { - paths.ensure()?; - fs::create_dir_all(paths.services_dir())?; - let mut record = ManagedServiceRecord::new( - paths, - service_id, - engine, - requested_model, - resolve.canonical_model_id.clone(), - host, - port, - "managed", - std::process::id(), - runtime_id.map(str::to_owned), - env_id.map(str::to_owned), - Some(device_policy_name(device_policy).to_owned()), - ); - record.gpu_indices = gpu_indices.to_vec(); - record.status = "starting".to_owned(); - record.write()?; - // A real allocator OOM signature in the log this invocation owns, so the - // summary renders memory guidance for a launch that actually OOM'd. - // - // The engine's own error block is reproduced below it, because that is what - // a real managed launch leaves on disk, and leaving it out hid a bug this - // scenario should have caught: the supervisor and the engine share one log, - // so when vLLM OOMs the engine bails with its startup-log context — which - // embeds rocm-cli's own OOM hint — and `exit_code_for` writes that whole - // error back into the same file, *after* the real line. A fixture carrying - // only the raw OOM line can never see the newest-first scan land on - // rocm-cli's own sentence instead of the user's error. - let real_line = "torch.OutOfMemoryError: HIP out of memory. Tried to allocate 7.21 GiB."; - fs::write( - &record.log_path, - format!( - "{real_line}\n\ - Error: vLLM server process exited before becoming ready (status: exit status: 1)\n\ - \n\ - Startup log, last {} lines as the terminal drew them:\n\ - {real_line}\n\ - \n\ - {}{real_line}). {}\n\ - For conditional remediation, run `{} 'vllm: {real_line}'`.\n", - rocm_engine_protocol::DEFAULT_LOG_TAIL_LINES, - rocm_core::VLLM_OOM_DETECTED_PREFIX, - rocm_core::VLLM_GPU_MEMORY_UTILIZATION_HINT, - rocm_core::VLLM_OOM_DIAGNOSE_COMMAND, - ), - ) - .with_context(|| format!("failed to write {}", record.log_path.display()))?; - Ok(ManagedLaunchReport { - service_id: service_id.to_owned(), - endpoint_url: format!("{}/v1", format_http_base_url(host, port)), - status: "starting".to_owned(), - already_running: false, - child_pid: None, - log_path: Some(record.log_path), - manifest_path: Some(record.manifest_path), - }) -} - -/// Probe `record`'s endpoint until it answers: once within the readiness -/// budget, or (with `wait_until_ready`) for as long as the server stays alive. -fn wait_for_record_ready( - paths: &AppPaths, - record: &ManagedServiceRecord, - wait_until_ready: bool, - endpoint_api_key: Option<&str>, - on_wait_tick: &mut dyn FnMut(Duration), -) -> EndpointReadiness { - let probe = |on_tick: &mut dyn FnMut(Duration)| { - wait_for_service_http_ready_with_progress( - &record.engine, - &record.host, - record.port, - &|| { - let mut latest = record.clone(); - let _ = latest.refresh_from_engine_state(); - latest.served_model_name().to_owned() - }, - endpoint_api_key, - READINESS_WAIT, - on_tick, - ) - }; - if wait_until_ready { - wait_past_readiness_budget( - || probe(&mut *on_wait_tick), - || service_still_live(paths, &record.service_id, service_pid_is_alive), - ) - } else { - probe(on_wait_tick) - } -} - -/// An equivalent service is already live. Without `--wait`, or once it is -/// serving, report it as is; otherwise wait for it like a fresh launch would. -fn wait_on_existing_service( - paths: &AppPaths, - mut report: ManagedLaunchReport, - wait_until_ready: bool, - on_wait_begin: &mut dyn FnMut(&str, u16), - on_wait_tick: &mut dyn FnMut(Duration), -) -> Result { - if !wait_until_ready || report.status == "ready" { - return Ok(report); - } - let Ok(mut record) = ManagedServiceRecord::read_manifest(paths, &report.service_id) else { - return Ok(report); - }; - // The id of the server being waited on, which is not the caller's freshly - // generated one when it attached to an existing server. - on_wait_begin(&report.service_id, record.port); - // The key the caller generated belongs to a launch that did not happen; the - // existing service answers to its own. - let stored_key = endpoint_keys::endpoint_api_key(paths, &report.service_id); - let readiness = - wait_for_record_ready(paths, &record, true, stored_key.as_deref(), on_wait_tick); - if let Some(error) = died_during_wait( - paths, - &report.service_id, - true, - readiness, - service_pid_is_alive, - ) { - return Err(error); - } - report.status = status_for_readiness(readiness).to_owned(); - record_wait_outcome( - paths, - &mut record, - &report.status, - readiness == EndpointReadiness::Serving, - )?; - Ok(report) -} - -/// Launch a managed service in the background: run the locked allocation and -/// spawn transaction, release the launch lock once the record is published, -/// then wait for HTTP readiness. -/// -/// Drops the endpoint key `serve()` freshly staged for this attempt if the -/// launch fails before any `ManagedServiceRecord` exists to own it (port -/// reserved/in use, range exhausted, other bind error) — such a failure -/// returns before `abandon_failed_managed_launch` ever runs, so nothing else -/// would clear it. Scoped to the spawn call only, mirroring -/// [`run_attached_service`]'s identical guard: a later failure (the readiness -/// wait, `record.write()`) leaves a record a live child actually owns, so it -/// must not also wipe that key. -#[allow(clippy::too_many_arguments)] -pub(crate) fn start_managed_service( - paths: &AppPaths, - engine: &str, - service_id: &str, - requested_model: &str, - resolve: &ResolveModelResponse, - host: &str, - port_request: serve_port::PortRequest, - device_policy: &DevicePolicy, - gpu_indices: &[u32], - runtime_id: Option<&str>, - env_id: Option<&str>, - engine_recipe: Option<&EngineRecipeHint>, - endpoint_api_key: Option<&str>, - launch_lock: rocm_core::FileLock, - require_api_key: bool, - gpu_refusal_waived_for_reuse: bool, - wait_until_ready: bool, - on_wait_begin: &mut dyn FnMut(&str, u16), - on_wait_tick: &mut dyn FnMut(Duration), -) -> Result { - #[cfg(feature = "e2e-oom-fault-injection")] - if e2e_simulate_oom_launch() { - return simulate_oom_managed_launch( - paths, - engine, - service_id, - requested_model, - resolve, - host, - port_request - .explicit() - .unwrap_or(rocm_core::DEFAULT_LOCAL_PORT), - device_policy, - gpu_indices, - runtime_id, - env_id, - ); - } - let (mut record, child_pid) = match spawn_managed_engine_child( - paths, - engine, - service_id, - requested_model, - resolve, - host, - port_request, - device_policy, - gpu_indices, - runtime_id, - env_id, - engine_recipe, - require_api_key, - gpu_refusal_waived_for_reuse, - ) - .inspect_err(|_| { - drop_orphaned_endpoint_key_on_already_running(paths, service_id, endpoint_api_key); - })? { - ManagedSpawn::AlreadyRunning(report) => { - // `--wait` applies to a service that is still starting too: the - // natural retry after `rocm serve m` printed `starting`. - drop(launch_lock); - return wait_on_existing_service( - paths, - report, - wait_until_ready, - on_wait_begin, - on_wait_tick, - ); - } - ManagedSpawn::Spawned { record, child_pid } => (*record, child_pid), - }; - // The transaction resolved the request to one concrete port; everything - // below — readiness, the endpoint URL, the audit line — uses that. - let port = record.port; - // The claiming service record is now persisted, so the selected GPU is - // visible to any concurrent auto-selection. Release the launch lock before - // the readiness wait below, which can block for many seconds — holding it - // that long would needlessly serialize unrelated serves. - drop(launch_lock); - - #[cfg(windows)] - thread::sleep(Duration::from_millis(200)); - - if wait_until_ready { - on_wait_begin(service_id, port); - } - let readiness = wait_for_record_ready( - paths, - &record, - wait_until_ready, - endpoint_api_key, - on_wait_tick, - ); - if let Some(error) = died_during_wait( - paths, - service_id, - wait_until_ready, - readiness, - service_pid_is_alive, - ) { - return Err(error); - } - let launch_status = status_for_readiness(readiness); - record_wait_outcome( - paths, - &mut record, - launch_status, - readiness == EndpointReadiness::Serving, - )?; - let endpoint_url = format!("{}/v1", format_http_base_url(host, port)); - record_cli_audit_event( - paths, - "service", - "managed_service_launch", - "info", - format!( - "launched managed service engine={} model={} endpoint={} readiness={}", - engine, resolve.canonical_model_id, endpoint_url, launch_status - ), - Some(service_id), - ); - Ok(ManagedLaunchReport { - service_id: service_id.to_owned(), - endpoint_url, - status: launch_status.to_owned(), - already_running: false, - child_pid: Some(child_pid), - log_path: Some(record.log_path), - manifest_path: Some(record.manifest_path), - }) -} - -/// Reproduce the original plain, line-by-line managed-launch output. Used for -/// non-interactive output (piped, CI, the chat assistant's `serve --managed`), -/// where the animated summary is inappropriate. The interactive path renders the -/// summary table via [`crate::serve_summary`] instead. -pub(crate) fn print_managed_launch_plain( - report: &ManagedLaunchReport, - endpoint_api_key: Option<&str>, -) { - if report.already_running { - println!("managed service already running"); - println!(" service_id: {}", report.service_id); - println!(" endpoint: {}", report.endpoint_url); - println!(" status: {}", report.status); - println!(" note: existing service detected; no second process spawned"); - return; - } - println!("managed service launched"); - println!(" service_id: {}", report.service_id); - if let Some(child_pid) = report.child_pid { - println!(" process_pid: {child_pid}"); - } - println!(" endpoint: {}", report.endpoint_url); - if let Some(key) = endpoint_api_key { - // Intentional one-time display of a freshly generated API key to the - // terminal so the user can copy it — the designed delivery channel - // documented on `render_endpoint_client_config`, not a log. The tag below - // is currently inert (Rust's CodeQL pack has no AlertSuppression.ql yet — - // github/codeql#21637) but will start working once that lands, since the - // tag must be the single line immediately before the flagged code. - print!( - // codeql[rust/cleartext-logging] - "{}", - render_endpoint_client_config(&report.endpoint_url, key) - ); - } - if let Some(log_path) = report.log_path.as_deref() { - println!(" log_path: {}", log_path.display()); - } - if let Some(manifest_path) = report.manifest_path.as_deref() { - println!(" manifest_path: {}", manifest_path.display()); - } - println!(" readiness: {}", report.status); -} - -/// The client configuration to show when a `--wait` begins, so a key generated for -/// a public bind reaches the user even if the wait is interrupted or fails. -/// `None` unless the wait is on the launch that key belongs to: an attach to an -/// already-starting server (`waiting_on` differs from `service_id`) answers to -/// its own key, which is not shown here. -pub(crate) fn wait_begin_client_config( - waiting_on: &str, - service_id: &str, - host: &str, - port: u16, - api_key: Option<&str>, -) -> Option { - let key = api_key.filter(|_| waiting_on == service_id)?; - let endpoint_url = format!("{}/v1", format_http_base_url(host, port)); - Some(render_endpoint_client_config(&endpoint_url, key)) -} - -/// Render the one-time secure client configuration for a public, authenticated -/// endpoint. This is the *intended* channel for delivering the key to the user -/// (unlike logs/status, which must never contain it) — it prints the key once at -/// launch alongside a ready-to-use example. Callers only invoke this for a -/// non-loopback bind that generated/received a key. -fn render_endpoint_client_config(endpoint_url: &str, api_key: &str) -> String { - let mut out = String::new(); - let _ = writeln!(out, " api key: {api_key}"); - let _ = writeln!( - out, - " note: this key is shown only now - clients must send `Authorization: Bearer `" - ); - let _ = writeln!( - out, - " example: curl -H \"Authorization: Bearer {api_key}\" {endpoint_url}/models" - ); - out -} - -/// Shared daemon-lifecycle entrypoint: ensures the background automation helper -/// (`rocm daemon`) is running when an automation watcher needs it, spawning it -/// detached if not. Liveness is read from the file-based automation runtime -/// state. Intentionally `pub(crate)` — reused by both the `serve --managed` path -/// and `automations enable`. Only the spawn result itself (`command.spawn()` / -/// `spawn_detached_no_inherit`) is logged rather than propagated; setup errors -/// (path discovery, config load, stdio attach) still return `Err`. -/// -/// The daemon's only work is running enabled automation watchers (server -/// recovery included — `server-recover` is one of them and is off by default), -/// and it exits on its own once none is enabled. So with no watcher enabled -/// nothing is spawned: a `serve` would otherwise leave a no-op daemon running -/// long after its service was stopped (ROCMAI-827). -pub(crate) fn ensure_background_helper_running() -> Result<()> { - ensure_background_helper_running_quiet(false) -} - -/// As [`ensure_background_helper_running`], but suppresses the stdout status line -/// when `quiet` is set. The interactive `rocm serve` summary path uses `quiet` so -/// the daemon-spawn note does not appear above the deployment summary table. -pub(crate) fn ensure_background_helper_running_quiet(quiet: bool) -> Result<()> { - let paths = AppPaths::discover()?; - ensure_background_helper_running_with(&paths, quiet, spawn_background_helper, &|| {}) -} - -/// The body of [`ensure_background_helper_running_quiet`], with the actual -/// process spawn injected so the decision around it is testable without -/// launching a daemon. `spawn` returns `Err` for a setup failure (propagated) -/// and `Ok(Err(..))` for a failed spawn (reported, not fatal). `on_lock_busy` is -/// called each time the autostart lock is found held by another process, which -/// lets a test wait for this call to be genuinely queued behind it. -fn ensure_background_helper_running_with( - paths: &AppPaths, - quiet: bool, - spawn: impl FnOnce(&AppPaths) -> Result>, - on_lock_busy: &dyn Fn(), -) -> Result<()> { - // An unreadable config is "no watcher": this also runs after `serve` has - // already launched its child, where failing would report a launch that - // happened as an error. - if !rocm_core::RocmCliConfig::load(paths).is_ok_and(|config| config.automation_daemon_enabled()) - { - return Ok(()); - } - - // Two concurrent callers (e.g. two `rocm serve`) can both read "not running" - // and each spawn a daemon. Serialize the decision on a lock file — the first - // holder spawns, later holders observe the now-running daemon and return - // without spawning. - // - // The "already running" read is deliberately made only under the lock, with - // no lock-free fast path. The daemon exits by itself once the last watcher - // is disabled, and it decides that under this same lock and publishes - // `running = false` before releasing it. Caller order is: save the enabling - // config, then come here. A lock-free read could still see the daemon that is - // about to exit as running, spawn nothing, and leave a watcher enabled with - // no daemon; under the lock the daemon has either seen the new config and - // stays, or has already published that it stopped. - let _autostart_lock = loop { - if let Some(lock) = - rocm_core::FileLock::try_acquire(paths.automation_autostart_lock_path())? - { - break lock; - } - on_lock_busy(); - std::thread::sleep(std::time::Duration::from_millis(10)); - }; - if background_helper_already_running(paths)? { - return Ok(()); - } - - // The lock alone does not close the window: the spawned daemon does not - // publish its `running` runtime state until well after `spawn()` (clap parse, - // runtime build, config load, banner flush). A second caller that acquires - // this lock during that gap still sees "not running" and would spawn a - // duplicate. Bridge the gap with a short-lived claim recording the child PID - // and spawn time: a holder that finds a live, recent claim defers instead. - let claim_path = paths.automation_autostart_claim_path(); - if autostart_spawn_in_flight( - read_autostart_claim(&claim_path), - now_unix_millis(), - AUTOSTART_CLAIM_TTL_MS, - rocm_core::process_is_running, - ) { - return Ok(()); - } - - match spawn(paths)? { - Ok(daemon_pid) => { - // Record the claim before returning (and thus releasing the lock) so a - // concurrent holder in the spawn→publish window defers. Best-effort: a - // failed write only reopens the original, already-tolerated race. - let _ = write_autostart_claim( - &claim_path, - AutostartClaim { - daemon_pid, - spawned_at_ms: now_unix_millis(), - }, - ); - if !quiet { - println!(" helper: started background automation daemon"); - } - } - Err(error) if !quiet => { - println!(" helper: could not start background automation daemon: {error}"); - } - Err(_) => {} - } - Ok(()) -} - -/// Spawn `rocm daemon` detached from this process. The outer `Result` is a setup -/// failure (resolving the executable, attaching stdio); the inner one is the -/// spawn itself. -fn spawn_background_helper(paths: &AppPaths) -> Result> { - let exe = managed_service_launcher_path() - .context("failed to resolve current rocm executable path")?; - let args = vec!["daemon".to_owned()]; - #[cfg(windows)] - let spawn_result = { - let env_values = app_path_env_var_values(paths, None); - let env_refs = app_path_env_var_refs(&env_values); - rocm_core::spawn_detached_no_inherit(&exe, &args, &env_refs) - }; - #[cfg(not(windows))] - let spawn_result = { - let mut command = managed_service_process_command(&exe, &args); - command.stdin(Stdio::null()); - attach_background_stdio(&mut command, None)?; - detach_background_command(&mut command); - apply_app_path_env(&mut command, paths); - command - .spawn() - .map(|child| child.id()) - .map_err(anyhow::Error::from) - }; - Ok(spawn_result) -} - -/// How long an autostart claim is honoured before it is treated as stale even if -/// its recorded PID is still alive. Comfortably longer than a cold daemon boot -/// (clap parse → runtime build → config load → state publish) yet short enough -/// that a crashed spawn cannot suppress autostart for long. -const AUTOSTART_CLAIM_TTL_MS: u128 = 30_000; - -/// A just-spawned daemon's autostart claim: the child PID and the wall-clock time -/// (milliseconds since the Unix epoch) the spawn was recorded. It lets a -/// concurrent autostart holder distinguish a live, in-flight spawn from a stale -/// leftover. Serialized as a single `" "` line — no dependency and -/// trivially forward-compatible. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct AutostartClaim { - daemon_pid: u32, - spawned_at_ms: u128, -} - -/// Milliseconds since the Unix epoch, or `0` if the clock is before the epoch -/// (which only makes a fresh claim look old — safe, it just permits a respawn). -fn now_unix_millis() -> u128 { - std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map_or(0, |elapsed| elapsed.as_millis()) -} - -/// Read an autostart claim, returning `None` when the file is absent or -/// unparseable (either is treated as "no claim", so a respawn is permitted). -fn read_autostart_claim(path: &Path) -> Option { - let text = std::fs::read_to_string(path).ok()?; - let mut parts = text.split_whitespace(); - let daemon_pid = parts.next()?.parse().ok()?; - let spawned_at_ms = parts.next()?.parse().ok()?; - Some(AutostartClaim { - daemon_pid, - spawned_at_ms, - }) -} - -/// Write an autostart claim as `" "`. Best-effort at the call site. -fn write_autostart_claim(path: &Path, claim: AutostartClaim) -> std::io::Result<()> { - std::fs::write( - path, - format!("{} {}", claim.daemon_pid, claim.spawned_at_ms), - ) -} - -/// Whether an existing autostart `claim` means a daemon spawn is still in flight, -/// so the current lock holder should defer rather than spawn a duplicate. A claim -/// counts as in-flight only while its child PID is alive *and* it is younger than -/// `ttl_ms` — the TTL bounds how long a crashed spawn (or a PID later reused by an -/// unrelated process) can suppress autostart. `pid_alive` is injected so the -/// decision is unit-testable without a live process. -fn autostart_spawn_in_flight( - claim: Option, - now_ms: u128, - ttl_ms: u128, - pid_alive: impl Fn(u32) -> bool, -) -> bool { - claim.is_some_and(|claim| { - now_ms.saturating_sub(claim.spawned_at_ms) < ttl_ms && pid_alive(claim.daemon_pid) - }) -} - -/// What ended an attached (`--verbose`/`--foreground`) streaming session. Kept -/// as a plain enum, separate from any terminal I/O, so the follow-up action -/// (detach note vs. stop the server) is unit-testable without a TTY. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum AttachOutcome { - /// Ctrl-D: leave the server running and hand the terminal back. - Detach, - /// Ctrl-C: stop the server, then hand the terminal back. - Stop, - /// The engine process exited on its own while we were streaming its log. - ServerExited, -} - -/// Attached serve path for `--verbose`/`--foreground`: spawn the engine as a -/// detached managed child (the same child the background path spawns) and stream -/// its log in this terminal. Unlike the old in-process foreground, the server -/// survives the session — Ctrl-D detaches and leaves it running, Ctrl-C stops it. -/// -/// `stop_service` is how Ctrl-C stops the server: the caller passes the managed -/// stop, so this module does not import the stop path, which itself imports this -/// module's spawn helpers. It returns the same `stop_server` answer -/// [`finish_attached_stop`] reads. -#[allow(clippy::too_many_arguments)] -pub(crate) fn run_attached_service( - paths: &AppPaths, - engine: &str, - service_id: &str, - requested_model: &str, - resolve: &ResolveModelResponse, - host: &str, - port_request: serve_port::PortRequest, - gpu_indices: &[u32], - runtime_id: Option<&str>, - env_id: Option<&str>, - endpoint_api_key: Option<&str>, - launch_lock: rocm_core::FileLock, - require_api_key: bool, - gpu_refusal_waived_for_reuse: bool, - stop_service: impl FnOnce(&AppPaths, &str) -> Result, -) -> Result<()> { - let spawn = spawn_managed_engine_child( - paths, - engine, - service_id, - requested_model, - resolve, - host, - port_request, - &resolve.device_policy, - gpu_indices, - runtime_id, - env_id, - resolve.engine_recipe.as_ref(), - require_api_key, - gpu_refusal_waived_for_reuse, - ) - // Mirrors `start_managed_service`'s cleanup: a pre-spawn - // failure (port reserved/in use, range exhausted, other bind error) returns - // before any service record exists, so nothing else clears the key just - // stored for this attempt. - .inspect_err(|_| { - drop_orphaned_endpoint_key_on_already_running(paths, service_id, endpoint_api_key); - })?; - // The claiming record is persisted (or an existing service was found), so the - // selected GPU is now visible to concurrent auto-selection. Release the launch - // lock before streaming logs, which blocks for the whole attached session. - drop(launch_lock); - - // Attached mode is a managed record too, so it shares the transaction's - // resolved concrete port. - let (service_id, log_path, port, child_pid) = match spawn { - // A server for this engine+model is already live. Don't fight it for the - // port — point the user at the existing one instead of tailing a log we - // did not start. - ManagedSpawn::AlreadyRunning(report) => { - println!("model already being served"); - println!(" service_id: {}", report.service_id); - println!(" endpoint: {}", report.endpoint_url); - println!(" status: {}", report.status); - println!(" logs: rocm logs {}", report.service_id); - println!( - " stop: {}", - service_hints::service_stop_hint(&report.service_id) - ); - drop_orphaned_endpoint_key_on_already_running(paths, service_id, endpoint_api_key); - return Ok(()); - } - ManagedSpawn::Spawned { record, child_pid } => ( - service_id.to_owned(), - record.log_path.clone(), - record.port, - child_pid, - ), - }; - - // The child is a managed service that outlives this session once detached. - // The background helper adds one thing for it: with the `server-recover` - // watcher enabled, it health-checks managed servers and restarts failed ones. - // Every duty of the helper is watcher-gated, so with no watcher enabled no - // helper is started. Nothing else depends on it: a self-exited server's - // record is settled by its own supervisor (`supervise_service` writes the - // exit via `record_supervised_exit` in `apps/rocmd/src/supervise.rs`) and on - // every read (`refresh_managed_service_runtime_liveness`), and the dashboard - // has its own embedded daemon. Match the background ordering (spawn, then - // ensure the helper) and keep it quiet so no status line breaks into the - // log stream. - ensure_background_helper_running_quiet(true)?; - - // The resolution detail (model, engine, runtime, GPU, warnings) was already - // printed as the "serve plan" block in `serve()`; extend it with the launch - // coordinates and the streaming hint rather than repeating it. - let endpoint = format!("{}/v1", format_http_base_url(host, port)); - println!(" service_id: {service_id}"); - println!(" endpoint: {endpoint}"); - if let Some(key) = endpoint_api_key { - // Same intentional one-time key display as `print_managed_launch_plain` - // above; see its rationale for why the tag below is currently inert. - // codeql[rust/cleartext-logging] - print!("{}", render_endpoint_client_config(&endpoint, key)); - } - println!(" streaming engine logs - Ctrl-D detaches (leaves it running), Ctrl-C stops it"); - println!(); - - let outcome = stream_attached_logs(&log_path, child_pid)?; - println!(); - - match outcome { - AttachOutcome::Detach => { - // Audit before printing: a closed stdout makes `println!` panic, - // which must not lose the event. - record_cli_audit_event( - paths, - "service", - "serve_detach", - "info", - format!("detached from attached serve service_id={service_id} endpoint={endpoint}"), - Some(&service_id), - ); - println!("detached - server still running"); - println!(" service_id: {service_id}"); - println!(" endpoint: {endpoint}"); - println!(" list: rocm services"); - println!(" logs: rocm logs {service_id}"); - println!(" stop: {}", service_hints::service_stop_hint(&service_id)); - Ok(()) - } - AttachOutcome::Stop => { - println!("stopping server..."); - let stop = stop_service(paths, &service_id); - finish_attached_stop( - paths, - &service_id, - child_pid, - stop, - rocm_core::terminate_process_tree, - &mut io::stdout().lock(), - &mut io::stderr().lock(), - ) - } - AttachOutcome::ServerExited => { - println!("server process exited"); - println!(" service_id: {service_id}"); - println!(" recent logs: rocm logs {service_id}"); - Ok(()) - } - } -} - -/// Restores cooked terminal mode when dropped, so [`stream_attached_logs`] leaves -/// the terminal usable on every exit path (normal return, `?` error, or panic). -struct RawModeGuard; - -impl Drop for RawModeGuard { - fn drop(&mut self) { - let _ = crossterm::terminal::disable_raw_mode(); - } -} - -/// Map a key press (control modifier + lowercased character) to the attach -/// action it triggers, if any. Factored out of the raw-mode reader loop so the -/// Ctrl-D/Ctrl-C mapping is unit-testable without a terminal. -const fn detach_key_outcome(ctrl: bool, ch: char) -> Option { - if !ctrl { - return None; - } - match ch { - 'c' => Some(AttachOutcome::Stop), - 'd' => Some(AttachOutcome::Detach), - _ => None, - } -} - -/// Follow `log_path` in the terminal until the user presses Ctrl-D (detach) or -/// Ctrl-C (stop), or the engine process exits. Uses crossterm raw mode to -/// capture the keys directly (in raw mode Ctrl-C does not raise SIGINT, so we see -/// it as a key event). When stdin is not a TTY (piped/CI), keystroke capture is -/// impossible, so we follow the log until the process exits instead. -fn stream_attached_logs(log_path: &Path, child_pid: u32) -> Result { - use std::io::IsTerminal as _; - use std::sync::Arc; - use std::sync::atomic::{AtomicBool, Ordering}; - use std::sync::mpsc; - - if !std::io::stdin().is_terminal() { - return stream_attached_logs_no_tty(log_path, child_pid); - } - - // Enter raw mode *before* spawning the key reader. In raw mode Ctrl-C arrives - // as a key event instead of SIGINT; if the reader started first, a Ctrl-C in - // that window would kill the CLI outright (leaving the detached child alive - // but printing no detach/stop message). The guard restores cooked mode on - // every exit path (normal return, `?` error, panic). - crossterm::terminal::enable_raw_mode().context("failed to enter raw terminal mode")?; - let _raw_guard = RawModeGuard; - - let stop = Arc::new(AtomicBool::new(false)); - let (tx, rx) = mpsc::channel::(); - - let reader_stop = Arc::clone(&stop); - let reader = thread::spawn(move || { - use crossterm::event::{self, Event, KeyCode, KeyEventKind, KeyModifiers}; - while !reader_stop.load(Ordering::Relaxed) { - match event::poll(Duration::from_millis(100)) { - Ok(true) => match event::read() { - Ok(Event::Key(key)) if key.kind != KeyEventKind::Release => { - let outcome = match key.code { - KeyCode::Char(ch) => detach_key_outcome( - key.modifiers.contains(KeyModifiers::CONTROL), - ch.to_ascii_lowercase(), - ), - _ => None, - }; - if let Some(outcome) = outcome { - let _ = tx.send(outcome); - break; - } - } - Ok(_) => {} - Err(_) => break, - }, - Ok(false) => {} - Err(_) => break, - } - } - }); - - let mut stdout = io::stdout(); - let mut log_reader: Option> = None; - let mut line = String::new(); - let outcome = loop { - if log_reader.is_none() { - log_reader = fs::File::open(log_path).ok().map(io::BufReader::new); - } - if let Some(reader) = log_reader.as_mut() { - loop { - line.clear(); - match reader.read_line(&mut line) { - Ok(0) => break, - Ok(_) => { - // Raw mode disables the terminal's own \n -> \r\n - // translation, so emit an explicit carriage return to - // keep the log left-aligned instead of stair-stepping. - let _ = write!(stdout, "{}\r\n", line.trim_end_matches('\n')); - let _ = stdout.flush(); - } - Err(_) => break, - } - } - } - - if let Ok(signal) = rx.try_recv() { - break signal; - } - if !process_is_running(child_pid) { - break AttachOutcome::ServerExited; - } - thread::sleep(Duration::from_millis(150)); - }; - - stop.store(true, Ordering::Relaxed); - let _ = reader.join(); - Ok(outcome) -} - -/// Non-interactive fallback for [`stream_attached_logs`]: no keystroke capture, -/// so just follow the log until the (detached) engine process exits. A Ctrl-C -/// here delivers SIGINT to this process and leaves the managed server running. -fn stream_attached_logs_no_tty(log_path: &Path, child_pid: u32) -> Result { - let mut stdout = io::stdout(); - let mut log_reader: Option> = None; - let mut line = String::new(); - loop { - if log_reader.is_none() { - log_reader = fs::File::open(log_path).ok().map(io::BufReader::new); - } - if let Some(reader) = log_reader.as_mut() { - loop { - line.clear(); - match reader.read_line(&mut line) { - Ok(0) => break, - Ok(_) => { - let _ = write!(stdout, "{line}"); - let _ = stdout.flush(); - } - Err(_) => break, - } - } - } - if !process_is_running(child_pid) { - return Ok(AttachOutcome::ServerExited); - } - thread::sleep(Duration::from_millis(200)); - } -} - -/// Finish the attached `rocm serve` Ctrl-C stop once the managed stop has -/// answered: what is printed where, what is signalled, what is audited, and -/// how the command exits. Everything but the stop itself and the real -/// signal is here, so a test can drive it. -/// -/// - Confirmed: the report, with the background-helper note `rocm services -/// stop` also prints, goes to `out` and the command succeeds. -/// - Unconfirmed: the same report `rocm services stop` gives, check lines -/// included, becomes the error, so it reaches stderr and the command exits -/// non-zero. Nothing is signalled on the stop's behalf: the stop has already -/// done what it safely could. -/// - The stop itself failed: `signal_child` gets the child this session -/// launched, so Ctrl-C never leaves the model orphaned when the stop path -/// breaks. The stop's error, and the signal's if that fails too, are noted on -/// `err`, and the audit event records whether the signal worked. The command -/// still exits 0, as it always has here. -fn finish_attached_stop( - paths: &AppPaths, - service_id: &str, - child_pid: u32, - stop: Result, - signal_child: impl FnOnce(u32) -> Result<()>, - out: &mut dyn Write, - err: &mut dyn Write, -) -> Result<()> { - match stop { - Ok(result) if service_stop_unconfirmed(&result) => { - record_cli_audit_event( - paths, - "service", - "serve_stop", - "error", - format!("stop of attached serve not confirmed service_id={service_id}"), - Some(service_id), - ); - bail!( - "{}", - render_service_action_result("stop_server", &result).trim_end() - ) - } - Ok(result) => { - record_cli_audit_event( - paths, - "service", - "serve_stop", - "info", - format!("stopped attached serve service_id={service_id}"), - Some(service_id), - ); - write!( - out, - "{}", - render_service_action_result_with_helper_note(paths, "stop_server", &result) - )?; - Ok(()) - } - Err(error) => { - let signal = signal_child(child_pid); - // Record the audit event before writing to stderr, so a failed - // write cannot skip it. - let message = match &signal { - Ok(()) => format!( - "stop failed; signaled attached serve child {child_pid} service_id={service_id}" - ), - Err(signal_error) => format!( - "stop failed; signaling attached serve child {child_pid} also failed: {signal_error:#} service_id={service_id}" - ), - }; - record_cli_audit_event( - paths, - "service", - "serve_stop", - if signal.is_ok() { "warn" } else { "error" }, - message, - Some(service_id), - ); - writeln!(err, " note: {error:#}")?; - if let Err(signal_error) = &signal { - writeln!( - err, - " note: signaling this session's server process {child_pid} directly also failed: {signal_error:#}" - )?; - } - Ok(()) - } - } -} - -#[cfg(test)] -mod tests { - #[test] - fn the_no_gpu_refusal_sends_a_bare_metal_user_to_dev_kfd() { - let text = super::no_usable_gpu_refusal_text("gpu_required", false); - assert!(text.contains("confirm /dev/kfd is present"), "{text}"); - assert!(!text.contains("ROCDXG"), "{text}"); - } - - #[test] - fn the_no_gpu_refusal_does_not_send_a_wsl_user_to_dev_kfd() { - // WSL has no /dev/kfd, so advice to confirm it can never clear the - // condition. The text names the status `examine` reports for the path - // the probe asked about, taken from the constant the producer uses, so - // the two cannot drift. That the status really is the one a refused - // host shows is asserted end to end by serve-42. - let text = super::no_usable_gpu_refusal_text("gpu_required", true); - assert!(!text.contains("/dev/kfd is present"), "{text}"); - assert!(text.contains("rocm examine"), "{text}"); - assert!(text.contains("rocm diagnose"), "{text}"); - // The advice is true only because `rocdxg_ready` counts a library on - // LD_LIBRARY_PATH; `a_rocdxg_only_on_the_loader_path_is_ready` in - // `rocm-core`'s host_gpu/wsl.rs proves that behavior. - assert!(text.contains("LD_LIBRARY_PATH"), "{text}"); - assert!( - text.contains(&format!("`{}`", rocm_core::WSL_ROCDXG_READY)), - "{text}" - ); - } - - #[test] - fn the_no_gpu_refusal_names_the_policy_on_both_platforms() { - for on_wsl in [false, true] { - let text = super::no_usable_gpu_refusal_text("gpu_required", on_wsl); - assert!( - text.starts_with("no usable AMD GPU detected; `rocm serve` requires a GPU under the gpu_required policy"), - "{text}" - ); - } - } - use super::*; - use crate::test_support::{runtime_state, set_helper_facts}; - - #[test] - fn a_child_that_exited_during_the_startup_window_is_reported_with_its_log() -> Result<()> { - let dir = tempfile::tempdir()?; - let log = dir.path().join("engine.log"); - fs::write(&log, "boom: engine could not start\n")?; - let settled = std::cell::Cell::new(false); - let error = ensure_spawned_child_survives_startup( - 4242, - &log, - || settled.set(true), - |pid| { - assert_eq!(pid, 4242); - assert!(settled.get(), "liveness must be probed after the wait"); - false - }, - ) - .expect_err("a dead child must fail the launch"); - assert!(settled.get(), "the startup window must elapse first"); - assert!(format!("{error:#}").contains("boom"), "{error:#}"); - ensure_spawned_child_survives_startup(4242, &log, || {}, |_| true)?; - Ok(()) - } - use crate::tests::test_paths; - use rocm_core::DEFAULT_LOCAL_HOST; - - #[test] - fn wait_past_readiness_budget_keeps_probing_a_live_starting_server() { - let mut readings = vec![ - EndpointReadiness::Unreachable, - EndpointReadiness::Unreachable, - EndpointReadiness::Serving, - ] - .into_iter(); - let mut probes = 0; - let result = wait_past_readiness_budget( - || { - probes += 1; - readings.next().expect("no probe after Serving") - }, - || true, - ); - assert_eq!(result, EndpointReadiness::Serving); - assert_eq!(probes, 3, "must outlast the first (45s) budget"); - } - - #[test] - fn wait_past_readiness_budget_stops_when_the_server_exits() { - let mut probes = 0; - let result = wait_past_readiness_budget( - || { - probes += 1; - EndpointReadiness::Unreachable - }, - || false, - ); - assert_eq!(result, EndpointReadiness::Unreachable); - assert_eq!(probes, 1); - } - - #[test] - fn wait_past_readiness_budget_returns_at_once_when_already_serving() { - let result = wait_past_readiness_budget( - || EndpointReadiness::Serving, - || panic!("liveness is irrelevant once serving"), - ); - assert_eq!(result, EndpointReadiness::Serving); - } - - fn record_tracking_pid(paths: &AppPaths, pid: u32, status: &str) -> ManagedServiceRecord { - let mut record = reusable_record(paths); - record.supervisor_pid = pid; - record.status = status.to_owned(); - record - } - - #[cfg(unix)] - fn exited_child_pid() -> u32 { - let mut child = std::process::Command::new("true").spawn().expect("spawn"); - let pid = child.id(); - child.wait().expect("reap"); - pid - } - - #[test] - #[cfg(unix)] - fn record_still_live_ends_the_wait_for_a_dead_pid_with_a_live_status() { - // The engine was killed before writing any terminal state: the status - // still says `starting`, but nothing tracked is running. - let (root, paths) = test_paths("wait-dead-pid"); - paths.ensure().expect("paths"); - let dead = record_tracking_pid(&paths, exited_child_pid(), "starting"); - assert!(!record_still_live(&dead, service_pid_is_alive)); - - let alive = record_tracking_pid(&paths, std::process::id(), "starting"); - assert!(record_still_live(&alive, service_pid_is_alive)); - - // A settled status ends the wait even while the pid is still around. - let stopped = record_tracking_pid(&paths, std::process::id(), "stopped"); - assert!(!record_still_live(&stopped, service_pid_is_alive)); - let _ = fs::remove_dir_all(root); - } - - #[test] - #[cfg(target_os = "linux")] - fn service_pid_is_alive_treats_an_unreaped_child_as_dead() { - // The serve supervisor is our own unreaped child; once it dies it is a - // zombie that `kill(pid, 0)` still reports as running. - let mut child = std::process::Command::new("true").spawn().expect("spawn"); - let pid = child.id(); - let deadline = std::time::Instant::now() + Duration::from_secs(10); - while !process_is_zombie(pid) { - assert!(std::time::Instant::now() < deadline, "child never exited"); - thread::sleep(Duration::from_millis(10)); - } - assert!( - process_is_running(pid), - "a zombie still answers kill(pid, 0)" - ); - assert!(!service_pid_is_alive(pid)); - child.wait().expect("reap"); - } - - #[test] - fn service_still_live_follows_the_manifest_on_disk() { - // `rocm services stop` from another terminal writes only the manifest. - let (root, paths) = test_paths("wait-manifest"); - paths.ensure().expect("paths"); - let mut record = record_tracking_pid(&paths, std::process::id(), "starting"); - record.write().expect("write"); - assert!(service_still_live(&paths, &record.service_id, |_| true)); - - record.status = "stopped".to_owned(); - record.write().expect("write"); - assert!(!service_still_live(&paths, &record.service_id, |_| true)); - assert!(!service_still_live(&paths, "no-such-service", |_| true)); - let _ = fs::remove_dir_all(root); - } - - #[test] - fn settle_exited_service_demotes_a_live_manifest_and_spares_a_settled_one() { - let (root, paths) = test_paths("wait-settle"); - paths.ensure().expect("paths"); - let mut record = record_tracking_pid(&paths, 0, "starting"); - record.write().expect("write"); - settle_exited_service(&paths, &record.service_id); - let after = ManagedServiceRecord::read_manifest(&paths, &record.service_id).unwrap(); - assert_eq!(after.status, "stopped"); - - // A record that already failed keeps its reason: the guard, not the - // overwrite, is what spares it (a `stopped` -> `stopped` rewrite would - // be invisible). - record.status = "failed".to_owned(); - record.write().expect("write"); - settle_exited_service(&paths, &record.service_id); - let after = ManagedServiceRecord::read_manifest(&paths, &record.service_id).unwrap(); - assert_eq!(after.status, "failed"); - let _ = fs::remove_dir_all(root); - } - - #[test] - fn wait_on_existing_service_only_waits_for_one_that_is_not_serving_yet() { - // The probing path itself blocks for the readiness budget, so only the - // cases that must return at once are driven here; a manifest planted - // as `starting` would hang a wrong guard for 45 seconds and fail it. - let (root, paths) = test_paths("wait-existing"); - paths.ensure().expect("paths"); - let mut record = record_tracking_pid(&paths, 0, "starting"); - record.write().expect("write"); - let report = |status: &str| ManagedLaunchReport { - service_id: record.service_id.clone(), - endpoint_url: record.endpoint_url.clone(), - status: status.to_owned(), - already_running: true, - child_pid: None, - log_path: None, - manifest_path: None, - }; - let mut no_tick = |_: Duration| panic!("must not probe"); - - let untouched = wait_on_existing_service( - &paths, - report("starting"), - false, - &mut |_, _| panic!("no --wait, no banner"), - &mut no_tick, - ) - .expect("no --wait"); - assert_eq!(untouched.status, "starting"); - let ready = wait_on_existing_service( - &paths, - report("ready"), - true, - &mut |_, _| panic!("already ready, no banner"), - &mut no_tick, - ) - .expect("already ready"); - assert_eq!(ready.status, "ready"); - record.status = "starting".to_owned(); - assert_eq!( - ManagedServiceRecord::read_manifest(&paths, &record.service_id) - .unwrap() - .status, - "starting" - ); - let _ = fs::remove_dir_all(root); - } - - #[test] - fn died_during_wait_fails_with_a_logs_pointer_and_records_the_death() { - let (root, paths) = test_paths("wait-died"); - paths.ensure().expect("paths"); - let record = record_tracking_pid(&paths, std::process::id(), "starting"); - record.write().expect("write"); - let id = record.service_id; - let status = || { - ManagedServiceRecord::read_manifest(&paths, &id) - .unwrap() - .status - }; - - // Not waiting, already serving, or still alive: no failure, no change. - let pending = EndpointReadiness::Unreachable; - assert!(died_during_wait(&paths, &id, false, pending, |_| false).is_none()); - assert!( - died_during_wait(&paths, &id, true, EndpointReadiness::Serving, |_| false).is_none() - ); - assert!(died_during_wait(&paths, &id, true, pending, |_| true).is_none()); - assert_eq!(status(), "starting"); - - // Dead server: the error names the logs command and the record is settled. - let error = died_during_wait(&paths, &id, true, pending, |_| false).expect("died"); - let message = error.to_string(); - assert!( - message.contains("exited before it became ready"), - "{message}" - ); - assert!( - message.contains(&format!("rocm logs --service {id}")), - "{message}" - ); - assert_eq!(status(), "stopped"); - let _ = fs::remove_dir_all(root); - } - - #[test] - fn died_during_wait_adds_the_oom_guidance_for_a_vllm_launch_only() { - let (root, paths) = test_paths("wait-died-oom"); - paths.ensure().expect("paths"); - let mut record = record_tracking_pid(&paths, std::process::id(), "starting"); - record.engine = "vllm".to_owned(); - record.write().expect("write"); - fs::write( - &record.log_path, - "torch.OutOfMemoryError: HIP out of memory. Tried to allocate 7.21 GiB.\n", - ) - .expect("log"); - let id = record.service_id; - let pending = EndpointReadiness::Unreachable; - let message = died_during_wait(&paths, &id, true, pending, |_| false) - .expect("died") - .to_string(); - assert!(message.contains("--gpu-memory-utilization"), "{message}"); - - // Same log, another engine: that engine has no such flag to suggest. - let mut record = ManagedServiceRecord::read_manifest(&paths, &id).unwrap(); - record.engine = "lemonade".to_owned(); - "starting".clone_into(&mut record.status); - record.write().expect("write"); - let message = died_during_wait(&paths, &id, true, pending, |_| false) - .expect("died") - .to_string(); - assert!(!message.contains("--gpu-memory-utilization"), "{message}"); - let _ = fs::remove_dir_all(root); - } - - #[test] - fn record_wait_outcome_keeps_what_changed_on_disk_during_the_wait() { - let (root, paths) = test_paths("wait-outcome"); - paths.ensure().expect("paths"); - let mut held = record_tracking_pid(&paths, 111, "starting"); - held.write().expect("write"); - // While the wait ran, `rocmd` recorded a restart and a new supervisor. - let mut elsewhere = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); - elsewhere.restart_count = 3; - elsewhere.supervisor_pid = 222; - elsewhere.write().expect("write"); - - record_wait_outcome(&paths, &mut held, "ready", true).expect("outcome"); - let after = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); - assert_eq!(after.status, "ready"); - assert!(after.inference_verified_at_unix_ms.is_some()); - assert_eq!(after.restart_count, 3, "the restart count was reverted"); - assert_eq!(after.supervisor_pid, 222, "the supervisor pid was reverted"); - - record_wait_outcome(&paths, &mut held, "starting", false).expect("outcome"); - let after = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); - assert_eq!(after.status, "starting"); - let _ = fs::remove_dir_all(root); - } - - #[test] - fn wait_begin_shows_the_key_only_for_the_launch_it_belongs_to() { - let own = wait_begin_client_config("svc-new", "svc-new", "0.0.0.0", 8000, Some("k-secret")) - .expect("own launch shows its key"); - // No `{own}` in the failure messages: the text carries the dummy key, - // and formatting it into an assert is what CodeQL flags as logging it. - assert!(own.contains("api key: k-secret"), "key line missing"); - assert!( - own.contains("http://0.0.0.0:8000/v1/models"), - "models url missing" - ); - // Attached to an already-starting server: its key is not this one. - assert!( - wait_begin_client_config("svc-old", "svc-new", "0.0.0.0", 8000, Some("k-secret")) - .is_none() - ); - // Loopback binds have no key to show. - assert!(wait_begin_client_config("svc-new", "svc-new", "127.0.0.1", 8000, None).is_none()); - } - - #[test] - fn serve_bind_validation_requires_public_ack() { - validate_bind_host("127.0.0.1", false).unwrap(); - validate_bind_host("localhost", false).unwrap(); - validate_bind_host("::1", false).unwrap(); - let error = validate_bind_host("0.0.0.0", false).unwrap_err(); - assert!( - error.to_string().contains("--allow-public-bind"), - "{error:#}" - ); - validate_bind_host("0.0.0.0", true).unwrap(); - } - - /// A `stop_server` answer whose one recorded process ended with `outcome`. - fn stop_answer(confirmed: bool, outcome: &str) -> serde_json::Value { - crate::services_text::internal_stop_server_value(serde_json::json!({ - "service_id": "svc-attached", - "status": "ready", - "signaled_pids": if outcome == "unverified" { vec![] } else { vec![4242] }, - "pid_outcomes": [{ "pid": 4242, "role": "supervisor_and_engine", "outcome": outcome }], - // An attached serve on a public bind stores a key; an unconfirmed - // stop keeps it. - "endpoint_key_kept": !confirmed, - "stopped": confirmed, - })) - } - - /// Run [`finish_attached_stop`] for `stop`, returning its result, what it - /// wrote to stdout, the PID it signalled (if any) and the audit event. - fn finish( - name: &str, - stop: Result, - ) -> (Result<()>, String, Option, serde_json::Value) { - finish_signalling(name, stop, Ok(())) - } - - /// [`finish`], with the direct signal answering `signal_result`. - fn finish_signalling( - name: &str, - stop: Result, - signal_result: Result<()>, - ) -> (Result<()>, String, Option, serde_json::Value) { - let (result, out, _err, signalled, audit) = finish_capturing(name, stop, signal_result); - (result, out, signalled, audit) - } - - /// [`finish_signalling`], also returning what was written to stderr. - fn finish_capturing( - name: &str, - stop: Result, - signal_result: Result<()>, - ) -> (Result<()>, String, String, Option, serde_json::Value) { - let (root, paths) = test_paths(name); - paths.ensure().unwrap(); - let mut out = Vec::new(); - let mut err = Vec::new(); - let mut signalled = None; - let result = finish_attached_stop( - &paths, - "svc-attached", - 777, - stop, - |pid| { - signalled = Some(pid); - signal_result - }, - &mut out, - &mut err, - ); - let audit_text = fs::read_to_string(paths.audit_events_path()).expect("audit log"); - let audit = serde_json::from_str(audit_text.lines().last().expect("an audit event")) - .expect("audit json"); - let _ = fs::remove_dir_all(root); - ( - result, - String::from_utf8(out).unwrap(), - String::from_utf8(err).unwrap(), - signalled, - audit, - ) - } - - #[test] - fn attached_ctrl_c_reports_a_confirmed_stop_on_stdout() { - let (result, out, signalled, audit) = - finish("attached-confirmed", Ok(stop_answer(true, "graceful"))); - result.expect("a confirmed stop succeeds"); - assert!(out.contains("Local server stopped"), "{out}"); - assert_eq!(signalled, None); - assert_eq!(audit["level"], "info", "{audit}"); - assert_eq!( - audit["message"], - "stopped attached serve service_id=svc-attached" - ); - } - - #[test] - fn attached_ctrl_c_fails_an_unconfirmed_stop_like_services_stop_and_signals_nothing() { - // The same report `rocm services stop` gives, as the error (stderr, - // non-zero exit), naming no retry; nothing is signalled on the stop's - // behalf and nothing reaches stdout. - let answer = stop_answer(false, "timed_out"); - let (result, out, signalled, audit) = finish("attached-timed-out", Ok(answer.clone())); - let error = format!("{:#}", result.expect_err("an unconfirmed stop fails")); - assert_eq!( - error, - render_service_action_result("stop_server", &answer).trim_end(), - "the report is `rocm services stop`'s own" - ); - assert!( - error.contains( - "process 4242 (supervisor_and_engine) or a process under it did not exit" - ), - "{error}" - ); - assert!( - error.lines().any(|line| line - == " kept: the server record, which still names its processes, and its endpoint key, which the server may still be enforcing"), - "{error}" - ); - crate::services_text::tests::assert_no_retry_line(&error); - assert!(out.is_empty(), "nothing on stdout: {out}"); - assert_eq!(signalled, None, "an unconfirmed stop signals nothing more"); - assert_eq!(audit["level"], "error", "{audit}"); - assert_eq!( - audit["message"], - "stop of attached serve not confirmed service_id=svc-attached" - ); - } - - #[test] - fn attached_ctrl_c_names_a_check_for_an_unverified_process() { - let (result, out, signalled, audit) = - finish("attached-unverified", Ok(stop_answer(false, "unverified"))); - let error = format!("{:#}", result.expect_err("an unverified stop fails")); - assert!(error.contains("not signaled: process 4242"), "{error}"); - assert!(error.contains("check: `ps -p 4242 -o pid,args`"), "{error}"); - crate::services_text::tests::assert_no_retry_line(&error); - assert!(out.is_empty(), "{out}"); - assert_eq!(signalled, None); - assert_eq!(audit["level"], "error", "{audit}"); - } - - #[test] - fn attached_ctrl_c_signals_its_own_child_when_the_stop_itself_fails() { - let (result, out, err, signalled, audit) = finish_capturing( - "attached-stop-error", - Err(anyhow::anyhow!("managed service `svc-attached` not found") - .context("failed to stop svc-attached")), - Ok(()), - ); - result.expect("a failed stop falls back and still exits cleanly, as before"); - assert_eq!(signalled, Some(777)); - assert_eq!( - err, " note: failed to stop svc-attached: managed service `svc-attached` not found\n", - "the stop's error goes to stderr, and a signal that worked adds nothing" - ); - assert!(out.is_empty(), "nothing on stdout: {out}"); - assert_eq!(audit["level"], "warn", "{audit}"); - assert_eq!( - audit["message"], - "stop failed; signaled attached serve child 777 service_id=svc-attached" - ); - - // The direct signal failing too is printed on stderr and recorded, with - // its error. - let (result, _, err, signalled, audit) = finish_capturing( - "attached-stop-error-signal-fails", - Err(anyhow::anyhow!("managed service `svc-attached` not found")), - Err(anyhow::anyhow!("no such process")), - ); - result.expect("still exits 0, as on main"); - assert_eq!( - err, - " note: managed service `svc-attached` not found\n \ - note: signaling this session's server process 777 directly also failed: no such process\n", - "both notes go to stderr" - ); - assert_eq!(signalled, Some(777)); - assert_eq!(audit["level"], "error", "{audit}"); - assert_eq!( - audit["message"], - "stop failed; signaling attached serve child 777 also failed: no such process service_id=svc-attached" - ); - } - - /// A writer whose every write fails, standing in for a closed stdout/stderr. - struct BrokenWriter; - - impl Write for BrokenWriter { - fn write(&mut self, _: &[u8]) -> std::io::Result { - Err(std::io::Error::from(std::io::ErrorKind::BrokenPipe)) - } - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } - } - - #[test] - fn attached_ctrl_c_records_the_audit_event_even_when_writing_the_outcome_fails() { - // One case per branch that writes after deciding: a confirmed stop - // (stdout) and a stop that itself failed (stderr notes). The write - // error is returned, but the audit event is already on disk. - let cases: [(&str, Result, &str); 2] = [ - ( - "attached-broken-out", - Ok(stop_answer(true, "graceful")), - "stopped attached serve service_id=svc-attached", - ), - ( - "attached-broken-err", - Err(anyhow::anyhow!("managed service `svc-attached` not found")), - "stop failed; signaled attached serve child 777 service_id=svc-attached", - ), - ]; - for (name, stop, expected_message) in cases { - let (root, paths) = test_paths(name); - paths.ensure().unwrap(); - let result = finish_attached_stop( - &paths, - "svc-attached", - 777, - stop, - |_| Ok(()), - &mut BrokenWriter, - &mut BrokenWriter, - ); - let audit_text = fs::read_to_string(paths.audit_events_path()).unwrap_or_default(); - let _ = fs::remove_dir_all(root); - assert!(result.is_err(), "{name}: the write error is surfaced"); - let audit: serde_json::Value = serde_json::from_str( - audit_text - .lines() - .last() - .unwrap_or_else(|| panic!("{name}: no audit event recorded")), - ) - .expect("audit json"); - assert_eq!(audit["message"], expected_message, "{name}: {audit}"); - } - } - - #[test] - fn detach_key_ctrl_d_detaches_ctrl_c_stops() { - assert_eq!(detach_key_outcome(true, 'd'), Some(AttachOutcome::Detach)); - assert_eq!(detach_key_outcome(true, 'c'), Some(AttachOutcome::Stop)); - } - - #[test] - fn detach_key_ignores_plain_and_unrelated_keys() { - // Without the control modifier, `d`/`c` are ordinary log-scroll input. - assert_eq!(detach_key_outcome(false, 'd'), None); - assert_eq!(detach_key_outcome(false, 'c'), None); - // Other control combos are not detach/stop triggers. - assert_eq!(detach_key_outcome(true, 'q'), None); - assert_eq!(detach_key_outcome(true, 'z'), None); - } - - /// A record with every reuse-relevant field set to the "reusable" value, so - /// each test below breaks exactly one term and nothing else. - fn reusable_record(paths: &AppPaths) -> ManagedServiceRecord { - let mut record = ManagedServiceRecord::new( - paths, - "vllm-qwen-1000", - "vllm", - "qwen", - "qwen-canonical".to_owned(), - "127.0.0.1", - 8000, - "managed", - 0, - None, - None, - Some("gpu_required".to_owned()), - ); - record.engine_recipe_json = None; - record.requires_api_key = true; - record - } - - #[test] - fn reuse_is_refused_on_every_term_the_spawn_and_the_gpu_bail_depend_on() { - let (root, paths) = test_paths("managed-reuse-predicate"); - paths.ensure().expect("paths"); - let base = reusable_record(&paths); - - // The baseline must be reusable, or the negative cases below could pass - // for the wrong reason. - assert!( - managed_service_reuse_refusal(&base, None, true, &DevicePolicy::GpuRequired).is_none(), - "a matching service with no recipe, a key, and a gpu_required policy is reusable" - ); - - // Recipe mismatch — the spawn's first refusal. - let mut mismatched = base.clone(); - mismatched.engine_recipe_json = Some("{\"engine\":\"vllm\"}".to_owned()); - assert!(matches!( - managed_service_reuse_refusal(&mismatched, None, true, &DevicePolicy::GpuRequired), - Some(ManagedReuseRefusal::RecipeMismatch) - )); - - // Unmet `--require-api-key` — the spawn's second refusal. Also pins that - // it is *only* refused when the key is actually demanded. - let mut anonymous = base.clone(); - anonymous.requires_api_key = false; - assert!(matches!( - managed_service_reuse_refusal(&anonymous, None, true, &DevicePolicy::GpuRequired), - Some(ManagedReuseRefusal::MissingApiKey) - )); - assert!( - managed_service_reuse_refusal(&anonymous, None, false, &DevicePolicy::GpuRequired) - .is_none(), - "an anonymous service is fine when no key was demanded" - ); - - // Device policy — the term `serve()` depends on, because it is the one - // gating the no-usable-GPU bail on the answer. Reusing a `cpu_only` - // service to answer a `gpu_required` serve is the silent CPU fallback. - let mut cpu_only = base; - cpu_only.device_policy = Some("cpu_only".to_owned()); - assert!(matches!( - managed_service_reuse_refusal(&cpu_only, None, true, &DevicePolicy::GpuRequired), - Some(ManagedReuseRefusal::DevicePolicyTooWeak) - )); - // `GpuRequired` is the only policy the CLI can produce — `--device - // gpu_preferred`/`auto`/`gpu` all parse to it and `cpu_only` is rejected - // (`parse_device_policy_defaults_to_gpu_required_without_cpu_fallback`) — - // so there is no second policy arm to discriminate here, and asserting - // one would pin an input `serve()` never produces. Both callers pass - // the same policy; the parameter is not optional, so neither can drop - // this term without a compile error. - // - // A cpu_only *invocation* never reaches the bail, so nothing to refuse. - assert!( - managed_service_reuse_refusal(&cpu_only, None, true, &DevicePolicy::CpuOnly).is_none() - ); - - let _ = std::fs::remove_dir_all(&root); - } - - #[test] - fn spawn_managed_engine_child_refuses_an_explicit_port_a_live_equivalent_does_not_serve() - -> Result<()> { - let (root, paths) = test_paths("dup-managed-explicit-port-differs"); - paths.ensure()?; - let mut existing = ManagedServiceRecord::new( - &paths, - "lemonade-qwen-1000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11510, - "managed", - std::process::id(), - None, - None, - None, - ); - existing.status = "ready".to_owned(); - existing.engine_pid = Some(std::process::id()); - existing.write()?; - - let resolve = ResolveModelResponse { - canonical_model_id: "qwen-canonical".to_owned(), - served_model_id: None, - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), - }; - let launch = |port_request| { - spawn_managed_engine_child( - &paths, - "lemonade", - "lemonade-qwen-2000", - "qwen", - &resolve, - "127.0.0.1", - port_request, - &resolve.device_policy, - &[], - None, - None, - None, - false, - false, - ) - }; - - let different = launch(serve_port::PortRequest::Explicit(11511)); - // The same port, and no port at all, still reuse the live service. - let same = launch(serve_port::PortRequest::Explicit(11510)); - let auto = launch(serve_port::PortRequest::Auto); - let _ = fs::remove_dir_all(root); - - let Err(error) = different else { - panic!("a port the live service does not serve must be refused") - }; - let message = error.to_string(); - assert!( - message.contains("on port 11510") && message.contains("port 11511 would not be used"), - "{message}" - ); - assert!(matches!(same?, ManagedSpawn::AlreadyRunning(_))); - assert!(matches!(auto?, ManagedSpawn::AlreadyRunning(_))); - Ok(()) - } - - #[test] - fn spawn_managed_engine_child_blocks_reuse_with_mismatched_recipe() -> Result<()> { - // A live service recorded with one recipe (e.g. a tool-call parser flag) - // must reject a relaunch requesting a different recipe rather than - // silently reusing the old server, and the error must not claim the - // mismatch is specifically about generation defaults when it could stem - // from any recipe field. - let (root, paths) = test_paths("dup-managed-recipe-mismatch"); - paths.ensure()?; - let mut existing = ManagedServiceRecord::new( - &paths, - "lemonade-qwen-1000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11510, - "managed", - std::process::id(), - None, - None, - None, - ); - existing.status = "ready".to_owned(); - existing.engine_pid = Some(std::process::id()); - existing.engine_recipe_json = Some(serde_json::to_string(&EngineRecipeHint { - contract_version: rocm_engine_protocol::ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "lemonade".to_owned(), - required_flags: vec!["--tool-call-parser".to_owned(), "hermes".to_owned()], - ..EngineRecipeHint::default() - })?); - existing.write()?; - - let resolve = ResolveModelResponse { - canonical_model_id: "qwen-canonical".to_owned(), - served_model_id: None, - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), - }; - let requested_recipe = EngineRecipeHint { - contract_version: rocm_engine_protocol::ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), - engine: "lemonade".to_owned(), - required_flags: vec!["--temperature".to_owned(), "0.5".to_owned()], - ..EngineRecipeHint::default() - }; - - let result = spawn_managed_engine_child( - &paths, - "lemonade", - "lemonade-qwen-2000", - "qwen", - &resolve, - "127.0.0.1", - serve_port::PortRequest::Explicit(11511), - &resolve.device_policy, - &[], - None, - None, - Some(&requested_recipe), - false, - false, - ); - let _ = fs::remove_dir_all(root); - - let Err(error) = result else { - panic!("mismatched recipe on a live service must be rejected") - }; - let message = error.to_string(); - assert!( - message.contains("different serve options"), - "message should describe the mismatch generically: {message}" - ); - assert!( - message.contains("recipe hint, tool-call parser, or generation defaults"), - "message should not single out generation defaults as the sole cause: {message}" - ); - Ok(()) - } - - #[test] - fn spawn_managed_engine_child_refuses_to_reuse_an_unauthenticated_service() -> Result<()> { - // `--require-api-key` used to be accepted and dropped on this path. The - // reuse branch returns before the flag is recorded and before the - // endpoint key is checked, so a caller demanding auth got a server that - // never had it, with no error. `rocm remote serve` then published that - // endpoint onto the tailnet and printed a freshly minted key under "the - // API key above is what stops anyone else calling it" — a false - // assurance about an endpoint the whole tailnet can reach. - // - // Asserted through `spawn_managed_engine_child` rather than against the - // guard's own arguments: the defect was the early return, so only the - // real call site can fail for it. - let (root, paths) = test_paths("dup-managed-unauthenticated-reuse"); - paths.ensure()?; - let mut existing = ManagedServiceRecord::new( - &paths, - "lemonade-qwen-3000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11520, - "managed", - std::process::id(), - None, - None, - None, - ); - existing.status = "ready".to_owned(); - existing.engine_pid = Some(std::process::id()); - // The state that matters: live, matching, and serving without auth. - existing.requires_api_key = false; - existing.write()?; - - let resolve = ResolveModelResponse { - canonical_model_id: "qwen-canonical".to_owned(), - served_model_id: None, - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), - }; - - let result = spawn_managed_engine_child( - &paths, - "lemonade", - "lemonade-qwen-3001", - "qwen", - &resolve, - "127.0.0.1", - serve_port::PortRequest::Explicit(11520), - &resolve.device_policy, - &[], - None, - None, - None, - // The demand that used to be silently discarded. - true, - false, - ); - let _ = fs::remove_dir_all(root); - - let Err(error) = result else { - panic!( - "reusing an unauthenticated service must not satisfy `--require-api-key`; a \ - satisfied reuse leaves the endpoint open while the caller is told it is not" - ) - }; - let message = error.to_string(); - assert!( - message.contains("without authentication"), - "the refusal must say why it refused: {message}" - ); - // The printed hint must be the exact, directly-runnable command: - // `main.rs`'s `service_actions_require_yes_and_render_sandbox_result` - // proves this exact form (`rocm services stop --yes`) is the one - // the approval gate itself recommends. That running this form actually - // clears *this* refusal — a service reused without the key it is now - // being asked to require — is proven by - // `@id:serve-require-api-key-refuses-unauthenticated-reuse` - // (`tests/e2e-cucumber/features/model_serving.feature`, serve-24), - // which reproduces this exact condition, runs the printed hint, and - // asserts the record is left `stopped` — not by `stop_managed_services` - // in `tests/e2e-cucumber/tests/e2e.rs`, which runs this same - // invocation as best-effort teardown and never asserts success. - assert!( - message.contains("rocm services stop lemonade-qwen-3000 --yes"), - "the refusal must name the way out, with the service to stop, directly runnable with --yes: {message}" - ); - Ok(()) - } - - #[test] - fn a_managed_spawn_refuses_an_invalid_key_file_on_a_service_that_requires_one() -> Result<()> { - // Drives the real call site, not the guard's own arguments. The service - // was launched with `--require-api-key`, and its key file is present but - // invalid — so `requires_api_key` is true while `key_present` is false, - // which is the only way to reach the first branch. A test calling - // `ensure_public_service_has_endpoint_key` directly cannot catch a - // mis-wired call site, which is the defect that has occurred here twice. - // - // The flag is passed explicitly rather than inferred from the key file. - // Inferring it marked every public bind as having demanded auth, because - // a public bind always has a key file whether or not it asked for one. - let (root, paths) = test_paths("managed-spawn-invalid-key"); - paths.ensure()?; - fs::create_dir_all(paths.services_dir())?; - - // Non-empty but invalid (embedded CR/LF), so the file exists and is not - // merely empty (requires_api_key = true) but yields no usable key - // (key_present = false) — `endpoint_api_key_from_file` rejects embedded - // control characters. An empty seed here would let the post-condition - // below pass even if the real cleanup call never ran, since - // `endpoint_api_key` already reads back `None` for an empty file whether - // or not it was ever deleted. - endpoint_keys::store_endpoint_api_key( - &paths, - "lemonade-qwen-3000", - "bad-key\r\nX-Injected: value", - )?; - - let resolve = ResolveModelResponse { - canonical_model_id: "qwen-canonical".to_owned(), - served_model_id: None, - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), - }; - - let result = spawn_managed_engine_child( - &paths, - "lemonade", - "lemonade-qwen-3000", - "qwen", - &resolve, - // Loopback on purpose: the public-bind branch must not be what - // refuses this, or the test would pass with the guard disabled. - "127.0.0.1", - serve_port::PortRequest::Explicit(11512), - &resolve.device_policy, - &[], - None, - None, - None, - true, - false, - ); - - // This guard fires before any child is spawned, so `record.engine_pid` - // is still `None` when `spawn_managed_engine_child`'s cleanup arm calls - // `abandon_failed_managed_launch` — proving its pre-spawn branch - // (delete everything) runs from the real call site, not just from the - // `abandoned_launch_leaves_no_record_log_or_endpoint_key` test that - // calls the helper directly. - let manifest_path = paths.service_manifest_path("lemonade-qwen-3000"); - let log_path = paths.service_log_path("lemonade-qwen-3000"); - let engine_state_path = paths.service_engine_state_path("lemonade", "lemonade-qwen-3000"); - let key_file_path = endpoint_keys::endpoint_key_file_path(&paths, "lemonade-qwen-3000"); - let live = live_managed_services(&paths); - let _ = fs::remove_dir_all(root); - - let Err(error) = result else { - panic!("a service requiring a key must not spawn with an unusable key file") - }; - let message = error.to_string(); - assert!( - message.contains("--require-api-key"), - "the refusal must name the flag the service was launched with: {message}" - ); - assert!( - message.contains("without authentication"), - "the refusal must say what the risk is: {message}" - ); - assert!( - !manifest_path.exists(), - "a pre-spawn failure must not leave a record" - ); - assert!( - !log_path.exists(), - "a pre-spawn failure must not leave a log" - ); - assert!( - !engine_state_path.exists(), - "a pre-spawn failure must not leave engine state" - ); - assert!( - !key_file_path.exists(), - "a pre-spawn failure must delete the unusable key file from disk, not merely \ - leave it unreadable as a key" - ); - assert!( - live.is_empty(), - "a pre-spawn failure must leave no reservation" - ); - Ok(()) - } - - #[test] - fn a_failed_spawn_past_the_first_write_terminates_the_child_and_keeps_an_abandoned_record() - -> Result<()> { - // Drives the real call site's *post*-spawn cleanup branch, which - // neither `abandoned_launch_*` test reaches (they call - // `abandon_failed_managed_launch` directly). An engine name this - // binary does not build in makes the real spawned child (a second - // `rocm` process) exit almost immediately via - // `run_builtin_engine_serve_http`'s `other => bail!(...)` arm, after - // `launch_managed_engine_child_locked` has already set - // `record.engine_pid` — so cleanup must terminate (already-dead) the - // child, keep the record and log for inspection, mark the record - // `stopped` (a `failed` one would be respawned by rocmd recovery), and - // clear only the endpoint key. - // - // The child always dies near-instantly, but `spawn_managed_engine_child` - // only reports that within a fixed startup-check window. On a loaded or - // antivirus-scanning Windows CI box, a fresh debug `rocm.exe` can - // occasionally be slow enough to start that the window is missed even - // though the child is already doomed — so poll by retrying a fresh - // attempt (its own service id/port, so no stale manifest collides) up - // to a deadline instead of asserting on one race-prone attempt. - let deadline = std::time::Instant::now() + Duration::from_secs(10); - let mut attempt: u16 = 0; - let (root, paths, service_id, result) = loop { - attempt += 1; - let (root, paths) = - test_paths(&format!("managed-spawn-unknown-engine-cleanup-{attempt}")); - paths.ensure()?; - let service_id = format!("no-such-engine-qwen-{}", 4000 + attempt); - endpoint_keys::store_endpoint_api_key(&paths, &service_id, "test-key")?; - - let resolve = ResolveModelResponse { - canonical_model_id: "qwen-canonical".to_owned(), - served_model_id: None, - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), - }; - - let result = spawn_managed_engine_child( - &paths, - "no-such-engine", - &service_id, - "qwen", - &resolve, - "127.0.0.1", - serve_port::PortRequest::Explicit(11_514 + attempt), - &resolve.device_policy, - &[], - None, - None, - None, - false, - false, - ); - - if result.is_ok() && std::time::Instant::now() < deadline { - let _ = fs::remove_dir_all(&root); - continue; - } - break (root, paths, service_id, result); - }; - - let manifest_path = paths.service_manifest_path(&service_id); - let log_path = paths.service_log_path(&service_id); - let persisted = load_managed_services(&paths) - .expect("re-read manifests") - .into_iter() - .find(|candidate| candidate.service_id == service_id); - let key = endpoint_keys::endpoint_api_key(&paths, &service_id); - let live = live_managed_services(&paths); - - let Err(error) = result else { - panic!("an engine this binary does not build in must not report success") - }; - let message = error.to_string(); - assert!( - message.contains("managed engine exited immediately"), - "the failure must come from the spawned child dying, not an earlier guard: {message}" - ); - assert!( - manifest_path.exists(), - "a post-spawn failure must keep the record for inspection" - ); - assert!( - log_path.exists(), - "a post-spawn failure must keep the log for inspection" - ); - let persisted = - persisted.expect("a post-spawn failure must leave the record on disk, not delete it"); - // `stopped`, not `failed`: rocmd's recovery restarts a `failed` record - // without a stop marker, which would respawn a launch the CLI just - // reported as failed (pinned on the rocmd side by - // `abandoned_launch_record_is_not_a_recovery_candidate`). - assert_eq!( - persisted.status, "stopped", - "a post-spawn failure must mark the record abandoned, not leave it stuck" - ); - assert_eq!( - key, None, - "a post-spawn failure must still clear the endpoint key" - ); - assert!( - live.is_empty(), - "an abandoned record must not still reserve its port" - ); - let _ = fs::remove_dir_all(root); - Ok(()) - } - - #[test] - fn failed_port_reservation_leaves_no_orphaned_endpoint_key() -> Result<()> { - // serve() stores the new attempt's endpoint key before calling - // start_managed_service. If spawn_managed_engine_child fails inside - // resolve_managed_port_in_transaction (explicit port already reserved - // by another live service), it returns before any - // ManagedServiceRecord exists for this attempt, so - // abandon_failed_managed_launch never runs. Driving the real function - // end to end (not a simulated call to - // drop_orphaned_endpoint_key_on_already_running) proves - // `start_managed_service`'s own `.inspect_err` actually wires the cleanup, mirroring how - // attached_launch_failing_before_spawn_leaves_no_orphaned_endpoint_key - // drives run_attached_service for the foreground path. - let (root, paths) = test_paths("orphaned-key-on-reserved-port"); - paths.ensure()?; - let mut other = ManagedServiceRecord::new( - &paths, - "lemonade-llama-1000", - "lemonade", - "llama", - "llama-canonical", - "127.0.0.1", - 11513, - "managed", - std::process::id(), - None, - None, - None, - ); - other.status = "ready".to_owned(); - other.engine_pid = Some(std::process::id()); - other.write()?; - - let resolve = ResolveModelResponse { - canonical_model_id: "qwen-canonical".to_owned(), - served_model_id: None, - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), - }; - - let service_id = "lemonade-qwen-2000"; - endpoint_keys::store_endpoint_api_key(&paths, service_id, "test-key")?; - let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; - - let result = start_managed_service( - &paths, - "lemonade", - service_id, - "qwen", - &resolve, - "127.0.0.1", - serve_port::PortRequest::Explicit(11513), - &resolve.device_policy, - &[], - None, - None, - None, - Some("test-key"), - launch_lock, - false, - false, - false, - &mut |_, _| {}, - &mut |_elapsed| {}, - ); - let Err(error) = result else { - let _ = fs::remove_dir_all(root); - panic!("explicit port already reserved by a live service must be rejected") - }; - assert!( - error - .to_string() - .contains("already reserved by managed service"), - "unexpected error: {error}" - ); - - assert!( - endpoint_keys::endpoint_api_key(&paths, service_id).is_none(), - "cleanup must drop the key this failed attempt staged before launching" - ); - - let _ = fs::remove_dir_all(root); - Ok(()) - } - - #[test] - fn attached_launch_failing_before_spawn_leaves_no_orphaned_endpoint_key() -> Result<()> { - // The background path's cleanup is the `.inspect_err` in - // start_managed_service. run_attached_service is the other caller of - // spawn_managed_engine_child and needs the identical .inspect_err() - // cleanup, not a simulated one: drive the real function end to end. - let (root, paths) = test_paths("orphaned-key-attached-reserved-port"); - paths.ensure()?; - fs::create_dir_all(paths.services_dir())?; - let mut other = ManagedServiceRecord::new( - &paths, - "lemonade-llama-1000", - "lemonade", - "llama", - "llama-canonical", - "127.0.0.1", - 11514, - "managed", - std::process::id(), - None, - None, - None, - ); - other.status = "ready".to_owned(); - other.engine_pid = Some(std::process::id()); - other.write()?; - - let resolve = ResolveModelResponse { - canonical_model_id: "qwen-canonical".to_owned(), - served_model_id: None, - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), - }; - - let service_id = "lemonade-qwen-attached-2000"; - endpoint_keys::store_endpoint_api_key(&paths, service_id, "test-key")?; - let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; - - let result = run_attached_service( - &paths, - "lemonade", - service_id, - "qwen", - &resolve, - "127.0.0.1", - serve_port::PortRequest::Explicit(11514), - &[], - None, - None, - Some("test-key"), - launch_lock, - false, - false, - |_, _| unreachable!("the launch fails before a Ctrl-C could stop it"), - ); - let Err(error) = result else { - let _ = fs::remove_dir_all(root); - panic!("explicit port already reserved by a live service must be rejected") - }; - assert!( - error - .to_string() - .contains("already reserved by managed service"), - "unexpected error: {error}" - ); - assert!( - endpoint_keys::endpoint_api_key(&paths, service_id).is_none(), - "a pre-spawn failure must drop the key this attempt staged before launching" - ); - - let _ = fs::remove_dir_all(root); - Ok(()) - } - - /// The launch lock must still be held when the child is spawned: it is what - /// serializes allocation against publication. Observed through the spawn - /// hook, since releasing it early changes nothing else a test can see. - #[cfg(not(windows))] - #[test] - fn start_and_attached_launches_hold_the_launch_lock_until_the_spawn() -> Result<()> { - let resolve = ResolveModelResponse { - canonical_model_id: "qwen-canonical".to_owned(), - served_model_id: None, - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), - }; - for attached in [false, true] { - let (root, paths) = test_paths("launch-lock-held-until-spawn"); - paths.ensure()?; - let lock_path = paths.managed_launch_lock_path(); - let lock_was_held = std::rc::Rc::new(std::cell::Cell::new(false)); - let observed = lock_was_held.clone(); - TEST_SPAWN_HOOK.with(|hook| { - *hook.borrow_mut() = Some(std::rc::Rc::new(move |_command| { - observed.set( - rocm_core::FileLock::acquire_bounded(&lock_path, Duration::ZERO).is_err(), - ); - Err(io::Error::other("spawn intercepted by test")) - })); - }); - let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; - let result = if attached { - run_attached_service( - &paths, - "lemonade", - "lemonade-lock-attached", - "qwen", - &resolve, - "127.0.0.1", - serve_port::PortRequest::Auto, - &[], - None, - None, - None, - launch_lock, - false, - false, - |_, _| unreachable!("the spawn fails before a Ctrl-C could stop it"), - ) - } else { - start_managed_service( - &paths, - "lemonade", - "lemonade-lock-start", - "qwen", - &resolve, - "127.0.0.1", - serve_port::PortRequest::Auto, - &resolve.device_policy, - &[], - None, - None, - None, - None, - launch_lock, - false, - false, - false, - &mut |_, _| {}, - &mut |_elapsed| {}, - ) - .map(|_| ()) - }; - TEST_SPAWN_HOOK.with(|hook| *hook.borrow_mut() = None); - let _ = fs::remove_dir_all(root); - assert!( - format!("{:#}", result.expect_err("the intercepted spawn fails")) - .contains("spawn intercepted by test") - ); - assert!( - lock_was_held.get(), - "the launch lock must be held at spawn (attached: {attached})" - ); - } - Ok(()) - } - - #[test] - fn resolve_endpoint_auth_loopback_stays_credential_free() { - // Loopback binds never require auth, even if a key is supplied. - for host in ["127.0.0.1", "localhost", "::1"] { - assert_eq!(resolve_endpoint_auth(host, None, false).unwrap(), None); - assert_eq!( - resolve_endpoint_auth(host, Some("ignored"), false).unwrap(), - None - ); - } - } - - #[test] - fn resolve_endpoint_auth_loopback_can_be_required_when_something_republishes_it() { - // "Loopback" describes the bind address, not who can reach the port. A - // tailnet publish, a proxy, or a container port map all leave the bind - // loopback while widening the audience, and the default policy would - // hand out an unauthenticated endpoint. Whoever widens the reach asks - // for the credential explicitly. - for host in ["127.0.0.1", "localhost", "::1"] { - let generated = resolve_endpoint_auth(host, None, true) - .unwrap() - .expect("a required key must be generated, not skipped"); - assert!(!generated.trim().is_empty()); - - assert_eq!( - resolve_endpoint_auth(host, Some("supplied-key"), true).unwrap(), - Some("supplied-key".to_owned()), - "a supplied key must be honoured rather than ignored as it is by default" - ); - } - - // The same validation a public bind gets: an empty key is a refusal, not - // a silent downgrade to no auth. - assert!(resolve_endpoint_auth("127.0.0.1", Some(" "), true).is_err()); - } - - #[test] - fn resolve_endpoint_auth_public_uses_supplied_key_trimmed() { - let key = resolve_endpoint_auth("0.0.0.0", Some(" my-key "), false) - .unwrap() - .expect("public bind must have a key"); - assert_eq!(key, "my-key"); - } - - #[test] - fn resolve_endpoint_auth_public_generates_key_when_absent() { - let key = resolve_endpoint_auth("0.0.0.0", None, false) - .unwrap() - .expect("public bind must generate a key"); - assert_eq!(key.len(), 48); - assert!(key.chars().all(|c| c.is_ascii_alphanumeric())); - } - - #[test] - fn resolve_endpoint_auth_public_rejects_empty_supplied_key() { - let error = resolve_endpoint_auth("0.0.0.0", Some(" "), false).unwrap_err(); - assert!(error.to_string().contains("non-empty"), "{error:#}"); - } - - #[test] - fn resolve_endpoint_auth_public_rejects_embedded_crlf() { - // A supplied key survives `trim()` with embedded CR/LF intact and would - // otherwise be interpolated into a raw `Authorization: Bearer` header, - // injecting an extra header line. It must be rejected at input validation. - for supplied in [ - "good-key\r\nX-Injected: value", - "good-key\nmore", - "line\rreturn", - ] { - let error = resolve_endpoint_auth("0.0.0.0", Some(supplied), false).unwrap_err(); - assert!(error.to_string().contains("control character"), "{error:#}"); - } - } - - #[test] - fn drop_orphaned_endpoint_key_on_already_running_clears_stored_key() { - let (root, paths) = test_paths("drop-orphaned-key-stored"); - let service_id = "svc-orphaned"; - endpoint_keys::store_endpoint_api_key(&paths, service_id, "secret-key").unwrap(); - - drop_orphaned_endpoint_key_on_already_running(&paths, service_id, Some("secret-key")); - - assert_eq!(endpoint_keys::endpoint_api_key(&paths, service_id), None); - assert!(!endpoint_keys::endpoint_key_file_path(&paths, service_id).exists()); - let _ = fs::remove_dir_all(root); - } - - #[test] - fn drop_orphaned_endpoint_key_on_already_running_is_noop_for_loopback() { - // A loopback attempt never stores a key (`freshly_stored == None`), so the - // helper must not panic or error, and no file must appear. - let (root, paths) = test_paths("drop-orphaned-key-loopback"); - let service_id = "svc-loopback"; - - drop_orphaned_endpoint_key_on_already_running(&paths, service_id, None); - - assert_eq!(endpoint_keys::endpoint_api_key(&paths, service_id), None); - assert!(!endpoint_keys::endpoint_key_file_path(&paths, service_id).exists()); - let _ = fs::remove_dir_all(root); - } - - #[test] - fn public_bind_fails_closed_for_windows_lemonade_only() { - // Windows + Lemonade + public bind: refuse (cannot enforce the key). - let error = ensure_public_bind_engine_supported("lemonade", true, true).unwrap_err(); - assert!(error.to_string().contains("lemonade"), "{error:#}"); - // Every other combination is allowed: - ensure_public_bind_engine_supported("vllm", true, true).unwrap(); // vLLM enforces auth on Windows - ensure_public_bind_engine_supported("lemonade", true, false).unwrap(); // non-Windows - ensure_public_bind_engine_supported("lemonade", false, true).unwrap(); // loopback needs no key - } - - #[test] - fn a_public_bind_is_refused_with_the_command_that_restores_it() { - // `resolve_endpoint_auth` mints a key for every non-loopback bind whether - // or not auth was demanded, so deriving `requires_api_key` from key-file - // presence marked ordinary public binds as having asked for it. The guard - // tests that field first, so those services were refused with a message - // naming a flag they never passed and a relaunch command that drops - // `--allow-public-bind` — coming back on loopback instead. - // - // A plain `--host 0.0.0.0 --allow-public-bind` launch: key file present, - // `--require-api-key` never passed. - let error = ensure_public_service_has_endpoint_key("0.0.0.0", false, false) - .expect_err("a public bind with no key must be refused"); - let rendered = format!("{error:#}"); - assert!( - rendered.contains("--allow-public-bind"), - "the refusal must name the command that restores the public bind: {rendered}" - ); - assert!( - !rendered.contains("--require-api-key"), - "a service that never passed the flag must not be told it did: {rendered}" - ); - - // And the loopback-with-forced-auth case still reports its own reason. - let error = ensure_public_service_has_endpoint_key("127.0.0.1", false, true) - .expect_err("a service that demanded auth must not come back without it"); - assert!( - format!("{error:#}").contains("--require-api-key"), - "{error:#}" - ); - } - - #[test] - fn respawn_fails_closed_for_a_public_service_whose_key_is_gone() { - // A stop deletes the key file, so a later restart of a public service - // would otherwise respawn it with no auth at all. - let error = ensure_public_service_has_endpoint_key("0.0.0.0", false, false).unwrap_err(); - let message = error.to_string(); - assert!(message.contains("0.0.0.0"), "{error:#}"); - assert!(message.contains("without authentication"), "{error:#}"); - // Actionable: name the command that mints a fresh key. - assert!(message.contains("--allow-public-bind"), "{error:#}"); - - // A public service that still has its key restarts normally. - ensure_public_service_has_endpoint_key("0.0.0.0", true, false).unwrap(); - } - - #[test] - fn respawn_allows_loopback_services_without_an_endpoint_key() { - // Loopback stays credential-free, so every accepted spelling must pass - // the guard with no key present — when nothing asked for auth. - for host in ["127.0.0.1", "localhost", "::1"] { - ensure_public_service_has_endpoint_key(host, false, false) - .unwrap_or_else(|error| panic!("{host} must not require a key: {error:#}")); - } - } - - #[test] - fn respawn_refuses_a_loopback_service_that_was_launched_with_a_key() { - // The hole this closes: a loopback bind that something else republishes - // — a tailnet publish, a proxy, a container port map. The publish - // outlives the process, so a restart after the key was dropped would - // reopen a reachable endpoint with no authentication, and the bind - // address gives the guard no way to notice. - for host in ["127.0.0.1", "localhost", "::1"] { - let error = ensure_public_service_has_endpoint_key(host, false, true) - .expect_err("a service launched with a key must not restart without one"); - let message = format!("{error:#}"); - assert!(message.contains("without authentication"), "{message}"); - assert!(message.contains("--require-api-key"), "{message}"); - } - - // With its key still present it restarts normally. - ensure_public_service_has_endpoint_key("127.0.0.1", true, true).unwrap(); - } - - #[test] - fn endpoint_client_config_shows_key_once_with_bearer_guidance() { - let rendered = render_endpoint_client_config("http://0.0.0.0:11435/v1", "secret-123"); - // Dummy literal exercising the intentional one-time key display documented - // on `render_endpoint_client_config`; not a real credential or a log write. - // codeql[rust/cleartext-logging] - assert!(rendered.contains("secret-123"), "{rendered}"); - // codeql[rust/cleartext-logging]: see rationale above. - assert!(rendered.contains("Authorization: Bearer"), "{rendered}"); - // codeql[rust/cleartext-logging]: see rationale above. - assert!(rendered.contains("shown only now"), "{rendered}"); - } - - /// ROCMAI-827: `rocm serve` started the background daemon on every launch, - /// and with no automation watcher enabled that daemon had nothing to do yet - /// kept running after every service was stopped. With the default config - /// (no watcher enabled) the helper must not be spawned at all. - #[test] - fn background_helper_not_spawned_when_no_automation_is_enabled() { - let (root, paths) = test_paths("helper-no-automation"); - paths.ensure().expect("create app dirs"); - // The stored flag alone is not "a watcher is enabled": nothing would run. - let mut config = rocm_core::RocmCliConfig::default(); - config.automations.daemon_enabled = true; - config.save(&paths).expect("save default config"); - - let mut spawned = false; - ensure_background_helper_running_with( - &paths, - true, - |_| { - spawned = true; - Ok(Ok(4242)) - }, - &|| {}, - ) - .expect("ensure helper"); - - let claim_written = paths.automation_autostart_claim_path().exists(); - let _ = fs::remove_dir_all(&root); - assert!( - !spawned, - "no watcher enabled ⇒ the daemon has no work and must not be started" - ); - assert!(!claim_written, "no spawn ⇒ no autostart claim"); - } - - /// The other half of the gate: once a watcher is enabled the helper is still - /// started (exactly once) and its autostart claim recorded. - #[test] - fn background_helper_spawned_when_an_automation_is_enabled() { - let (root, paths) = test_paths("helper-automation-enabled"); - paths.ensure().expect("create app dirs"); - let mut config = rocm_core::RocmCliConfig::default(); - config.watcher_config_mut("server-recover").enabled = true; - config.save(&paths).expect("save config"); - - let mut spawns = 0; - ensure_background_helper_running_with( - &paths, - true, - |_| { - spawns += 1; - Ok(Ok(4242)) - }, - &|| {}, - ) - .expect("ensure helper"); - - let claim = read_autostart_claim(&paths.automation_autostart_claim_path()); - let _ = fs::remove_dir_all(&root); - assert_eq!(spawns, 1, "an enabled watcher needs the daemon"); - assert_eq!(claim.map(|claim| claim.daemon_pid), Some(4242)); - } - - /// ROCMAI-827 race, enable side: the daemon that is about to exit still has - /// `running = true` and a live pid in its state when `automations enable` - /// (config already saved) asks for a helper. It decides under the autostart - /// lock and publishes "stopped" before releasing it, so the enable must wait - /// on that lock and then start a new helper. A lock-free "already running" - /// read would return here without spawning, leaving a watcher with no daemon. - #[test] - fn background_helper_is_started_when_the_exiting_daemon_publishes_stopped() { - let (root, paths) = test_paths("helper-exit-window"); - paths.ensure().expect("create app dirs"); - let mut config = rocm_core::RocmCliConfig::default(); - config.watcher_config_mut("server-recover").enabled = true; - config.save(&paths).expect("save config"); - // The old daemon: alive (this process stands in) and still `running`. - runtime_state(true, std::process::id()) - .write(&paths) - .expect("write runtime state"); - - // The exiting daemon holds the autostart lock while it decides. - let exiting_daemon_lock = - rocm_core::FileLock::acquire(paths.automation_autostart_lock_path()) - .expect("hold the lock"); - let enable_paths = paths.clone(); - let queued_flag = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); - let queued = queued_flag.clone(); - let enable = std::thread::spawn(move || { - let mut spawns = 0; - ensure_background_helper_running_with( - &enable_paths, - true, - |_| { - spawns += 1; - Ok(Ok(4242)) - }, - &|| queued.store(true, std::sync::atomic::Ordering::SeqCst), - ) - .expect("ensure helper"); - spawns - }); - // Wait until the enable path has actually found the lock held and queued - // behind it, rather than assuming how long that takes. - let deadline = std::time::Instant::now() + std::time::Duration::from_mins(1); - // A caller that skipped the lock finishes without ever queuing. - while !queued_flag.load(std::sync::atomic::Ordering::SeqCst) && !enable.is_finished() { - assert!(std::time::Instant::now() < deadline, "enable never queued"); - std::thread::yield_now(); - } - runtime_state(false, std::process::id()) - .write(&paths) - .expect("daemon publishes stopped"); - drop(exiting_daemon_lock); - - let spawns = enable.join().expect("enable thread"); - let _ = fs::remove_dir_all(&root); - assert_eq!( - spawns, 1, - "the enable must start a helper once the old one has published it stopped" - ); - } - - fn stop_result() -> serde_json::Value { - serde_json::json!({"output": {"result": { - "service": {"service_id": "svc-a", "status": "stopped"}, - "signaled_pids": [1] - }}}) - } - - /// `rocm services stop` prints the note for a stop only, and only the variant - /// that matches the state. - #[test] - fn stop_output_carries_the_helper_note_that_matches_the_state() { - use crate::services_text::render_service_action_result_with_helper_note as render; - - let (root, paths) = test_paths("stop-helper-note"); - paths.ensure().expect("create app dirs"); - set_helper_facts(&paths, false, false); - let quiet = render(&paths, "stop_server", &stop_result()); - set_helper_facts(&paths, true, true); - let outlives = render(&paths, "stop_server", &stop_result()); - let restart = render(&paths, "restart_server", &stop_result()); - set_helper_facts(&paths, true, false); - let not_running = render(&paths, "stop_server", &stop_result()); - let _ = fs::remove_dir_all(&root); - - assert!(!quiet.contains("background automation helper"), "{quiet}"); - assert!( - outlives.contains("stopped processes: 1") - && outlives.contains("keeps running while an automation is enabled") - && outlives.contains("`rocm automations disable server-recover`"), - "{outlives}" - ); - assert!( - not_running.contains("is not running") && !not_running.contains("keeps running"), - "{not_running}" - ); - assert!( - !restart.contains("background automation helper"), - "{restart}" - ); - } - - /// `serve` calls the helper autostart after its child is already running. An - /// unreadable config must not turn that launch into an error: it reads as "no - /// watcher enabled" and nothing is spawned. - #[test] - fn background_helper_autostart_treats_an_unreadable_config_as_no_watcher() { - let (root, paths) = test_paths("helper-bad-config"); - paths.ensure().expect("create app dirs"); - fs::write(paths.config_path(), b"{ this is not json").expect("write bad config"); - assert!(rocm_core::RocmCliConfig::load(&paths).is_err()); - - let mut spawned = false; - let result = ensure_background_helper_running_with( - &paths, - true, - |_| { - spawned = true; - Ok(Ok(4242)) - }, - &|| {}, - ); - let _ = fs::remove_dir_all(&root); - - assert!(result.is_ok(), "{result:?}"); - assert!(!spawned); - } - - #[test] - fn autostart_spawn_in_flight_defers_only_for_a_live_recent_claim() { - let claim = AutostartClaim { - daemon_pid: 4242, - spawned_at_ms: 1_000, - }; - // A recent claim whose PID is still alive ⇒ a spawn is in flight, defer. - assert!(autostart_spawn_in_flight( - Some(claim), - 1_000 + 5_000, - AUTOSTART_CLAIM_TTL_MS, - |_| true, - )); - // Same claim but the child PID is gone (crashed spawn) ⇒ respawn. - assert!(!autostart_spawn_in_flight( - Some(claim), - 1_000 + 5_000, - AUTOSTART_CLAIM_TTL_MS, - |_| false, - )); - // Older than the TTL, even with a live PID (possible PID reuse) ⇒ respawn. - assert!(!autostart_spawn_in_flight( - Some(claim), - 1_000 + AUTOSTART_CLAIM_TTL_MS, - AUTOSTART_CLAIM_TTL_MS, - |_| true, - )); - // No claim at all ⇒ nothing in flight, spawn. - assert!(!autostart_spawn_in_flight( - None, - 1_000, - AUTOSTART_CLAIM_TTL_MS, - |_| true, - )); - } - - #[test] - fn autostart_claim_round_trips_and_absent_or_garbage_reads_as_none() { - let (root, paths) = test_paths("autostart-claim-roundtrip"); - let path = paths.automation_autostart_claim_path(); - fs::create_dir_all(path.parent().expect("claim parent")).expect("mkdir claim dir"); - // Absent file ⇒ no claim. - assert_eq!(read_autostart_claim(&path), None); - // Round-trip a written claim. - let claim = AutostartClaim { - daemon_pid: 9987, - spawned_at_ms: 1_726_000_000_123, - }; - write_autostart_claim(&path, claim).expect("write claim"); - assert_eq!(read_autostart_claim(&path), Some(claim)); - // Garbage / partial content ⇒ no claim (permits a respawn, never panics). - fs::write(&path, "not-a-pid").expect("write garbage"); - assert_eq!(read_autostart_claim(&path), None); - let _ = fs::remove_dir_all(&root); - } - - #[test] - fn duplicate_managed_launch_detected_across_distinct_service_ids() -> Result<()> { - // `generate_service_id` embeds a timestamp, so a second launch for the - // same engine+model has a DIFFERENT service_id. The guard must still - // detect the live service by (engine, canonical_model_id), and return - // the newest live match. `starting` skips the endpoint probe; the - // current process id is a guaranteed-live PID. - let (root, paths) = test_paths("dup-managed-distinct-ids"); - paths.ensure()?; - - // Older, dead manifest for the same engine+model (distinct service_id). - let mut dead = ManagedServiceRecord::new( - &paths, - "lemonade-qwen-1000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11500, - "managed", - 999_999_999, - None, - None, - None, - ); - dead.status = "ready".to_owned(); - dead.engine_pid = Some(999_999_999); - dead.created_at_unix_ms = 1000; - dead.write()?; - - // Newer, live manifest for the same engine+model (distinct service_id). - let mut live = ManagedServiceRecord::new( - &paths, - "lemonade-qwen-2000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11501, - "managed", - std::process::id(), - None, - None, - None, - ); - live.status = "starting".to_owned(); - live.engine_pid = Some(std::process::id()); - live.created_at_unix_ms = 2000; - live.write()?; - - let live_services = live_managed_services(&paths); - let found = - existing_live_managed_service_in(&live_services, "lemonade", "qwen-canonical").cloned(); - let _ = fs::remove_dir_all(root); - - let found = found.expect("a live managed service should be detected by engine+model"); - assert_eq!( - found.service_id, "lemonade-qwen-2000", - "should return the newest live match" - ); - assert!(managed_service_is_live(&found)); - Ok(()) - } - - #[test] - fn dead_managed_service_allows_relaunch() -> Result<()> { - // A stale manifest with dead PIDs must NOT block a relaunch: liveness - // refresh demotes it to "stopped", so the guard returns None. - let (root, paths) = test_paths("dup-managed-dead"); - paths.ensure()?; - let mut record = ManagedServiceRecord::new( - &paths, - "lemonade-qwen-3000", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - 11502, - "managed", - 999_999_999, - None, - None, - None, - ); - record.status = "ready".to_owned(); - record.engine_pid = Some(999_999_999); - record.write()?; - - let live = live_managed_services(&paths); - let found = existing_live_managed_service_in(&live, "lemonade", "qwen-canonical"); - let _ = fs::remove_dir_all(root); - - assert!( - found.is_none(), - "a dead managed service must not block relaunch" - ); - Ok(()) - } - - #[test] - fn live_service_for_other_model_does_not_block() -> Result<()> { - // A live service for a DIFFERENT model must not match — the guard keys - // on the model, not just the engine. - let (root, paths) = test_paths("dup-managed-other-model"); - paths.ensure()?; - let mut record = ManagedServiceRecord::new( - &paths, - "lemonade-other-1", - "lemonade", - "other", - "other-canonical", - "127.0.0.1", - 11503, - "managed", - std::process::id(), - None, - None, - None, - ); - record.status = "starting".to_owned(); - record.engine_pid = Some(std::process::id()); - record.write()?; - - let live = live_managed_services(&paths); - let found = existing_live_managed_service_in(&live, "lemonade", "qwen-canonical"); - let _ = fs::remove_dir_all(root); - - assert!( - found.is_none(), - "a live service for a different model must not match" - ); - Ok(()) - } - - #[test] - fn missing_manifest_allows_launch() { - // No services dir / manifests → nothing to detect, launch proceeds. - let (root, paths) = test_paths("dup-managed-missing"); - let live = live_managed_services(&paths); - let found = existing_live_managed_service_in(&live, "lemonade", "qwen-canonical"); - let _ = fs::remove_dir_all(root); - assert!(found.is_none()); - } - - fn write_live_service_record( - paths: &AppPaths, - service_id: &str, - canonical_model_id: &str, - port: u16, - status: &str, - ) -> Result { - let mut record = ManagedServiceRecord::new( - paths, - service_id, - "lemonade", - canonical_model_id, - canonical_model_id.to_owned(), - rocm_core::DEFAULT_LOCAL_HOST, - port, - "managed", - std::process::id(), - None, - None, - None, - ); - record.status = status.to_owned(); - record.engine_pid = Some(std::process::id()); - record.write()?; - Ok(record) - } - - fn never_probe(_port: u16) -> std::io::Result { - panic!("no port may be probed on this path") - } - - fn always_free(port: u16) -> std::io::Result { - Ok(rocm_core::LoopbackPortLease::engine_owned(port)) - } - - #[test] - fn allocation_transaction_answers_an_equivalent_live_service_before_leasing() -> Result<()> { - let (root, paths) = test_paths("alloc-txn-already-live"); - paths.ensure()?; - let existing = write_live_service_record( - &paths, - "lemonade-qwen-9000", - "qwen-canonical", - 11501, - "starting", - )?; - - let decision = resolve_managed_port_in_transaction( - &paths, - "lemonade", - "qwen-canonical", - "127.0.0.1", - serve_port::PortRequest::Auto, - &mut never_probe, - )?; - let _ = fs::remove_dir_all(root); - - match decision { - ManagedPortDecision::AlreadyLive(record) => { - assert_eq!(record.service_id, existing.service_id); - assert_eq!(record.port, existing.port); - } - ManagedPortDecision::Lease(_) => { - panic!("an equivalent live service must be answered, not leased a new port") - } - } - Ok(()) - } - - #[test] - fn allocation_transaction_skips_ports_reserved_by_other_live_services() -> Result<()> { - let (root, paths) = test_paths("alloc-txn-skips-reserved"); - paths.ensure()?; - write_live_service_record(&paths, "lemonade-a", "model-a", 11_435, "running")?; - write_live_service_record(&paths, "lemonade-b", "model-b", 11_436, "recovering")?; - - let decision = resolve_managed_port_in_transaction( - &paths, - "lemonade", - "model-c", - "127.0.0.1", - serve_port::PortRequest::Auto, - &mut always_free, - )?; - let _ = fs::remove_dir_all(root); - - match decision { - ManagedPortDecision::Lease(lease) => assert_eq!(lease.port(), 11_437), - ManagedPortDecision::AlreadyLive(record) => { - panic!("model-c has no live service, expected a lease, got {record:?}") - } - } - Ok(()) - } - - #[test] - fn allocation_transaction_ignores_stopped_records() -> Result<()> { - let (root, paths) = test_paths("alloc-txn-ignores-stopped"); - paths.ensure()?; - let mut stale = - write_live_service_record(&paths, "lemonade-stale", "model-stale", 11_435, "running")?; - // A dead pid demotes the record to stopped when liveness is refreshed, so - // it must not reserve its port. - stale.engine_pid = Some(999_999_999); - stale.supervisor_pid = 999_999_999; - stale.write()?; - - let decision = resolve_managed_port_in_transaction( - &paths, - "lemonade", - "model-fresh", - "127.0.0.1", - serve_port::PortRequest::Auto, - &mut always_free, - )?; - let _ = fs::remove_dir_all(root); - - match decision { - ManagedPortDecision::Lease(lease) => assert_eq!(lease.port(), 11_435), - ManagedPortDecision::AlreadyLive(record) => { - panic!("a stopped record must not block a fresh launch, got {record:?}") - } - } - Ok(()) - } - - #[test] - fn stopping_the_reserving_record_frees_its_explicit_port_for_reuse() -> Result<()> { - // Proves the mechanism `rocm services stop` relies on: it never touches - // the port directly, it only flips the stopped record's `status` (see - // the `all_stopped` branch in `main.rs`'s stop handler). This recreates - // that exact disk state and checks the same explicit port a live - // competitor refused now resolves, closing the loop the E2E - // `assert_port_reserved_message` step only checks by wording. - let (root, paths) = test_paths("alloc-explicit-port-freed-by-stop"); - paths.ensure()?; - let mut holder = ManagedServiceRecord::new( - &paths, - "lemonade-a-1", - "lemonade", - "model-a", - "model-a".to_owned(), - DEFAULT_LOCAL_HOST, - 11_435, - "managed", - std::process::id(), - None, - None, - None, - ); - holder.status = "running".to_owned(); - holder.engine_pid = Some(std::process::id()); - holder.write()?; - - let still_reserved = resolve_managed_port_in_transaction( - &paths, - "lemonade", - "model-b", - DEFAULT_LOCAL_HOST, - serve_port::PortRequest::Explicit(11_435), - &mut always_free, - ); - // The refusal names the service AND the command that frees the port; - // the stop below is that command's own implementation, and the grant - // after it is the outcome, so advice and result are pinned together. - let stop_hint = service_hints::service_stop_hint("lemonade-a-1"); - assert!( - matches!(still_reserved, Err(ref error) if error.to_string().contains("already reserved by managed service lemonade-a-1") - && error.to_string().contains(&stop_hint)), - "unexpected result while still running: {still_reserved:?}" - ); - - // Go through the real stop path (`rocm services stop --yes`'s own - // implementation) rather than writing `status = "stopped"` to disk - // directly, so this test actually proves the advice serve-45 pins: that - // running the stop command frees the port. The termination closure is - // the same seam production injects a real signal through; here it just - // confirms every recorded process gone without touching the test - // process itself. - crate::service_control::stop_internal_managed_service_with(&paths, "lemonade-a-1", |_| { - crate::service_control::RecordedStopTermination::confirmed_stopped() - })?; - - let decision = resolve_managed_port_in_transaction( - &paths, - "lemonade", - "model-b", - DEFAULT_LOCAL_HOST, - serve_port::PortRequest::Explicit(11_435), - &mut always_free, - ); - let _ = fs::remove_dir_all(root); - - match decision? { - ManagedPortDecision::Lease(lease) => assert_eq!(lease.port(), 11_435), - ManagedPortDecision::AlreadyLive(existing) => { - panic!( - "a stopped record must not block reuse: {}", - existing.service_id - ) - } - } - Ok(()) - } - - #[test] - fn allocation_transaction_rejects_an_explicit_port_a_recovering_service_holds() -> Result<()> { - let (root, paths) = test_paths("alloc-txn-rejects-explicit-collision"); - paths.ensure()?; - let existing = write_live_service_record( - &paths, - "lemonade-recovering", - "model-recovering", - 11_435, - "recovering", - )?; - - let result = resolve_managed_port_in_transaction( - &paths, - "lemonade", - "model-other", - "127.0.0.1", - serve_port::PortRequest::Explicit(11_435), - &mut never_probe, - ); - let _ = fs::remove_dir_all(root); - - let Err(error) = result else { - panic!("an explicit port a recovering service holds must be refused") - }; - let message = error.to_string(); - assert!( - message.contains(&existing.service_id), - "the refusal must name the service holding the port: {message}" - ); - assert!( - message.contains("recovering"), - "the refusal must say why the port is unavailable: {message}" - ); - Ok(()) - } - - #[test] - fn concurrent_auto_transactions_publish_distinct_ports() -> Result<()> { - let (root, paths) = test_paths("alloc-txn-concurrent-distinct"); - paths.ensure()?; - let paths = std::sync::Arc::new(paths); - - let spawn_one = |model: &'static str| { - let paths = std::sync::Arc::clone(&paths); - thread::spawn(move || -> Result { - let _launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; - let lease = match resolve_managed_port_in_transaction( - &paths, - "lemonade", - model, - "127.0.0.1", - serve_port::PortRequest::Auto, - &mut serve_port::loopback_bind_probe, - )? { - ManagedPortDecision::Lease(lease) => lease, - ManagedPortDecision::AlreadyLive(record) => { - panic!("fresh model must not already be live: {record:?}") - } - }; - let port = lease.port(); - write_live_service_record(&paths, model, model, port, "running")?; - lease.release(); - Ok(port) - }) - }; - - let first = spawn_one("model-one"); - let second = spawn_one("model-two"); - let first_port = first.join().expect("thread one panicked")?; - let second_port = second.join().expect("thread two panicked")?; - let _ = fs::remove_dir_all(root); - - assert_ne!(first_port, second_port); - for port in [first_port, second_port] { - assert!( - (serve_port::AUTO_PORT_FIRST..=serve_port::AUTO_PORT_LAST).contains(&port), - "leased port {port} must fall within the auto-select range" - ); - } - Ok(()) - } - - #[test] - fn concurrent_equivalent_transactions_converge_on_one_record() -> Result<()> { - let (root, paths) = test_paths("alloc-txn-concurrent-converge"); - paths.ensure()?; - let paths = std::sync::Arc::new(paths); - - let race_once = || { - let paths = std::sync::Arc::clone(&paths); - thread::spawn(move || -> Result { - let _launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; - match resolve_managed_port_in_transaction( - &paths, - "lemonade", - "shared-model", - "127.0.0.1", - serve_port::PortRequest::Auto, - &mut serve_port::loopback_bind_probe, - )? { - ManagedPortDecision::Lease(lease) => { - let port = lease.port(); - write_live_service_record( - &paths, - "lemonade-shared", - "shared-model", - port, - "running", - )?; - lease.release(); - Ok(true) - } - ManagedPortDecision::AlreadyLive(_) => Ok(false), - } - }) - }; - - let first = race_once(); - let second = race_once(); - let first_launched = first.join().expect("thread one panicked")?; - let second_launched = second.join().expect("thread two panicked")?; - - let live_count = live_managed_services(&paths).len(); - let _ = fs::remove_dir_all(root); - - assert_ne!( - first_launched, second_launched, - "exactly one racing transaction must launch" - ); - assert_eq!(live_count, 1, "only one record may end up live"); - Ok(()) - } - - #[test] - fn select_gpu_indices_under_launch_lock_blocks_until_the_holder_releases() -> Result<()> { - // Exercises the real lock-acquisition function production callers use - // (`select_gpu_indices_under_launch_lock`) against a lock another thread - // actually holds, so a refactor that dropped the lock early would fail - // here. `spawn_managed_engine_child` is then run for real, but only via - // its `AlreadyLive` short-circuit (a pre-seeded duplicate record), so no - // child process is ever launched. The holder writes nothing: this does - // not show a write made under the lock being observed. - use std::sync::{Arc, Barrier}; - use std::time::Instant; - - let (root, paths) = test_paths("launch-lock-holds-across-spawn"); - paths.ensure()?; - write_live_service_record(&paths, "lemonade-held-1", "held-canonical", 11_520, "ready")?; - - let barrier = Arc::new(Barrier::new(2)); - let hold_for = Duration::from_millis(250); - - let holder_paths = paths.clone(); - let holder_barrier = Arc::clone(&barrier); - let holder = thread::spawn(move || -> Result<()> { - let _lock = rocm_core::FileLock::acquire(holder_paths.managed_launch_lock_path())?; - holder_barrier.wait(); - thread::sleep(hold_for); - Ok(()) - }); - - barrier.wait(); - let started = Instant::now(); - let (gpu_indices, launch_lock) = crate::gpu_policy::select_gpu_indices_under_launch_lock( - &paths, - true, - None, - || None, - None, - None, - )?; - let waited = started.elapsed(); - - let resolve = ResolveModelResponse { - canonical_model_id: "held-canonical".to_owned(), - served_model_id: None, - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), - }; - let spawn = spawn_managed_engine_child( - &paths, - "lemonade", - "lemonade-held-2", - "held-model", - &resolve, - DEFAULT_LOCAL_HOST, - serve_port::PortRequest::Auto, - &resolve.device_policy, - &gpu_indices, - None, - None, - None, - false, - false, - ); - drop(launch_lock); - holder.join().expect("holder thread")?; - let _ = fs::remove_dir_all(root); - - // `thread::sleep` only guarantees *at least* `hold_for`, but under a - // loaded host (e.g. a full parallel `cargo test` run on WSL2) timer - // coalescing can make the measured wait a hair short of the nominal - // hold. Allow a small margin so the test still fails loudly if the - // call returns early, without being flaky on actual block duration. - let margin = Duration::from_millis(20); - assert!( - waited + margin >= hold_for, - "select_gpu_indices_under_launch_lock returned before the holder released the lock: waited {waited:?}, held {hold_for:?}" - ); - match spawn? { - ManagedSpawn::AlreadyRunning(report) => { - assert_eq!(report.service_id, "lemonade-held-1"); - } - ManagedSpawn::Spawned { .. } => { - panic!( - "expected the pre-seeded AlreadyLive record to short-circuit, not a real spawn" - ) - } - } - Ok(()) - } - - #[test] - fn abandoned_launch_leaves_no_record_log_or_endpoint_key() -> Result<()> { - let (root, paths) = test_paths("abandon-launch-cleanup"); - paths.ensure()?; - let mut record = write_live_service_record( - &paths, - "lemonade-abandon", - "abandon-model", - 11_490, - "running", - )?; - fs::create_dir_all( - record - .engine_state_path - .parent() - .expect("engine state parent"), - )?; - fs::write(&record.engine_state_path, "{}")?; - endpoint_keys::store_endpoint_api_key(&paths, &record.service_id, "secret-key")?; - // Simulate a pre-spawn failure: nothing ever ran. - record.engine_pid = None; - - abandon_failed_managed_launch(&paths, &record); - - assert!(!record.manifest_path.exists()); - assert!(!record.log_path.exists()); - assert!(!record.engine_state_path.exists()); - assert_eq!( - endpoint_keys::endpoint_api_key(&paths, &record.service_id), - None - ); - assert!(live_managed_services(&paths).is_empty()); - let _ = fs::remove_dir_all(root); - Ok(()) - } - - #[test] - #[cfg(not(windows))] - #[ignore = "spawned as a child process by abandoned_launch_terminates_the_spawned_child"] - fn managed_launch_cleanup_sleeper_child() { - if std::env::var_os("ROCM_TEST_SLEEPER_CHILD").is_some() { - thread::sleep(Duration::from_mins(2)); - } - } - - #[test] - #[cfg(target_os = "linux")] - fn abandoned_launch_never_signals_a_pid_that_no_longer_matches_the_spawn_token() -> Result<()> { - // The child exited and its pid now belongs to an unrelated live - // process: the start token recorded at spawn no longer matches, so the - // forced tree kill must not touch it. - let (root, paths) = test_paths("abandon-launch-recycled-pid"); - paths.ensure()?; - let mut record = write_live_service_record( - &paths, - "lemonade-abandon-recycled", - "abandon-recycled-model", - 11_493, - "running", - )?; - let current_exe = std::env::current_exe().expect("current test exe"); - let mut bystander = ProcessCommand::new(current_exe) - .arg("--exact") - .arg("service_launch::tests::managed_launch_cleanup_sleeper_child") - .arg("--ignored") - .arg("--test-threads=1") - .env("ROCM_TEST_SLEEPER_CHILD", "1") - .spawn() - .expect("spawn bystander"); - record.record_engine_identity(bystander.id()); - record.engine_start_ticks = record.engine_start_ticks.map(|ticks| ticks + 1); - - abandon_failed_managed_launch(&paths, &record); - - let mismatched_left_alone = bystander.try_wait().expect("poll bystander").is_none(); - // A child that was gone before its token could be captured leaves none: - // that must not fall back to "best-effort" signalling either. - record.engine_start_ticks = None; - abandon_failed_managed_launch(&paths, &record); - let still_running = - mismatched_left_alone && bystander.try_wait().expect("poll bystander").is_none(); - let _ = bystander.kill(); - let _ = bystander.wait(); - let _ = fs::remove_dir_all(root); - assert!( - still_running, - "a pid with a mismatched token must be left alone" - ); - Ok(()) - } - - #[test] - #[cfg(not(windows))] - fn abandoned_launch_terminates_the_spawned_child() -> Result<()> { - let (root, paths) = test_paths("abandon-launch-terminates-child"); - paths.ensure()?; - let mut record = write_live_service_record( - &paths, - "lemonade-abandon-child", - "abandon-child-model", - 11_491, - "running", - )?; - - let current_exe = std::env::current_exe().expect("current test exe"); - let mut child = ProcessCommand::new(current_exe) - .arg("--exact") - .arg("service_launch::tests::managed_launch_cleanup_sleeper_child") - .arg("--ignored") - .arg("--test-threads=1") - .env("ROCM_TEST_SLEEPER_CHILD", "1") - .spawn() - .expect("spawn sleeper child"); - let child_pid = child.id(); - record.record_engine_identity(child_pid); - - abandon_failed_managed_launch(&paths, &record); - - let wait_result = child.wait(); - - match wait_result { - Ok(status) => assert!( - !status.success(), - "the sleeper child must be terminated, not exit cleanly" - ), - Err(error) => panic!("failed to wait for sleeper child: {error}"), - } - let persisted = load_managed_services(&paths) - .expect("re-read manifests") - .into_iter() - .find(|candidate| candidate.service_id == record.service_id); - let _ = fs::remove_dir_all(root); - let persisted = persisted.expect( - "a launch that spawned a child must keep its record so `rocm services`/`rocm logs` can show the attempt", - ); - // `stopped`, not `failed`: `rocmd`'s recovery restarts a `failed` record - // that carries no stop marker, which would respawn a launch the CLI just - // reported as failed. The rocmd side of the pair is - // `abandoned_launch_record_is_not_a_recovery_candidate` in - // `apps/rocmd/src/watchers.rs`. - assert_eq!( - persisted.status, "stopped", - "a spawned child's record must be kept, but in a status recovery ignores" - ); - Ok(()) - } - - #[test] - #[cfg(not(windows))] - fn launch_releases_the_leased_port_before_spawning_the_child() -> Result<()> { - // `lease.release()` must run before the child is spawned, not after, - // or the engine is handed a port the CLI is still holding open. - // Proven via the injected `spawn_override` seam: it asserts a fresh - // bind on the leased port succeeds at the exact instant production - // would spawn, which only holds if the lease was already released. - let (root, paths) = test_paths("launch-releases-port-before-spawn"); - paths.ensure()?; - fs::create_dir_all(paths.services_dir())?; - - // Ephemeral, not a fixed number in the automatic range, which a real - // `rocm serve` on the developer's machine could be holding. - let port = std::net::TcpListener::bind(("127.0.0.1", 0))? - .local_addr()? - .port(); - let lease = rocm_core::lease_loopback_port(port)?; - let mut record = ManagedServiceRecord::new( - &paths, - "lemonade-release-before-spawn", - "lemonade", - "qwen", - "qwen-canonical", - "127.0.0.1", - port, - "managed", - 0, - None, - None, - None, - ); - record.write()?; - - let resolve = ResolveModelResponse { - canonical_model_id: "qwen-canonical".to_owned(), - served_model_id: None, - task: "chat".to_owned(), - source: "hf".to_owned(), - revision: "main".to_owned(), - loader: "llama.cpp".to_owned(), - trust_remote_code: false, - chat_template_mode: "auto".to_owned(), - dtype: "auto".to_owned(), - device_policy: DevicePolicy::GpuPreferred, - estimated_memory: "unknown".to_owned(), - launch_defaults: serde_json::json!({}), - engine_recipe: None, - warnings: Vec::new(), - }; - - let bind_succeeded = std::cell::Cell::new(false); - let spawn_override: &dyn Fn(&mut ProcessCommand) -> io::Result = - &|_command| { - // Retry briefly: a sibling test's concurrent fork can hold a copy - // of the just-released socket until its exec closes it. - let deadline = std::time::Instant::now() + Duration::from_secs(2); - while !bind_succeeded.get() && std::time::Instant::now() < deadline { - bind_succeeded.set(std::net::TcpListener::bind(("127.0.0.1", port)).is_ok()); - } - // A long-lived no-op child stands in for the engine: `_command` - // carries real serve args this test binary cannot execute as - // itself (see `managed_launch_cleanup_sleeper_child`). - ProcessCommand::new(std::env::current_exe().expect("test binary")) - .arg("service_launch::tests::managed_launch_cleanup_sleeper_child") - .arg("--exact") - .arg("--ignored") - .env("ROCM_TEST_SLEEPER_CHILD", "1") - .stdin(Stdio::null()) - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .spawn() - }; - - let result = launch_managed_engine_child_locked( - &paths, - &mut record, - lease, - "lemonade", - "lemonade-release-before-spawn", - &resolve, - "127.0.0.1", - &resolve.device_policy, - &[], - None, - None, - None, - Some(spawn_override), - ); - if let Ok(child_pid) = result.as_ref() { - let _ = rocm_core::terminate_process(*child_pid); - } - let _ = fs::remove_dir_all(root); - - assert!( - bind_succeeded.get(), - "the port must already be bindable (released) by the time the child is spawned" - ); - result?; - Ok(()) - } -} +pub(crate) use spawn::{ + attach_background_stdio, detach_background_command, managed_engine_startup_failure_detail, + managed_service_process_command, +}; +pub(crate) use start::{ + print_managed_launch_plain, start_managed_service, wait_begin_client_config, +}; diff --git a/apps/rocm/src/service_launch/attach.rs b/apps/rocm/src/service_launch/attach.rs new file mode 100644 index 0000000000..cd1bfe7700 --- /dev/null +++ b/apps/rocm/src/service_launch/attach.rs @@ -0,0 +1,768 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The attached serve path (`--verbose`/`--foreground`): spawn the managed child +//! and stream its log in this terminal until the user detaches or stops it. + +use super::endpoint_auth::drop_orphaned_endpoint_key_on_already_running; +use super::helper_autostart::ensure_background_helper_running_quiet; +use super::spawn::{ManagedSpawn, spawn_managed_engine_child}; +use super::start::render_endpoint_client_config; +use crate::record_cli_audit_event; +use crate::serve_port; +use crate::service_hints; +use crate::services_text::{ + render_service_action_result, render_service_action_result_with_helper_note, + service_stop_unconfirmed, +}; +use anyhow::{Context, Result, bail}; +use rocm_core::AppPaths; +use rocm_core::{format_http_base_url, process_is_running}; +use rocm_engine_protocol::ResolveModelResponse; +use std::fs; +use std::io::{self, BufRead, Write}; +use std::path::Path; +use std::thread; +use std::time::Duration; + +/// What ended an attached (`--verbose`/`--foreground`) streaming session. Kept +/// as a plain enum, separate from any terminal I/O, so the follow-up action +/// (detach note vs. stop the server) is unit-testable without a TTY. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum AttachOutcome { + /// Ctrl-D: leave the server running and hand the terminal back. + Detach, + /// Ctrl-C: stop the server, then hand the terminal back. + Stop, + /// The engine process exited on its own while we were streaming its log. + ServerExited, +} + +/// Attached serve path for `--verbose`/`--foreground`: spawn the engine as a +/// detached managed child (the same child the background path spawns) and stream +/// its log in this terminal. Unlike the old in-process foreground, the server +/// survives the session — Ctrl-D detaches and leaves it running, Ctrl-C stops it. +/// +/// `stop_service` is how Ctrl-C stops the server: the caller passes the managed +/// stop, so the `service_launch` module does not import the stop path, which +/// itself imports its spawn helpers. It returns the same `stop_server` answer +/// [`finish_attached_stop`] reads. +#[allow(clippy::too_many_arguments)] +pub(crate) fn run_attached_service( + paths: &AppPaths, + engine: &str, + service_id: &str, + requested_model: &str, + resolve: &ResolveModelResponse, + host: &str, + port_request: serve_port::PortRequest, + gpu_indices: &[u32], + runtime_id: Option<&str>, + env_id: Option<&str>, + endpoint_api_key: Option<&str>, + launch_lock: rocm_core::FileLock, + require_api_key: bool, + gpu_refusal_waived_for_reuse: bool, + stop_service: impl FnOnce(&AppPaths, &str) -> Result, +) -> Result<()> { + let spawn = spawn_managed_engine_child( + paths, + engine, + service_id, + requested_model, + resolve, + host, + port_request, + &resolve.device_policy, + gpu_indices, + runtime_id, + env_id, + resolve.engine_recipe.as_ref(), + require_api_key, + gpu_refusal_waived_for_reuse, + ) + // Mirrors `start_managed_service`'s cleanup: a pre-spawn + // failure (port reserved/in use, range exhausted, other bind error) returns + // before any service record exists, so nothing else clears the key just + // stored for this attempt. + .inspect_err(|_| { + drop_orphaned_endpoint_key_on_already_running(paths, service_id, endpoint_api_key); + })?; + // The claiming record is persisted (or an existing service was found), so the + // selected GPU is now visible to concurrent auto-selection. Release the launch + // lock before streaming logs, which blocks for the whole attached session. + drop(launch_lock); + + // Attached mode is a managed record too, so it shares the transaction's + // resolved concrete port. + let (service_id, log_path, port, child_pid) = match spawn { + // A server for this engine+model is already live. Don't fight it for the + // port — point the user at the existing one instead of tailing a log we + // did not start. + ManagedSpawn::AlreadyRunning(report) => { + println!("model already being served"); + println!(" service_id: {}", report.service_id); + println!(" endpoint: {}", report.endpoint_url); + println!(" status: {}", report.status); + println!(" logs: rocm logs {}", report.service_id); + println!( + " stop: {}", + service_hints::service_stop_hint(&report.service_id) + ); + drop_orphaned_endpoint_key_on_already_running(paths, service_id, endpoint_api_key); + return Ok(()); + } + ManagedSpawn::Spawned { record, child_pid } => ( + service_id.to_owned(), + record.log_path.clone(), + record.port, + child_pid, + ), + }; + + // The child is a managed service that outlives this session once detached. + // The background helper adds one thing for it: with the `server-recover` + // watcher enabled, it health-checks managed servers and restarts failed ones. + // Every duty of the helper is watcher-gated, so with no watcher enabled no + // helper is started. Nothing else depends on it: a self-exited server's + // record is settled by its own supervisor (`supervise_service` writes the + // exit via `record_supervised_exit` in `apps/rocmd/src/supervise.rs`) and on + // every read (`refresh_managed_service_runtime_liveness`), and the dashboard + // has its own embedded daemon. Match the background ordering (spawn, then + // ensure the helper) and keep it quiet so no status line breaks into the + // log stream. + ensure_background_helper_running_quiet(true)?; + + // The resolution detail (model, engine, runtime, GPU, warnings) was already + // printed as the "serve plan" block in `serve()`; extend it with the launch + // coordinates and the streaming hint rather than repeating it. + let endpoint = format!("{}/v1", format_http_base_url(host, port)); + println!(" service_id: {service_id}"); + println!(" endpoint: {endpoint}"); + if let Some(key) = endpoint_api_key { + // Same intentional one-time key display as `print_managed_launch_plain` + // in `service_launch/start.rs`; see its rationale for why the tag below is + // currently inert. + // codeql[rust/cleartext-logging] + print!("{}", render_endpoint_client_config(&endpoint, key)); + } + println!(" streaming engine logs - Ctrl-D detaches (leaves it running), Ctrl-C stops it"); + println!(); + + let outcome = stream_attached_logs(&log_path, child_pid)?; + println!(); + + match outcome { + AttachOutcome::Detach => { + // Audit before printing: a closed stdout makes `println!` panic, + // which must not lose the event. + record_cli_audit_event( + paths, + "service", + "serve_detach", + "info", + format!("detached from attached serve service_id={service_id} endpoint={endpoint}"), + Some(&service_id), + ); + println!("detached - server still running"); + println!(" service_id: {service_id}"); + println!(" endpoint: {endpoint}"); + println!(" list: rocm services"); + println!(" logs: rocm logs {service_id}"); + println!(" stop: {}", service_hints::service_stop_hint(&service_id)); + Ok(()) + } + AttachOutcome::Stop => { + println!("stopping server..."); + let stop = stop_service(paths, &service_id); + finish_attached_stop( + paths, + &service_id, + child_pid, + stop, + rocm_core::terminate_process_tree, + &mut io::stdout().lock(), + &mut io::stderr().lock(), + ) + } + AttachOutcome::ServerExited => { + println!("server process exited"); + println!(" service_id: {service_id}"); + println!(" recent logs: rocm logs {service_id}"); + Ok(()) + } + } +} + +/// Restores cooked terminal mode when dropped, so [`stream_attached_logs`] leaves +/// the terminal usable on every exit path (normal return, `?` error, or panic). +struct RawModeGuard; + +impl Drop for RawModeGuard { + fn drop(&mut self) { + let _ = crossterm::terminal::disable_raw_mode(); + } +} + +/// Map a key press (control modifier + lowercased character) to the attach +/// action it triggers, if any. Factored out of the raw-mode reader loop so the +/// Ctrl-D/Ctrl-C mapping is unit-testable without a terminal. +const fn detach_key_outcome(ctrl: bool, ch: char) -> Option { + if !ctrl { + return None; + } + match ch { + 'c' => Some(AttachOutcome::Stop), + 'd' => Some(AttachOutcome::Detach), + _ => None, + } +} + +/// Follow `log_path` in the terminal until the user presses Ctrl-D (detach) or +/// Ctrl-C (stop), or the engine process exits. Uses crossterm raw mode to +/// capture the keys directly (in raw mode Ctrl-C does not raise SIGINT, so we see +/// it as a key event). When stdin is not a TTY (piped/CI), keystroke capture is +/// impossible, so we follow the log until the process exits instead. +fn stream_attached_logs(log_path: &Path, child_pid: u32) -> Result { + use std::io::IsTerminal as _; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::mpsc; + + if !std::io::stdin().is_terminal() { + return stream_attached_logs_no_tty(log_path, child_pid); + } + + // Enter raw mode *before* spawning the key reader. In raw mode Ctrl-C arrives + // as a key event instead of SIGINT; if the reader started first, a Ctrl-C in + // that window would kill the CLI outright (leaving the detached child alive + // but printing no detach/stop message). The guard restores cooked mode on + // every exit path (normal return, `?` error, panic). + crossterm::terminal::enable_raw_mode().context("failed to enter raw terminal mode")?; + let _raw_guard = RawModeGuard; + + let stop = Arc::new(AtomicBool::new(false)); + let (tx, rx) = mpsc::channel::(); + + let reader_stop = Arc::clone(&stop); + let reader = thread::spawn(move || { + use crossterm::event::{self, Event, KeyCode, KeyEventKind, KeyModifiers}; + while !reader_stop.load(Ordering::Relaxed) { + match event::poll(Duration::from_millis(100)) { + Ok(true) => match event::read() { + Ok(Event::Key(key)) if key.kind != KeyEventKind::Release => { + let outcome = match key.code { + KeyCode::Char(ch) => detach_key_outcome( + key.modifiers.contains(KeyModifiers::CONTROL), + ch.to_ascii_lowercase(), + ), + _ => None, + }; + if let Some(outcome) = outcome { + let _ = tx.send(outcome); + break; + } + } + Ok(_) => {} + Err(_) => break, + }, + Ok(false) => {} + Err(_) => break, + } + } + }); + + let mut stdout = io::stdout(); + let mut log_reader: Option> = None; + let mut line = String::new(); + let outcome = loop { + if log_reader.is_none() { + log_reader = fs::File::open(log_path).ok().map(io::BufReader::new); + } + if let Some(reader) = log_reader.as_mut() { + loop { + line.clear(); + match reader.read_line(&mut line) { + Ok(0) => break, + Ok(_) => { + // Raw mode disables the terminal's own \n -> \r\n + // translation, so emit an explicit carriage return to + // keep the log left-aligned instead of stair-stepping. + let _ = write!(stdout, "{}\r\n", line.trim_end_matches('\n')); + let _ = stdout.flush(); + } + Err(_) => break, + } + } + } + + if let Ok(signal) = rx.try_recv() { + break signal; + } + if !process_is_running(child_pid) { + break AttachOutcome::ServerExited; + } + thread::sleep(Duration::from_millis(150)); + }; + + stop.store(true, Ordering::Relaxed); + let _ = reader.join(); + Ok(outcome) +} + +/// Non-interactive fallback for [`stream_attached_logs`]: no keystroke capture, +/// so just follow the log until the (detached) engine process exits. A Ctrl-C +/// here delivers SIGINT to this process and leaves the managed server running. +fn stream_attached_logs_no_tty(log_path: &Path, child_pid: u32) -> Result { + let mut stdout = io::stdout(); + let mut log_reader: Option> = None; + let mut line = String::new(); + loop { + if log_reader.is_none() { + log_reader = fs::File::open(log_path).ok().map(io::BufReader::new); + } + if let Some(reader) = log_reader.as_mut() { + loop { + line.clear(); + match reader.read_line(&mut line) { + Ok(0) => break, + Ok(_) => { + let _ = write!(stdout, "{line}"); + let _ = stdout.flush(); + } + Err(_) => break, + } + } + } + if !process_is_running(child_pid) { + return Ok(AttachOutcome::ServerExited); + } + thread::sleep(Duration::from_millis(200)); + } +} + +/// Finish the attached `rocm serve` Ctrl-C stop once the managed stop has +/// answered: what is printed where, what is signalled, what is audited, and +/// how the command exits. Everything but the stop itself and the real +/// signal is here, so a test can drive it. +/// +/// - Confirmed: the report, with the background-helper note `rocm services +/// stop` also prints, goes to `out` and the command succeeds. +/// - Unconfirmed: the same report `rocm services stop` gives, check lines +/// included, becomes the error, so it reaches stderr and the command exits +/// non-zero. Nothing is signalled on the stop's behalf: the stop has already +/// done what it safely could. +/// - The stop itself failed: `signal_child` gets the child this session +/// launched, so Ctrl-C never leaves the model orphaned when the stop path +/// breaks. The stop's error, and the signal's if that fails too, are noted on +/// `err`, and the audit event records whether the signal worked. The command +/// still exits 0, as it always has here. +fn finish_attached_stop( + paths: &AppPaths, + service_id: &str, + child_pid: u32, + stop: Result, + signal_child: impl FnOnce(u32) -> Result<()>, + out: &mut dyn Write, + err: &mut dyn Write, +) -> Result<()> { + match stop { + Ok(result) if service_stop_unconfirmed(&result) => { + record_cli_audit_event( + paths, + "service", + "serve_stop", + "error", + format!("stop of attached serve not confirmed service_id={service_id}"), + Some(service_id), + ); + bail!( + "{}", + render_service_action_result("stop_server", &result).trim_end() + ) + } + Ok(result) => { + record_cli_audit_event( + paths, + "service", + "serve_stop", + "info", + format!("stopped attached serve service_id={service_id}"), + Some(service_id), + ); + write!( + out, + "{}", + render_service_action_result_with_helper_note(paths, "stop_server", &result) + )?; + Ok(()) + } + Err(error) => { + let signal = signal_child(child_pid); + // Record the audit event before writing to stderr, so a failed + // write cannot skip it. + let message = match &signal { + Ok(()) => format!( + "stop failed; signaled attached serve child {child_pid} service_id={service_id}" + ), + Err(signal_error) => format!( + "stop failed; signaling attached serve child {child_pid} also failed: {signal_error:#} service_id={service_id}" + ), + }; + record_cli_audit_event( + paths, + "service", + "serve_stop", + if signal.is_ok() { "warn" } else { "error" }, + message, + Some(service_id), + ); + writeln!(err, " note: {error:#}")?; + if let Err(signal_error) = &signal { + writeln!( + err, + " note: signaling this session's server process {child_pid} directly also failed: {signal_error:#}" + )?; + } + Ok(()) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::endpoint_keys; + use crate::tests::test_paths; + use rocm_core::ManagedServiceRecord; + use rocm_engine_protocol::DevicePolicy; + + /// A `stop_server` answer whose one recorded process ended with `outcome`. + fn stop_answer(confirmed: bool, outcome: &str) -> serde_json::Value { + crate::services_text::internal_stop_server_value(serde_json::json!({ + "service_id": "svc-attached", + "status": "ready", + "signaled_pids": if outcome == "unverified" { vec![] } else { vec![4242] }, + "pid_outcomes": [{ "pid": 4242, "role": "supervisor_and_engine", "outcome": outcome }], + // An attached serve on a public bind stores a key; an unconfirmed + // stop keeps it. + "endpoint_key_kept": !confirmed, + "stopped": confirmed, + })) + } + + /// Run [`finish_attached_stop`] for `stop`, returning its result, what it + /// wrote to stdout, the PID it signalled (if any) and the audit event. + fn finish( + name: &str, + stop: Result, + ) -> (Result<()>, String, Option, serde_json::Value) { + finish_signalling(name, stop, Ok(())) + } + + /// [`finish`], with the direct signal answering `signal_result`. + fn finish_signalling( + name: &str, + stop: Result, + signal_result: Result<()>, + ) -> (Result<()>, String, Option, serde_json::Value) { + let (result, out, _err, signalled, audit) = finish_capturing(name, stop, signal_result); + (result, out, signalled, audit) + } + + /// [`finish_signalling`], also returning what was written to stderr. + fn finish_capturing( + name: &str, + stop: Result, + signal_result: Result<()>, + ) -> (Result<()>, String, String, Option, serde_json::Value) { + let (root, paths) = test_paths(name); + paths.ensure().unwrap(); + let mut out = Vec::new(); + let mut err = Vec::new(); + let mut signalled = None; + let result = finish_attached_stop( + &paths, + "svc-attached", + 777, + stop, + |pid| { + signalled = Some(pid); + signal_result + }, + &mut out, + &mut err, + ); + let audit_text = fs::read_to_string(paths.audit_events_path()).expect("audit log"); + let audit = serde_json::from_str(audit_text.lines().last().expect("an audit event")) + .expect("audit json"); + let _ = fs::remove_dir_all(root); + ( + result, + String::from_utf8(out).unwrap(), + String::from_utf8(err).unwrap(), + signalled, + audit, + ) + } + + #[test] + fn attached_ctrl_c_reports_a_confirmed_stop_on_stdout() { + let (result, out, signalled, audit) = + finish("attached-confirmed", Ok(stop_answer(true, "graceful"))); + result.expect("a confirmed stop succeeds"); + assert!(out.contains("Local server stopped"), "{out}"); + assert_eq!(signalled, None); + assert_eq!(audit["level"], "info", "{audit}"); + assert_eq!( + audit["message"], + "stopped attached serve service_id=svc-attached" + ); + } + + #[test] + fn attached_ctrl_c_fails_an_unconfirmed_stop_like_services_stop_and_signals_nothing() { + // The same report `rocm services stop` gives, as the error (stderr, + // non-zero exit), naming no retry; nothing is signalled on the stop's + // behalf and nothing reaches stdout. + let answer = stop_answer(false, "timed_out"); + let (result, out, signalled, audit) = finish("attached-timed-out", Ok(answer.clone())); + let error = format!("{:#}", result.expect_err("an unconfirmed stop fails")); + assert_eq!( + error, + render_service_action_result("stop_server", &answer).trim_end(), + "the report is `rocm services stop`'s own" + ); + assert!( + error.contains( + "process 4242 (supervisor_and_engine) or a process under it did not exit" + ), + "{error}" + ); + assert!( + error.lines().any(|line| line + == " kept: the server record, which still names its processes, and its endpoint key, which the server may still be enforcing"), + "{error}" + ); + crate::services_text::tests::assert_no_retry_line(&error); + assert!(out.is_empty(), "nothing on stdout: {out}"); + assert_eq!(signalled, None, "an unconfirmed stop signals nothing more"); + assert_eq!(audit["level"], "error", "{audit}"); + assert_eq!( + audit["message"], + "stop of attached serve not confirmed service_id=svc-attached" + ); + } + + #[test] + fn attached_ctrl_c_names_a_check_for_an_unverified_process() { + let (result, out, signalled, audit) = + finish("attached-unverified", Ok(stop_answer(false, "unverified"))); + let error = format!("{:#}", result.expect_err("an unverified stop fails")); + assert!(error.contains("not signaled: process 4242"), "{error}"); + assert!(error.contains("check: `ps -p 4242 -o pid,args`"), "{error}"); + crate::services_text::tests::assert_no_retry_line(&error); + assert!(out.is_empty(), "{out}"); + assert_eq!(signalled, None); + assert_eq!(audit["level"], "error", "{audit}"); + } + + #[test] + fn attached_ctrl_c_signals_its_own_child_when_the_stop_itself_fails() { + let (result, out, err, signalled, audit) = finish_capturing( + "attached-stop-error", + Err(anyhow::anyhow!("managed service `svc-attached` not found") + .context("failed to stop svc-attached")), + Ok(()), + ); + result.expect("a failed stop falls back and still exits cleanly, as before"); + assert_eq!(signalled, Some(777)); + assert_eq!( + err, " note: failed to stop svc-attached: managed service `svc-attached` not found\n", + "the stop's error goes to stderr, and a signal that worked adds nothing" + ); + assert!(out.is_empty(), "nothing on stdout: {out}"); + assert_eq!(audit["level"], "warn", "{audit}"); + assert_eq!( + audit["message"], + "stop failed; signaled attached serve child 777 service_id=svc-attached" + ); + + // The direct signal failing too is printed on stderr and recorded, with + // its error. + let (result, _, err, signalled, audit) = finish_capturing( + "attached-stop-error-signal-fails", + Err(anyhow::anyhow!("managed service `svc-attached` not found")), + Err(anyhow::anyhow!("no such process")), + ); + result.expect("still exits 0, as on main"); + assert_eq!( + err, + " note: managed service `svc-attached` not found\n \ + note: signaling this session's server process 777 directly also failed: no such process\n", + "both notes go to stderr" + ); + assert_eq!(signalled, Some(777)); + assert_eq!(audit["level"], "error", "{audit}"); + assert_eq!( + audit["message"], + "stop failed; signaling attached serve child 777 also failed: no such process service_id=svc-attached" + ); + } + + /// A writer whose every write fails, standing in for a closed stdout/stderr. + struct BrokenWriter; + + impl Write for BrokenWriter { + fn write(&mut self, _: &[u8]) -> std::io::Result { + Err(std::io::Error::from(std::io::ErrorKind::BrokenPipe)) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + + #[test] + fn attached_ctrl_c_records_the_audit_event_even_when_writing_the_outcome_fails() { + // One case per branch that writes after deciding: a confirmed stop + // (stdout) and a stop that itself failed (stderr notes). The write + // error is returned, but the audit event is already on disk. + let cases: [(&str, Result, &str); 2] = [ + ( + "attached-broken-out", + Ok(stop_answer(true, "graceful")), + "stopped attached serve service_id=svc-attached", + ), + ( + "attached-broken-err", + Err(anyhow::anyhow!("managed service `svc-attached` not found")), + "stop failed; signaled attached serve child 777 service_id=svc-attached", + ), + ]; + for (name, stop, expected_message) in cases { + let (root, paths) = test_paths(name); + paths.ensure().unwrap(); + let result = finish_attached_stop( + &paths, + "svc-attached", + 777, + stop, + |_| Ok(()), + &mut BrokenWriter, + &mut BrokenWriter, + ); + let audit_text = fs::read_to_string(paths.audit_events_path()).unwrap_or_default(); + let _ = fs::remove_dir_all(root); + assert!(result.is_err(), "{name}: the write error is surfaced"); + let audit: serde_json::Value = serde_json::from_str( + audit_text + .lines() + .last() + .unwrap_or_else(|| panic!("{name}: no audit event recorded")), + ) + .expect("audit json"); + assert_eq!(audit["message"], expected_message, "{name}: {audit}"); + } + } + + #[test] + fn detach_key_ctrl_d_detaches_ctrl_c_stops() { + assert_eq!(detach_key_outcome(true, 'd'), Some(AttachOutcome::Detach)); + assert_eq!(detach_key_outcome(true, 'c'), Some(AttachOutcome::Stop)); + } + + #[test] + fn detach_key_ignores_plain_and_unrelated_keys() { + // Without the control modifier, `d`/`c` are ordinary log-scroll input. + assert_eq!(detach_key_outcome(false, 'd'), None); + assert_eq!(detach_key_outcome(false, 'c'), None); + // Other control combos are not detach/stop triggers. + assert_eq!(detach_key_outcome(true, 'q'), None); + assert_eq!(detach_key_outcome(true, 'z'), None); + } + + #[test] + fn attached_launch_failing_before_spawn_leaves_no_orphaned_endpoint_key() -> Result<()> { + // The background path's cleanup is the `.inspect_err` in + // start_managed_service. run_attached_service is the other caller of + // spawn_managed_engine_child and needs the identical .inspect_err() + // cleanup, not a simulated one: drive the real function end to end. + let (root, paths) = test_paths("orphaned-key-attached-reserved-port"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + let mut other = ManagedServiceRecord::new( + &paths, + "lemonade-llama-1000", + "lemonade", + "llama", + "llama-canonical", + "127.0.0.1", + 11514, + "managed", + std::process::id(), + None, + None, + None, + ); + other.status = "ready".to_owned(); + other.engine_pid = Some(std::process::id()); + other.write()?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + + let service_id = "lemonade-qwen-attached-2000"; + endpoint_keys::store_endpoint_api_key(&paths, service_id, "test-key")?; + let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + + let result = run_attached_service( + &paths, + "lemonade", + service_id, + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Explicit(11514), + &[], + None, + None, + Some("test-key"), + launch_lock, + false, + false, + |_, _| unreachable!("the launch fails before a Ctrl-C could stop it"), + ); + let Err(error) = result else { + let _ = fs::remove_dir_all(root); + panic!("explicit port already reserved by a live service must be rejected") + }; + assert!( + error + .to_string() + .contains("already reserved by managed service"), + "unexpected error: {error}" + ); + assert!( + endpoint_keys::endpoint_api_key(&paths, service_id).is_none(), + "a pre-spawn failure must drop the key this attempt staged before launching" + ); + + let _ = fs::remove_dir_all(root); + Ok(()) + } +} diff --git a/apps/rocm/src/service_launch/endpoint_auth.rs b/apps/rocm/src/service_launch/endpoint_auth.rs new file mode 100644 index 0000000000..6a647314bc --- /dev/null +++ b/apps/rocm/src/service_launch/endpoint_auth.rs @@ -0,0 +1,377 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Bind-host validation and the endpoint API key checks that gate a public +//! bind: [`validate_bind_host`], [`resolve_endpoint_auth`] and the guards that +//! keep a respawn from reopening a service without its key. + +use crate::endpoint_keys; +use anyhow::{Result, bail}; +use rocm_core::AppPaths; + +pub(crate) fn validate_bind_host(host: &str, allow_public_bind: bool) -> Result<()> { + if !is_loopback_host(host) && !allow_public_bind { + bail!( + "`rocm serve --host {host}` is not loopback; pass `--allow-public-bind` before binding a non-local interface" + ); + } + Ok(()) +} + +/// Inverse of [`rocm_engine_protocol::is_public_bind_host`], which owns the +/// policy so `rocmd` classifies a recorded `host` identically when it respawns +/// the service. +pub(crate) fn is_loopback_host(host: &str) -> bool { + !rocm_engine_protocol::is_public_bind_host(host) +} + +/// Resolve the API key that will guard this endpoint, applying the +/// loopback-vs-public policy for `rocm serve`. +/// +/// - **Loopback host** → `None`: local serving stays credential-free (the +/// unchanged default). Any key supplied for a loopback bind is ignored and +/// nothing is persisted — loopback needs no auth. +/// - **Public host** → `Some(key)`: use the user-supplied key when present, +/// otherwise generate a strong random one so a public endpoint can never come +/// up anonymous. An empty/whitespace supplied key is rejected rather than +/// silently treated as "no auth". +/// - **`required`** → treat a loopback bind as public for this purpose. +/// +/// That last case exists because "loopback" is a statement about the bind +/// address, not about who can reach the port. Publishing the port onto a +/// tailnet, proxying it, or mapping it out of a container all leave the bind +/// loopback while widening the audience — and the policy above would then hand +/// out an unauthenticated endpoint. Whoever widens the reach is responsible for +/// asking for the credential, so this is an explicit flag rather than a guess. +pub(crate) fn resolve_endpoint_auth( + host: &str, + supplied: Option<&str>, + required: bool, +) -> Result> { + if is_loopback_host(host) && !required { + return Ok(None); + } + match supplied { + Some(raw) => { + let trimmed = raw.trim(); + if trimmed.is_empty() { + bail!( + "`rocm serve --api-key` (or ROCM_SERVE_API_KEY) was empty; a public \ + endpoint must be protected by a non-empty API key" + ); + } + // The key is later interpolated verbatim into raw `Authorization: + // Bearer {key}\r\n` header lines; reject a control character (e.g. an + // embedded CR/LF) here so a crafted key cannot inject extra headers. + if rocm_core::endpoint_api_key_has_forbidden_chars(trimmed) { + bail!( + "`rocm serve --api-key` (or ROCM_SERVE_API_KEY) contained a control \ + character such as a carriage return or newline; an endpoint API key \ + must be a single line of printable characters" + ); + } + Ok(Some(trimmed.to_owned())) + } + None => Ok(Some(rocm_core::generate_endpoint_api_key())), + } +} + +/// When an equivalent managed service is already running, or the launch fails +/// before any record exists, the endpoint key serve() freshly stored for this +/// attempt is unused — drop it so it is not orphaned in storage. The already-running service keeps its own key. +/// Best-effort and idempotent; a loopback attempt (`freshly_stored == None`) +/// is a no-op. +pub(crate) fn drop_orphaned_endpoint_key_on_already_running( + paths: &AppPaths, + service_id: &str, + freshly_stored: Option<&str>, +) { + if freshly_stored.is_some() { + endpoint_keys::clear_endpoint_api_key(paths, service_id); + } +} + +/// Reject engine/platform combinations that cannot enforce a public endpoint's +/// API key, so a public bind fails closed instead of coming up unauthenticated. +/// +/// The one such case today: Windows managed Lemonade. Its server reads the +/// value-typed `LEMONADE_API_KEY` env var, but the Windows detached-spawn +/// primitive only carries path-valued env overrides, so the key never reaches it. +/// vLLM enforces auth on every platform (`VLLM_API_KEY`), and loopback binds +/// (`public_bind == false`) need no key — both pass. `is_windows` is a parameter +/// so both branches are unit-testable off-Windows. +pub(crate) fn ensure_public_bind_engine_supported( + engine: &str, + public_bind: bool, + is_windows: bool, +) -> Result<()> { + if public_bind && is_windows && engine == "lemonade" { + bail!( + "public binding with the lemonade engine is not supported on Windows: the endpoint \ + API key cannot be enforced there. Use `--engine vllm`, or bind a loopback host \ + (the default 127.0.0.1)." + ); + } + Ok(()) +} + +/// Refuse to (re)spawn a managed service recorded on a public host when its +/// endpoint API key is gone, so a respawn cannot reopen the endpoint anonymously. +/// +/// `serve()` applies the loopback-vs-public policy once, via +/// [`resolve_endpoint_auth`], and persists the resulting key. Every later spawn +/// — `rocm services restart`, and `rocmd`'s recovery supervisor — reads that key +/// file back and would otherwise treat "no key file" as "no auth wanted", +/// silently downgrading a protected public endpoint to an open one. The real +/// invariant is a property of the *host*, not of the file: a non-loopback bind +/// must always be authenticated. +/// +/// The gap is reachable through ordinary commands, because a stop deletes the +/// key file and `rocm services restart` accepts a stopped service id (its help +/// points at `rocm services list --all`). +/// +/// `key_present` is a plain `bool` rather than a path so both branches are +/// unit-testable without touching the filesystem, mirroring `is_windows` in +/// [`ensure_public_bind_engine_supported`]. +pub(crate) fn ensure_public_service_has_endpoint_key( + host: &str, + key_present: bool, + requires_api_key: bool, +) -> Result<()> { + // Two ways a service can need a key. A public bind is the obvious one. The + // other is a service that asked for auth on a loopback bind, because + // something outside this process republishes the port — a tailnet publish + // survives a reboot, let alone a restart, so "loopback" stops meaning + // "only this machine" and the bind address can no longer be trusted to + // answer the question on its own. + if requires_api_key && !key_present { + bail!( + "managed service was launched with `--require-api-key` but has no endpoint API key, \ + so restarting it would reopen it without authentication. Something outside this \ + machine may still be publishing its port. The key is dropped when a service stops \ + and cannot be recovered. Launch it again with \ + `rocm serve --require-api-key` (add `--api-key `, or set ROCM_SERVE_API_KEY, \ + to choose the key instead of generating one)." + ); + } + if rocm_engine_protocol::is_public_bind_host(host) && !key_present { + bail!( + "managed service is bound to the public host `{host}` but has no endpoint API key, \ + so restarting it would reopen it without authentication. The key is dropped when a \ + service stops and cannot be recovered. Launch it again with \ + `rocm serve --host {host} --port --allow-public-bind` (add `--api-key `, or set \ + ROCM_SERVE_API_KEY, to choose the key instead of generating one)." + ); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tests::test_paths; + use std::fs; + + #[test] + fn serve_bind_validation_requires_public_ack() { + validate_bind_host("127.0.0.1", false).unwrap(); + validate_bind_host("localhost", false).unwrap(); + validate_bind_host("::1", false).unwrap(); + let error = validate_bind_host("0.0.0.0", false).unwrap_err(); + assert!( + error.to_string().contains("--allow-public-bind"), + "{error:#}" + ); + validate_bind_host("0.0.0.0", true).unwrap(); + } + + #[test] + fn resolve_endpoint_auth_loopback_stays_credential_free() { + // Loopback binds never require auth, even if a key is supplied. + for host in ["127.0.0.1", "localhost", "::1"] { + assert_eq!(resolve_endpoint_auth(host, None, false).unwrap(), None); + assert_eq!( + resolve_endpoint_auth(host, Some("ignored"), false).unwrap(), + None + ); + } + } + + #[test] + fn resolve_endpoint_auth_loopback_can_be_required_when_something_republishes_it() { + // "Loopback" describes the bind address, not who can reach the port. A + // tailnet publish, a proxy, or a container port map all leave the bind + // loopback while widening the audience, and the default policy would + // hand out an unauthenticated endpoint. Whoever widens the reach asks + // for the credential explicitly. + for host in ["127.0.0.1", "localhost", "::1"] { + let generated = resolve_endpoint_auth(host, None, true) + .unwrap() + .expect("a required key must be generated, not skipped"); + assert!(!generated.trim().is_empty()); + + assert_eq!( + resolve_endpoint_auth(host, Some("supplied-key"), true).unwrap(), + Some("supplied-key".to_owned()), + "a supplied key must be honoured rather than ignored as it is by default" + ); + } + + // The same validation a public bind gets: an empty key is a refusal, not + // a silent downgrade to no auth. + assert!(resolve_endpoint_auth("127.0.0.1", Some(" "), true).is_err()); + } + + #[test] + fn resolve_endpoint_auth_public_uses_supplied_key_trimmed() { + let key = resolve_endpoint_auth("0.0.0.0", Some(" my-key "), false) + .unwrap() + .expect("public bind must have a key"); + assert_eq!(key, "my-key"); + } + + #[test] + fn resolve_endpoint_auth_public_generates_key_when_absent() { + let key = resolve_endpoint_auth("0.0.0.0", None, false) + .unwrap() + .expect("public bind must generate a key"); + assert_eq!(key.len(), 48); + assert!(key.chars().all(|c| c.is_ascii_alphanumeric())); + } + + #[test] + fn resolve_endpoint_auth_public_rejects_empty_supplied_key() { + let error = resolve_endpoint_auth("0.0.0.0", Some(" "), false).unwrap_err(); + assert!(error.to_string().contains("non-empty"), "{error:#}"); + } + + #[test] + fn resolve_endpoint_auth_public_rejects_embedded_crlf() { + // A supplied key survives `trim()` with embedded CR/LF intact and would + // otherwise be interpolated into a raw `Authorization: Bearer` header, + // injecting an extra header line. It must be rejected at input validation. + for supplied in [ + "good-key\r\nX-Injected: value", + "good-key\nmore", + "line\rreturn", + ] { + let error = resolve_endpoint_auth("0.0.0.0", Some(supplied), false).unwrap_err(); + assert!(error.to_string().contains("control character"), "{error:#}"); + } + } + + #[test] + fn drop_orphaned_endpoint_key_on_already_running_clears_stored_key() { + let (root, paths) = test_paths("drop-orphaned-key-stored"); + let service_id = "svc-orphaned"; + endpoint_keys::store_endpoint_api_key(&paths, service_id, "secret-key").unwrap(); + + drop_orphaned_endpoint_key_on_already_running(&paths, service_id, Some("secret-key")); + + assert_eq!(endpoint_keys::endpoint_api_key(&paths, service_id), None); + assert!(!endpoint_keys::endpoint_key_file_path(&paths, service_id).exists()); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn drop_orphaned_endpoint_key_on_already_running_is_noop_for_loopback() { + // A loopback attempt never stores a key (`freshly_stored == None`), so the + // helper must not panic or error, and no file must appear. + let (root, paths) = test_paths("drop-orphaned-key-loopback"); + let service_id = "svc-loopback"; + + drop_orphaned_endpoint_key_on_already_running(&paths, service_id, None); + + assert_eq!(endpoint_keys::endpoint_api_key(&paths, service_id), None); + assert!(!endpoint_keys::endpoint_key_file_path(&paths, service_id).exists()); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn public_bind_fails_closed_for_windows_lemonade_only() { + // Windows + Lemonade + public bind: refuse (cannot enforce the key). + let error = ensure_public_bind_engine_supported("lemonade", true, true).unwrap_err(); + assert!(error.to_string().contains("lemonade"), "{error:#}"); + // Every other combination is allowed: + ensure_public_bind_engine_supported("vllm", true, true).unwrap(); // vLLM enforces auth on Windows + ensure_public_bind_engine_supported("lemonade", true, false).unwrap(); // non-Windows + ensure_public_bind_engine_supported("lemonade", false, true).unwrap(); // loopback needs no key + } + + #[test] + fn a_public_bind_is_refused_with_the_command_that_restores_it() { + // `resolve_endpoint_auth` mints a key for every non-loopback bind whether + // or not auth was demanded, so deriving `requires_api_key` from key-file + // presence marked ordinary public binds as having asked for it. The guard + // tests that field first, so those services were refused with a message + // naming a flag they never passed and a relaunch command that drops + // `--allow-public-bind` — coming back on loopback instead. + // + // A plain `--host 0.0.0.0 --allow-public-bind` launch: key file present, + // `--require-api-key` never passed. + let error = ensure_public_service_has_endpoint_key("0.0.0.0", false, false) + .expect_err("a public bind with no key must be refused"); + let rendered = format!("{error:#}"); + assert!( + rendered.contains("--allow-public-bind"), + "the refusal must name the command that restores the public bind: {rendered}" + ); + assert!( + !rendered.contains("--require-api-key"), + "a service that never passed the flag must not be told it did: {rendered}" + ); + + // And the loopback-with-forced-auth case still reports its own reason. + let error = ensure_public_service_has_endpoint_key("127.0.0.1", false, true) + .expect_err("a service that demanded auth must not come back without it"); + assert!( + format!("{error:#}").contains("--require-api-key"), + "{error:#}" + ); + } + + #[test] + fn respawn_fails_closed_for_a_public_service_whose_key_is_gone() { + // A stop deletes the key file, so a later restart of a public service + // would otherwise respawn it with no auth at all. + let error = ensure_public_service_has_endpoint_key("0.0.0.0", false, false).unwrap_err(); + let message = error.to_string(); + assert!(message.contains("0.0.0.0"), "{error:#}"); + assert!(message.contains("without authentication"), "{error:#}"); + // Actionable: name the command that mints a fresh key. + assert!(message.contains("--allow-public-bind"), "{error:#}"); + + // A public service that still has its key restarts normally. + ensure_public_service_has_endpoint_key("0.0.0.0", true, false).unwrap(); + } + + #[test] + fn respawn_allows_loopback_services_without_an_endpoint_key() { + // Loopback stays credential-free, so every accepted spelling must pass + // the guard with no key present — when nothing asked for auth. + for host in ["127.0.0.1", "localhost", "::1"] { + ensure_public_service_has_endpoint_key(host, false, false) + .unwrap_or_else(|error| panic!("{host} must not require a key: {error:#}")); + } + } + + #[test] + fn respawn_refuses_a_loopback_service_that_was_launched_with_a_key() { + // The hole this closes: a loopback bind that something else republishes + // — a tailnet publish, a proxy, a container port map. The publish + // outlives the process, so a restart after the key was dropped would + // reopen a reachable endpoint with no authentication, and the bind + // address gives the guard no way to notice. + for host in ["127.0.0.1", "localhost", "::1"] { + let error = ensure_public_service_has_endpoint_key(host, false, true) + .expect_err("a service launched with a key must not restart without one"); + let message = format!("{error:#}"); + assert!(message.contains("without authentication"), "{message}"); + assert!(message.contains("--require-api-key"), "{message}"); + } + + // With its key still present it restarts normally. + ensure_public_service_has_endpoint_key("127.0.0.1", true, true).unwrap(); + } +} diff --git a/apps/rocm/src/service_launch/helper_autostart.rs b/apps/rocm/src/service_launch/helper_autostart.rs new file mode 100644 index 0000000000..7a54cb5134 --- /dev/null +++ b/apps/rocm/src/service_launch/helper_autostart.rs @@ -0,0 +1,477 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Starting the background automation helper (`rocm daemon`) when a watcher needs +//! it, with the autostart claim that keeps concurrent callers from spawning two. + +#[cfg(not(windows))] +use super::spawn::{ + attach_background_stdio, detach_background_command, managed_service_process_command, +}; +use crate::background_helper::background_helper_already_running; +#[cfg(not(windows))] +use crate::engine_runtime_select::apply_app_path_env; +use crate::engine_runtime_select::managed_service_launcher_path; +#[cfg(windows)] +use crate::engine_runtime_select::{app_path_env_var_refs, app_path_env_var_values}; +use anyhow::{Context, Result}; +use rocm_core::AppPaths; +use std::path::Path; +#[cfg(not(windows))] +use std::process::Stdio; + +/// Shared daemon-lifecycle entrypoint: ensures the background automation helper +/// (`rocm daemon`) is running when an automation watcher needs it, spawning it +/// detached if not. Liveness is read from the file-based automation runtime +/// state. Intentionally `pub(crate)` — reused by both the `serve --managed` path +/// and `automations enable`. Only the spawn result itself (`command.spawn()` / +/// `spawn_detached_no_inherit`) is logged rather than propagated; setup errors +/// (path discovery, config load, stdio attach) still return `Err`. +/// +/// The daemon's only work is running enabled automation watchers (server +/// recovery included — `server-recover` is one of them and is off by default), +/// and it exits on its own once none is enabled. So with no watcher enabled +/// nothing is spawned: a `serve` would otherwise leave a no-op daemon running +/// long after its service was stopped (ROCMAI-827). +pub(crate) fn ensure_background_helper_running() -> Result<()> { + ensure_background_helper_running_quiet(false) +} + +/// As [`ensure_background_helper_running`], but suppresses the stdout status line +/// when `quiet` is set. The interactive `rocm serve` summary path uses `quiet` so +/// the daemon-spawn note does not appear above the deployment summary table. +pub(crate) fn ensure_background_helper_running_quiet(quiet: bool) -> Result<()> { + let paths = AppPaths::discover()?; + ensure_background_helper_running_with(&paths, quiet, spawn_background_helper, &|| {}) +} + +/// The body of [`ensure_background_helper_running_quiet`], with the actual +/// process spawn injected so the decision around it is testable without +/// launching a daemon. `spawn` returns `Err` for a setup failure (propagated) +/// and `Ok(Err(..))` for a failed spawn (reported, not fatal). `on_lock_busy` is +/// called each time the autostart lock is found held by another process, which +/// lets a test wait for this call to be genuinely queued behind it. +fn ensure_background_helper_running_with( + paths: &AppPaths, + quiet: bool, + spawn: impl FnOnce(&AppPaths) -> Result>, + on_lock_busy: &dyn Fn(), +) -> Result<()> { + // An unreadable config is "no watcher": this also runs after `serve` has + // already launched its child, where failing would report a launch that + // happened as an error. + if !rocm_core::RocmCliConfig::load(paths).is_ok_and(|config| config.automation_daemon_enabled()) + { + return Ok(()); + } + + // Two concurrent callers (e.g. two `rocm serve`) can both read "not running" + // and each spawn a daemon. Serialize the decision on a lock file — the first + // holder spawns, later holders observe the now-running daemon and return + // without spawning. + // + // The "already running" read is deliberately made only under the lock, with + // no lock-free fast path. The daemon exits by itself once the last watcher + // is disabled, and it decides that under this same lock and publishes + // `running = false` before releasing it. Caller order is: save the enabling + // config, then come here. A lock-free read could still see the daemon that is + // about to exit as running, spawn nothing, and leave a watcher enabled with + // no daemon; under the lock the daemon has either seen the new config and + // stays, or has already published that it stopped. + let _autostart_lock = loop { + if let Some(lock) = + rocm_core::FileLock::try_acquire(paths.automation_autostart_lock_path())? + { + break lock; + } + on_lock_busy(); + std::thread::sleep(std::time::Duration::from_millis(10)); + }; + if background_helper_already_running(paths)? { + return Ok(()); + } + + // The lock alone does not close the window: the spawned daemon does not + // publish its `running` runtime state until well after `spawn()` (clap parse, + // runtime build, config load, banner flush). A second caller that acquires + // this lock during that gap still sees "not running" and would spawn a + // duplicate. Bridge the gap with a short-lived claim recording the child PID + // and spawn time: a holder that finds a live, recent claim defers instead. + let claim_path = paths.automation_autostart_claim_path(); + if autostart_spawn_in_flight( + read_autostart_claim(&claim_path), + now_unix_millis(), + AUTOSTART_CLAIM_TTL_MS, + rocm_core::process_is_running, + ) { + return Ok(()); + } + + match spawn(paths)? { + Ok(daemon_pid) => { + // Record the claim before returning (and thus releasing the lock) so a + // concurrent holder in the spawn→publish window defers. Best-effort: a + // failed write only reopens the original, already-tolerated race. + let _ = write_autostart_claim( + &claim_path, + AutostartClaim { + daemon_pid, + spawned_at_ms: now_unix_millis(), + }, + ); + if !quiet { + println!(" helper: started background automation daemon"); + } + } + Err(error) if !quiet => { + println!(" helper: could not start background automation daemon: {error}"); + } + Err(_) => {} + } + Ok(()) +} + +/// Spawn `rocm daemon` detached from this process. The outer `Result` is a setup +/// failure (resolving the executable, attaching stdio); the inner one is the +/// spawn itself. +fn spawn_background_helper(paths: &AppPaths) -> Result> { + let exe = managed_service_launcher_path() + .context("failed to resolve current rocm executable path")?; + let args = vec!["daemon".to_owned()]; + #[cfg(windows)] + let spawn_result = { + let env_values = app_path_env_var_values(paths, None); + let env_refs = app_path_env_var_refs(&env_values); + rocm_core::spawn_detached_no_inherit(&exe, &args, &env_refs) + }; + #[cfg(not(windows))] + let spawn_result = { + let mut command = managed_service_process_command(&exe, &args); + command.stdin(Stdio::null()); + attach_background_stdio(&mut command, None)?; + detach_background_command(&mut command); + apply_app_path_env(&mut command, paths); + command + .spawn() + .map(|child| child.id()) + .map_err(anyhow::Error::from) + }; + Ok(spawn_result) +} + +/// How long an autostart claim is honoured before it is treated as stale even if +/// its recorded PID is still alive. Comfortably longer than a cold daemon boot +/// (clap parse → runtime build → config load → state publish) yet short enough +/// that a crashed spawn cannot suppress autostart for long. +const AUTOSTART_CLAIM_TTL_MS: u128 = 30_000; + +/// A just-spawned daemon's autostart claim: the child PID and the wall-clock time +/// (milliseconds since the Unix epoch) the spawn was recorded. It lets a +/// concurrent autostart holder distinguish a live, in-flight spawn from a stale +/// leftover. Serialized as a single `" "` line — no dependency and +/// trivially forward-compatible. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct AutostartClaim { + daemon_pid: u32, + spawned_at_ms: u128, +} + +/// Milliseconds since the Unix epoch, or `0` if the clock is before the epoch +/// (which only makes a fresh claim look old — safe, it just permits a respawn). +fn now_unix_millis() -> u128 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0, |elapsed| elapsed.as_millis()) +} + +/// Read an autostart claim, returning `None` when the file is absent or +/// unparseable (either is treated as "no claim", so a respawn is permitted). +fn read_autostart_claim(path: &Path) -> Option { + let text = std::fs::read_to_string(path).ok()?; + let mut parts = text.split_whitespace(); + let daemon_pid = parts.next()?.parse().ok()?; + let spawned_at_ms = parts.next()?.parse().ok()?; + Some(AutostartClaim { + daemon_pid, + spawned_at_ms, + }) +} + +/// Write an autostart claim as `" "`. Best-effort at the call site. +fn write_autostart_claim(path: &Path, claim: AutostartClaim) -> std::io::Result<()> { + std::fs::write( + path, + format!("{} {}", claim.daemon_pid, claim.spawned_at_ms), + ) +} + +/// Whether an existing autostart `claim` means a daemon spawn is still in flight, +/// so the current lock holder should defer rather than spawn a duplicate. A claim +/// counts as in-flight only while its child PID is alive *and* it is younger than +/// `ttl_ms` — the TTL bounds how long a crashed spawn (or a PID later reused by an +/// unrelated process) can suppress autostart. `pid_alive` is injected so the +/// decision is unit-testable without a live process. +fn autostart_spawn_in_flight( + claim: Option, + now_ms: u128, + ttl_ms: u128, + pid_alive: impl Fn(u32) -> bool, +) -> bool { + claim.is_some_and(|claim| { + now_ms.saturating_sub(claim.spawned_at_ms) < ttl_ms && pid_alive(claim.daemon_pid) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::{runtime_state, set_helper_facts}; + use crate::tests::test_paths; + use std::fs; + + /// ROCMAI-827: `rocm serve` started the background daemon on every launch, + /// and with no automation watcher enabled that daemon had nothing to do yet + /// kept running after every service was stopped. With the default config + /// (no watcher enabled) the helper must not be spawned at all. + #[test] + fn background_helper_not_spawned_when_no_automation_is_enabled() { + let (root, paths) = test_paths("helper-no-automation"); + paths.ensure().expect("create app dirs"); + // The stored flag alone is not "a watcher is enabled": nothing would run. + let mut config = rocm_core::RocmCliConfig::default(); + config.automations.daemon_enabled = true; + config.save(&paths).expect("save default config"); + + let mut spawned = false; + ensure_background_helper_running_with( + &paths, + true, + |_| { + spawned = true; + Ok(Ok(4242)) + }, + &|| {}, + ) + .expect("ensure helper"); + + let claim_written = paths.automation_autostart_claim_path().exists(); + let _ = fs::remove_dir_all(&root); + assert!( + !spawned, + "no watcher enabled ⇒ the daemon has no work and must not be started" + ); + assert!(!claim_written, "no spawn ⇒ no autostart claim"); + } + + /// The other half of the gate: once a watcher is enabled the helper is still + /// started (exactly once) and its autostart claim recorded. + #[test] + fn background_helper_spawned_when_an_automation_is_enabled() { + let (root, paths) = test_paths("helper-automation-enabled"); + paths.ensure().expect("create app dirs"); + let mut config = rocm_core::RocmCliConfig::default(); + config.watcher_config_mut("server-recover").enabled = true; + config.save(&paths).expect("save config"); + + let mut spawns = 0; + ensure_background_helper_running_with( + &paths, + true, + |_| { + spawns += 1; + Ok(Ok(4242)) + }, + &|| {}, + ) + .expect("ensure helper"); + + let claim = read_autostart_claim(&paths.automation_autostart_claim_path()); + let _ = fs::remove_dir_all(&root); + assert_eq!(spawns, 1, "an enabled watcher needs the daemon"); + assert_eq!(claim.map(|claim| claim.daemon_pid), Some(4242)); + } + + /// ROCMAI-827 race, enable side: the daemon that is about to exit still has + /// `running = true` and a live pid in its state when `automations enable` + /// (config already saved) asks for a helper. It decides under the autostart + /// lock and publishes "stopped" before releasing it, so the enable must wait + /// on that lock and then start a new helper. A lock-free "already running" + /// read would return here without spawning, leaving a watcher with no daemon. + #[test] + fn background_helper_is_started_when_the_exiting_daemon_publishes_stopped() { + let (root, paths) = test_paths("helper-exit-window"); + paths.ensure().expect("create app dirs"); + let mut config = rocm_core::RocmCliConfig::default(); + config.watcher_config_mut("server-recover").enabled = true; + config.save(&paths).expect("save config"); + // The old daemon: alive (this process stands in) and still `running`. + runtime_state(true, std::process::id()) + .write(&paths) + .expect("write runtime state"); + + // The exiting daemon holds the autostart lock while it decides. + let exiting_daemon_lock = + rocm_core::FileLock::acquire(paths.automation_autostart_lock_path()) + .expect("hold the lock"); + let enable_paths = paths.clone(); + let queued_flag = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let queued = queued_flag.clone(); + let enable = std::thread::spawn(move || { + let mut spawns = 0; + ensure_background_helper_running_with( + &enable_paths, + true, + |_| { + spawns += 1; + Ok(Ok(4242)) + }, + &|| queued.store(true, std::sync::atomic::Ordering::SeqCst), + ) + .expect("ensure helper"); + spawns + }); + // Wait until the enable path has actually found the lock held and queued + // behind it, rather than assuming how long that takes. + let deadline = std::time::Instant::now() + std::time::Duration::from_mins(1); + // A caller that skipped the lock finishes without ever queuing. + while !queued_flag.load(std::sync::atomic::Ordering::SeqCst) && !enable.is_finished() { + assert!(std::time::Instant::now() < deadline, "enable never queued"); + std::thread::yield_now(); + } + runtime_state(false, std::process::id()) + .write(&paths) + .expect("daemon publishes stopped"); + drop(exiting_daemon_lock); + + let spawns = enable.join().expect("enable thread"); + let _ = fs::remove_dir_all(&root); + assert_eq!( + spawns, 1, + "the enable must start a helper once the old one has published it stopped" + ); + } + + fn stop_result() -> serde_json::Value { + serde_json::json!({"output": {"result": { + "service": {"service_id": "svc-a", "status": "stopped"}, + "signaled_pids": [1] + }}}) + } + + /// `rocm services stop` prints the note for a stop only, and only the variant + /// that matches the state. + #[test] + fn stop_output_carries_the_helper_note_that_matches_the_state() { + use crate::services_text::render_service_action_result_with_helper_note as render; + + let (root, paths) = test_paths("stop-helper-note"); + paths.ensure().expect("create app dirs"); + set_helper_facts(&paths, false, false); + let quiet = render(&paths, "stop_server", &stop_result()); + set_helper_facts(&paths, true, true); + let outlives = render(&paths, "stop_server", &stop_result()); + let restart = render(&paths, "restart_server", &stop_result()); + set_helper_facts(&paths, true, false); + let not_running = render(&paths, "stop_server", &stop_result()); + let _ = fs::remove_dir_all(&root); + + assert!(!quiet.contains("background automation helper"), "{quiet}"); + assert!( + outlives.contains("stopped processes: 1") + && outlives.contains("keeps running while an automation is enabled") + && outlives.contains("`rocm automations disable server-recover`"), + "{outlives}" + ); + assert!( + not_running.contains("is not running") && !not_running.contains("keeps running"), + "{not_running}" + ); + assert!( + !restart.contains("background automation helper"), + "{restart}" + ); + } + + /// `serve` calls the helper autostart after its child is already running. An + /// unreadable config must not turn that launch into an error: it reads as "no + /// watcher enabled" and nothing is spawned. + #[test] + fn background_helper_autostart_treats_an_unreadable_config_as_no_watcher() { + let (root, paths) = test_paths("helper-bad-config"); + paths.ensure().expect("create app dirs"); + fs::write(paths.config_path(), b"{ this is not json").expect("write bad config"); + assert!(rocm_core::RocmCliConfig::load(&paths).is_err()); + + let mut spawned = false; + let result = ensure_background_helper_running_with( + &paths, + true, + |_| { + spawned = true; + Ok(Ok(4242)) + }, + &|| {}, + ); + let _ = fs::remove_dir_all(&root); + + assert!(result.is_ok(), "{result:?}"); + assert!(!spawned); + } + + #[test] + fn autostart_spawn_in_flight_defers_only_for_a_live_recent_claim() { + let claim = AutostartClaim { + daemon_pid: 4242, + spawned_at_ms: 1_000, + }; + // A recent claim whose PID is still alive ⇒ a spawn is in flight, defer. + assert!(autostart_spawn_in_flight( + Some(claim), + 1_000 + 5_000, + AUTOSTART_CLAIM_TTL_MS, + |_| true, + )); + // Same claim but the child PID is gone (crashed spawn) ⇒ respawn. + assert!(!autostart_spawn_in_flight( + Some(claim), + 1_000 + 5_000, + AUTOSTART_CLAIM_TTL_MS, + |_| false, + )); + // Older than the TTL, even with a live PID (possible PID reuse) ⇒ respawn. + assert!(!autostart_spawn_in_flight( + Some(claim), + 1_000 + AUTOSTART_CLAIM_TTL_MS, + AUTOSTART_CLAIM_TTL_MS, + |_| true, + )); + // No claim at all ⇒ nothing in flight, spawn. + assert!(!autostart_spawn_in_flight( + None, + 1_000, + AUTOSTART_CLAIM_TTL_MS, + |_| true, + )); + } + + #[test] + fn autostart_claim_round_trips_and_absent_or_garbage_reads_as_none() { + let (root, paths) = test_paths("autostart-claim-roundtrip"); + let path = paths.automation_autostart_claim_path(); + fs::create_dir_all(path.parent().expect("claim parent")).expect("mkdir claim dir"); + // Absent file ⇒ no claim. + assert_eq!(read_autostart_claim(&path), None); + // Round-trip a written claim. + let claim = AutostartClaim { + daemon_pid: 9987, + spawned_at_ms: 1_726_000_000_123, + }; + write_autostart_claim(&path, claim).expect("write claim"); + assert_eq!(read_autostart_claim(&path), Some(claim)); + // Garbage / partial content ⇒ no claim (permits a respawn, never panics). + fs::write(&path, "not-a-pid").expect("write garbage"); + assert_eq!(read_autostart_claim(&path), None); + let _ = fs::remove_dir_all(&root); + } +} diff --git a/apps/rocm/src/service_launch/launch_transaction.rs b/apps/rocm/src/service_launch/launch_transaction.rs new file mode 100644 index 0000000000..13fe90ed1b --- /dev/null +++ b/apps/rocm/src/service_launch/launch_transaction.rs @@ -0,0 +1,1002 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The locked launch transaction around a managed service: the port allocation +//! decision against live records and the OS, the cleanup of a failed launch +//! attempt, and the reuse refusals (including the no-usable-GPU wording) that +//! decide whether a live service may answer a `serve` request. + +use crate::endpoint_keys; +use crate::managed_service_status::{ + existing_live_managed_service_in, load_managed_services, managed_service_is_live, +}; +use crate::serve_port; +use crate::{device_policy_name, record_cli_audit_event}; +use anyhow::Result; +use rocm_core::{AppPaths, ManagedServiceRecord}; +use rocm_engine_protocol::DevicePolicy; +use std::fs; +use std::time::Duration; + +/// Outcome of the locked allocation step. +/// +/// `Debug` so tests can assert on `Result::expect_err`. +#[derive(Debug)] +pub(super) enum ManagedPortDecision { + /// An equivalent managed service is already live; nothing may be spawned. + /// Boxed to keep the variants a similar size (clippy::large_enum_variant). + AlreadyLive(Box), + /// The concrete port this launch owns until it hands it to the child. + Lease(rocm_core::LoopbackPortLease), +} + +/// Live managed services, with liveness refreshed by [`load_managed_services`] +/// (dead PIDs demote to "stopped" and therefore neither satisfy the duplicate +/// check nor reserve a port). +/// +/// A read failure degrades to "nothing is live": for an already-`ready` +/// service the OS preflight still guards its port (the engine holds it), and +/// refusing to serve because the services directory could not be listed would +/// be a worse failure than launching. That guard does not cover a `starting` +/// record: its port is released back to the OS in +/// `launch_managed_engine_child_locked` (see `lease.release()`), right before the +/// spawn, and stays unbound until the new engine child has actually bound it, so a concurrent launch that hits this +/// read failure in that narrow window can probe the same port successfully. +/// Accepted as a known gap — narrower than the general services-dir read +/// failure case, and only reachable by a second, near-simultaneous I/O +/// failure on top of one already in flight. +pub(super) fn live_managed_services(paths: &AppPaths) -> Vec { + load_managed_services(paths) + .unwrap_or_default() + .into_iter() + .filter(managed_service_is_live) + .collect() +} + +/// Ports that live managed services still own. +/// +/// Conservative by design: on the default loopback host a record reserves its +/// port regardless of the host it recorded, because a `0.0.0.0` (or `localhost`) +/// listener also blocks the loopback address a new local server would bind. +/// Over-reserving costs one candidate in a 101-port scan; under-reserving costs +/// a collision. On a custom host the recorded host is compared (see +/// `serve_port::hosts_overlap`). +fn reserved_service_ports(live: &[ManagedServiceRecord]) -> Vec { + live.iter() + .map(|record| serve_port::ReservedPort { + port: record.port, + host: record.host.clone(), + service_id: record.service_id.clone(), + status: record.status.clone(), + }) + .collect() +} + +/// The decision half of the launch transaction: refresh liveness, apply the +/// `(engine, canonical_model_id)` idempotency guard, then lease a concrete port +/// against both live records and the real OS. +/// +/// Callers MUST hold [`rocm_core::AppPaths::managed_launch_lock_path`] across +/// this call *and* the record publication that follows, or two launches can +/// pick the same candidate. +/// `probe` is injected so tests can drive deterministic bind outcomes without a +/// GPU or a spawn. +pub(super) fn resolve_managed_port_in_transaction( + paths: &AppPaths, + engine: &str, + canonical_model_id: &str, + host: &str, + request: serve_port::PortRequest, + probe: &mut dyn FnMut(u16) -> std::io::Result, +) -> Result { + let live = live_managed_services(paths); + // Idempotency first: an already-live equivalent service means no port is + // needed at all, so it must be answered before any candidate is leased. + if let Some(existing) = existing_live_managed_service_in(&live, engine, canonical_model_id) { + return Ok(ManagedPortDecision::AlreadyLive(Box::new(existing.clone()))); + } + let reserved = reserved_service_ports(&live); + serve_port::resolve_serve_port(host, request, &reserved, probe).map(ManagedPortDecision::Lease) +} + +/// Grace given to a child that must be terminated because its record could not +/// be published. Short: the child is milliseconds old and has nothing to flush. +const MANAGED_LAUNCH_CLEANUP_GRACE: Duration = Duration::from_secs(2); + +/// Undo a failed managed launch attempt before the launch lock is released, +/// so no failed transaction leaves a live reservation or a stray endpoint key. +/// +/// Order matters: a spawned child is terminated *first*, so it is not still +/// holding the port or writing to the log by the time cleanup runs. The +/// termination outcome is audited but not acted on: the record is marked +/// abandoned (`stopped`, releasing the port) even when death was not confirmed. If no child was ever spawned (`record.engine_pid` is still `None`), +/// every artifact this attempt created is removed: there is nothing for +/// `rocm services`/`rocm logs` to show. If a child did run, its record and log +/// are kept instead (marked `status = "stopped"`, see +/// [`ManagedServiceRecord::mark_launch_abandoned`]), since the error that led here +/// already told the user to go inspect that log; only the endpoint key, which a +/// failed attempt has no use for, is cleared. +/// Every step is best-effort and idempotent; cleanup must never mask the +/// original failure with a second one. +pub(super) fn abandon_failed_managed_launch(paths: &AppPaths, record: &ManagedServiceRecord) { + if let Some(pid) = record.engine_pid.filter(|pid| *pid != 0) { + // The token captured at spawn, not a fresh `capture`: when the child + // already exited, a fresh one would describe a recycled pid's new owner + // and the forced tree kill would hit it. Where start tokens exist + // (Linux), a missing one means the child was gone before it could be + // captured, so the pid is unverifiable and nothing is signalled + // (`terminate_verified` alone would "best-effort proceed"). Windows + // records no token at all, so there the kill proceeds as before. + let outcome = if record.engine_start_ticks.is_none() && cfg!(target_os = "linux") { + rocm_core::TerminationOutcome::Unverified + } else { + rocm_core::terminate_verified( + &rocm_core::ProcessIdentity::new(pid, record.engine_start_ticks), + rocm_core::KillScope::Tree, + MANAGED_LAUNCH_CLEANUP_GRACE, + true, + ) + }; + record_cli_audit_event( + paths, + "service", + "managed_service_launch_cleanup", + "warn", + format!( + "terminated engine child pid={pid} after a failed launch publication outcome={}", + outcome.as_str() + ), + Some(&record.service_id), + ); + let mut abandoned = record.clone(); + abandoned.mark_launch_abandoned(); + let _ = abandoned.write(); + endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); + return; + } + for path in [ + record.manifest_path.as_path(), + record.log_path.as_path(), + record.engine_state_path.as_path(), + ] { + let _ = fs::remove_file(path); + } + endpoint_keys::clear_endpoint_api_key(paths, &record.service_id); +} + +/// The ways a live managed service that already matches on liveness, engine and +/// canonical model id can still be unusable by this invocation. +/// +/// Those three terms are necessary but not sufficient: three further terms can +/// still refuse, one variant each. [`managed_service_reuse_refusal`] is the +/// single decision procedure both callers ask, so the two cannot answer +/// differently — which they previously did; see that function for what the +/// disagreement cost. +pub(crate) enum ManagedReuseRefusal { + /// The live service was started with different serve options. + RecipeMismatch, + /// `--require-api-key` was asked for and the live service has no key. + /// + /// The engine reads its key once, at launch, from the environment the spawn + /// builds — so a server started without one keeps serving anonymously no + /// matter what is written afterwards. Upgrading the record would be worse + /// than doing nothing: it would claim auth the live process does not + /// enforce, and `ensure_public_service_has_endpoint_key` would pass on the + /// strength of a key file nothing reads. `rocm remote serve` relies on this: + /// it publishes a loopback port onto the tailnet and prints "the API key + /// above is what stops anyone else calling it", which reusing an + /// unauthenticated service silently would make false about an endpoint the + /// whole tailnet can reach. Refusing is the only answer that fails closed. + MissingApiKey, + /// The live service runs under a device policy that does not satisfy this + /// invocation's. Reusing a `cpu_only` service to answer a `gpu_required` + /// serve is the silent CPU fallback AGENTS.md section 6 forbids. + DevicePolicyTooWeak, +} + +/// Why this invocation cannot reuse `existing` — or `None` when it genuinely +/// can. See [`ManagedReuseRefusal`] for the terms this answers on. +/// +/// This exists because `serve()`'s reuse pre-gate lets a matching live service +/// skip the no-usable-AMD-GPU refusal. Deciding that bypass on a *weaker* test +/// than the spawn then uses to accept reuse produced a dead end: on a GPU-less +/// host (`HIP_VISIBLE_DEVICES=` with a server live from another shell) a serve +/// with different options bypassed the refusal, reached the spawn, and was told +/// to stop the server and run the command again — and the rerun, with nothing +/// left to reuse, hit the no-GPU refusal. The advice named a command that could +/// not clear the condition, which AGENTS.md section 3 forbids. Sharing the +/// predicate means a service that cannot actually be reused never buys the +/// bypass, so the user gets the accurate refusal first and keeps their server. +/// +/// `requested_device_policy` is taken by value, not as an `Option`, on purpose. +/// It was optional so the spawn could opt out of the device-policy term, and +/// that left two holes at once: the spawn's `DevicePolicyTooWeak` arm was +/// unreachable (an untestable bail), and `serve()` — the caller that *does* +/// gate the GPU refusal on this answer — could silently drop the term by +/// passing `None`, with the whole suite still green. Both callers have a +/// `DevicePolicy` in hand, so requiring it costs nothing and makes that +/// mutation a compile error. It widens nothing observable today either: the +/// only policy a record can carry is the one `parse_device_policy` admits +/// (see below), so `existing.device_policy == Some("cpu_only")` is not +/// reachable from any `rocm serve` this CLI can run. +/// The `rocm serve` no-usable-GPU refusal, worded once. +/// +/// Two places raise it and they must say the same thing: `serve()`'s fail-fast +/// pre-flight, and [`spawn_managed_engine_child`](super::spawn::spawn_managed_engine_child) +/// when the reuse that waived that pre-flight turns out not to be available after +/// all. +pub(crate) fn no_usable_gpu_refusal(device_policy: &DevicePolicy) -> String { + // Keyed on the same `is_wsl_host()` the usable-GPU probe branches on, so the + // advice follows the platform whose answer produced the refusal. + no_usable_gpu_refusal_text(device_policy_name(device_policy), rocm_core::is_wsl_host()) +} + +/// The refusal text for `policy`, split on whether the probe that refused was +/// the WSL one. +/// +/// WSL reaches the GPU through `/dev/dxg` and ROCDXG, and has no `/dev/kfd`, so +/// telling a WSL user to confirm that device is present names something that can +/// never clear the condition. There the advice points at the status `rocm +/// examine` reports for the path the probe actually asked about +/// ([`rocm_core::WSL_ROCDXG_READY`]), and at `rocm diagnose`, which names what is +/// missing. +fn no_usable_gpu_refusal_text(policy: &str, on_wsl: bool) -> String { + let remedy = if on_wsl { + format!( + "WSL reaches the GPU through /dev/dxg and ROCDXG rather than /dev/kfd: run \ + `rocm examine` and check that `driver_status` is `{ready}` (`rocm diagnose` \ + names what is missing; a ROCDXG library the loader reaches only through \ + LD_LIBRARY_PATH needs that variable exported in this shell, see docs/wsl.md), and ensure HIP_VISIBLE_DEVICES / ROCR_VISIBLE_DEVICES \ + are not masking every device.", + ready = rocm_core::WSL_ROCDXG_READY + ) + } else { + "Check the driver with `rocm examine`, confirm /dev/kfd is present, and ensure \ + HIP_VISIBLE_DEVICES / ROCR_VISIBLE_DEVICES are not masking every device." + .to_owned() + }; + format!( + "no usable AMD GPU detected; `rocm serve` requires a GPU under the {policy} policy \ + and does not fall back to CPU. {remedy}" + ) +} + +pub(crate) fn managed_service_reuse_refusal( + existing: &ManagedServiceRecord, + requested_recipe_json: Option<&str>, + require_api_key: bool, + requested_device_policy: &DevicePolicy, +) -> Option { + if existing.engine_recipe_json.as_deref() != requested_recipe_json { + return Some(ManagedReuseRefusal::RecipeMismatch); + } + if require_api_key && !existing.requires_api_key { + return Some(ManagedReuseRefusal::MissingApiKey); + } + // Only `gpu_required` can be let down by the record's policy, and the + // enumeration is exhaustive rather than narrow: `parse_device_policy` + // (`main.rs`) maps `auto`, `gpu`, `gpu_required` *and* `gpu_preferred` onto + // `GpuRequired`, and rejects `cpu_only` with an error, so `GpuRequired` is + // the only policy `serve()` can hand this predicate — pinned by + // `parse_device_policy_defaults_to_gpu_required_without_cpu_fallback`. There + // is deliberately no wider match on "anything but `CpuOnly`": it would be + // identical at runtime and would read as a guard against a `gpu_preferred` + // bypass that no CLI invocation can reach. A `cpu_only` invocation would be + // satisfied by anything anyway and never reaches the bail. README.md's + // "that is the only policy pair refused" is the contract, and this matches + // it exactly. + if matches!(requested_device_policy, DevicePolicy::GpuRequired) + && existing.device_policy.as_deref() == Some("cpu_only") + { + return Some(ManagedReuseRefusal::DevicePolicyTooWeak); + } + None +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::service_hints; + #[cfg(not(windows))] + use crate::service_launch::test_support::sleeper_test_name; + use crate::service_launch::test_support::{reusable_record, write_live_service_record}; + use crate::tests::test_paths; + use rocm_core::DEFAULT_LOCAL_HOST; + #[cfg(not(windows))] + use std::process::Command as ProcessCommand; + use std::thread; + + #[test] + fn the_no_gpu_refusal_sends_a_bare_metal_user_to_dev_kfd() { + let text = super::no_usable_gpu_refusal_text("gpu_required", false); + assert!(text.contains("confirm /dev/kfd is present"), "{text}"); + assert!(!text.contains("ROCDXG"), "{text}"); + } + + #[test] + fn the_no_gpu_refusal_does_not_send_a_wsl_user_to_dev_kfd() { + // WSL has no /dev/kfd, so advice to confirm it can never clear the + // condition. The text names the status `examine` reports for the path + // the probe asked about, taken from the constant the producer uses, so + // the two cannot drift. That the status really is the one a refused + // host shows is asserted end to end by serve-42. + let text = super::no_usable_gpu_refusal_text("gpu_required", true); + assert!(!text.contains("/dev/kfd is present"), "{text}"); + assert!(text.contains("rocm examine"), "{text}"); + assert!(text.contains("rocm diagnose"), "{text}"); + // The advice is true only because `rocdxg_ready` counts a library on + // LD_LIBRARY_PATH; `a_rocdxg_only_on_the_loader_path_is_ready` in + // `rocm-core`'s host_gpu/wsl.rs proves that behavior. + assert!(text.contains("LD_LIBRARY_PATH"), "{text}"); + assert!( + text.contains(&format!("`{}`", rocm_core::WSL_ROCDXG_READY)), + "{text}" + ); + } + + #[test] + fn the_no_gpu_refusal_names_the_policy_on_both_platforms() { + for on_wsl in [false, true] { + let text = super::no_usable_gpu_refusal_text("gpu_required", on_wsl); + assert!( + text.starts_with("no usable AMD GPU detected; `rocm serve` requires a GPU under the gpu_required policy"), + "{text}" + ); + } + } + + #[test] + fn reuse_is_refused_on_every_term_the_spawn_and_the_gpu_bail_depend_on() { + let (root, paths) = test_paths("managed-reuse-predicate"); + paths.ensure().expect("paths"); + let base = reusable_record(&paths); + + // The baseline must be reusable, or the negative cases below could pass + // for the wrong reason. + assert!( + managed_service_reuse_refusal(&base, None, true, &DevicePolicy::GpuRequired).is_none(), + "a matching service with no recipe, a key, and a gpu_required policy is reusable" + ); + + // Recipe mismatch — the spawn's first refusal. + let mut mismatched = base.clone(); + mismatched.engine_recipe_json = Some("{\"engine\":\"vllm\"}".to_owned()); + assert!(matches!( + managed_service_reuse_refusal(&mismatched, None, true, &DevicePolicy::GpuRequired), + Some(ManagedReuseRefusal::RecipeMismatch) + )); + + // Unmet `--require-api-key` — the spawn's second refusal. Also pins that + // it is *only* refused when the key is actually demanded. + let mut anonymous = base.clone(); + anonymous.requires_api_key = false; + assert!(matches!( + managed_service_reuse_refusal(&anonymous, None, true, &DevicePolicy::GpuRequired), + Some(ManagedReuseRefusal::MissingApiKey) + )); + assert!( + managed_service_reuse_refusal(&anonymous, None, false, &DevicePolicy::GpuRequired) + .is_none(), + "an anonymous service is fine when no key was demanded" + ); + + // Device policy — the term `serve()` depends on, because it is the one + // gating the no-usable-GPU bail on the answer. Reusing a `cpu_only` + // service to answer a `gpu_required` serve is the silent CPU fallback. + let mut cpu_only = base; + cpu_only.device_policy = Some("cpu_only".to_owned()); + assert!(matches!( + managed_service_reuse_refusal(&cpu_only, None, true, &DevicePolicy::GpuRequired), + Some(ManagedReuseRefusal::DevicePolicyTooWeak) + )); + // `GpuRequired` is the only policy the CLI can produce — `--device + // gpu_preferred`/`auto`/`gpu` all parse to it and `cpu_only` is rejected + // (`parse_device_policy_defaults_to_gpu_required_without_cpu_fallback`) — + // so there is no second policy arm to discriminate here, and asserting + // one would pin an input `serve()` never produces. Both callers pass + // the same policy; the parameter is not optional, so neither can drop + // this term without a compile error. + // + // A cpu_only *invocation* never reaches the bail, so nothing to refuse. + assert!( + managed_service_reuse_refusal(&cpu_only, None, true, &DevicePolicy::CpuOnly).is_none() + ); + + let _ = std::fs::remove_dir_all(&root); + } + + #[test] + fn duplicate_managed_launch_detected_across_distinct_service_ids() -> Result<()> { + // `generate_service_id` embeds a timestamp, so a second launch for the + // same engine+model has a DIFFERENT service_id. The guard must still + // detect the live service by (engine, canonical_model_id), and return + // the newest live match. `starting` skips the endpoint probe; the + // current process id is a guaranteed-live PID. + let (root, paths) = test_paths("dup-managed-distinct-ids"); + paths.ensure()?; + + // Older, dead manifest for the same engine+model (distinct service_id). + let mut dead = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-1000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11500, + "managed", + 999_999_999, + None, + None, + None, + ); + dead.status = "ready".to_owned(); + dead.engine_pid = Some(999_999_999); + dead.created_at_unix_ms = 1000; + dead.write()?; + + // Newer, live manifest for the same engine+model (distinct service_id). + let mut live = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-2000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11501, + "managed", + std::process::id(), + None, + None, + None, + ); + live.status = "starting".to_owned(); + live.engine_pid = Some(std::process::id()); + live.created_at_unix_ms = 2000; + live.write()?; + + let live_services = live_managed_services(&paths); + let found = + existing_live_managed_service_in(&live_services, "lemonade", "qwen-canonical").cloned(); + let _ = fs::remove_dir_all(root); + + let found = found.expect("a live managed service should be detected by engine+model"); + assert_eq!( + found.service_id, "lemonade-qwen-2000", + "should return the newest live match" + ); + assert!(managed_service_is_live(&found)); + Ok(()) + } + + #[test] + fn dead_managed_service_allows_relaunch() -> Result<()> { + // A stale manifest with dead PIDs must NOT block a relaunch: liveness + // refresh demotes it to "stopped", so the guard returns None. + let (root, paths) = test_paths("dup-managed-dead"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-3000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11502, + "managed", + 999_999_999, + None, + None, + None, + ); + record.status = "ready".to_owned(); + record.engine_pid = Some(999_999_999); + record.write()?; + + let live = live_managed_services(&paths); + let found = existing_live_managed_service_in(&live, "lemonade", "qwen-canonical"); + let _ = fs::remove_dir_all(root); + + assert!( + found.is_none(), + "a dead managed service must not block relaunch" + ); + Ok(()) + } + + #[test] + fn live_service_for_other_model_does_not_block() -> Result<()> { + // A live service for a DIFFERENT model must not match — the guard keys + // on the model, not just the engine. + let (root, paths) = test_paths("dup-managed-other-model"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-other-1", + "lemonade", + "other", + "other-canonical", + "127.0.0.1", + 11503, + "managed", + std::process::id(), + None, + None, + None, + ); + record.status = "starting".to_owned(); + record.engine_pid = Some(std::process::id()); + record.write()?; + + let live = live_managed_services(&paths); + let found = existing_live_managed_service_in(&live, "lemonade", "qwen-canonical"); + let _ = fs::remove_dir_all(root); + + assert!( + found.is_none(), + "a live service for a different model must not match" + ); + Ok(()) + } + + #[test] + fn missing_manifest_allows_launch() { + // No services dir / manifests → nothing to detect, launch proceeds. + let (root, paths) = test_paths("dup-managed-missing"); + let live = live_managed_services(&paths); + let found = existing_live_managed_service_in(&live, "lemonade", "qwen-canonical"); + let _ = fs::remove_dir_all(root); + assert!(found.is_none()); + } + + fn never_probe(_port: u16) -> std::io::Result { + panic!("no port may be probed on this path") + } + + fn always_free(port: u16) -> std::io::Result { + Ok(rocm_core::LoopbackPortLease::engine_owned(port)) + } + + #[test] + fn allocation_transaction_answers_an_equivalent_live_service_before_leasing() -> Result<()> { + let (root, paths) = test_paths("alloc-txn-already-live"); + paths.ensure()?; + let existing = write_live_service_record( + &paths, + "lemonade-qwen-9000", + "qwen-canonical", + 11501, + "starting", + )?; + + let decision = resolve_managed_port_in_transaction( + &paths, + "lemonade", + "qwen-canonical", + "127.0.0.1", + serve_port::PortRequest::Auto, + &mut never_probe, + )?; + let _ = fs::remove_dir_all(root); + + match decision { + ManagedPortDecision::AlreadyLive(record) => { + assert_eq!(record.service_id, existing.service_id); + assert_eq!(record.port, existing.port); + } + ManagedPortDecision::Lease(_) => { + panic!("an equivalent live service must be answered, not leased a new port") + } + } + Ok(()) + } + + #[test] + fn allocation_transaction_skips_ports_reserved_by_other_live_services() -> Result<()> { + let (root, paths) = test_paths("alloc-txn-skips-reserved"); + paths.ensure()?; + write_live_service_record(&paths, "lemonade-a", "model-a", 11_435, "running")?; + write_live_service_record(&paths, "lemonade-b", "model-b", 11_436, "recovering")?; + + let decision = resolve_managed_port_in_transaction( + &paths, + "lemonade", + "model-c", + "127.0.0.1", + serve_port::PortRequest::Auto, + &mut always_free, + )?; + let _ = fs::remove_dir_all(root); + + match decision { + ManagedPortDecision::Lease(lease) => assert_eq!(lease.port(), 11_437), + ManagedPortDecision::AlreadyLive(record) => { + panic!("model-c has no live service, expected a lease, got {record:?}") + } + } + Ok(()) + } + + #[test] + fn allocation_transaction_ignores_stopped_records() -> Result<()> { + let (root, paths) = test_paths("alloc-txn-ignores-stopped"); + paths.ensure()?; + let mut stale = + write_live_service_record(&paths, "lemonade-stale", "model-stale", 11_435, "running")?; + // A dead pid demotes the record to stopped when liveness is refreshed, so + // it must not reserve its port. + stale.engine_pid = Some(999_999_999); + stale.supervisor_pid = 999_999_999; + stale.write()?; + + let decision = resolve_managed_port_in_transaction( + &paths, + "lemonade", + "model-fresh", + "127.0.0.1", + serve_port::PortRequest::Auto, + &mut always_free, + )?; + let _ = fs::remove_dir_all(root); + + match decision { + ManagedPortDecision::Lease(lease) => assert_eq!(lease.port(), 11_435), + ManagedPortDecision::AlreadyLive(record) => { + panic!("a stopped record must not block a fresh launch, got {record:?}") + } + } + Ok(()) + } + + #[test] + fn stopping_the_reserving_record_frees_its_explicit_port_for_reuse() -> Result<()> { + // Proves the mechanism `rocm services stop` relies on: it never touches + // the port directly, it only flips the stopped record's `status` (see + // the `all_stopped` branch in `main.rs`'s stop handler). This recreates + // that exact disk state and checks the same explicit port a live + // competitor refused now resolves, closing the loop the E2E + // `assert_port_reserved_message` step only checks by wording. + let (root, paths) = test_paths("alloc-explicit-port-freed-by-stop"); + paths.ensure()?; + let mut holder = ManagedServiceRecord::new( + &paths, + "lemonade-a-1", + "lemonade", + "model-a", + "model-a".to_owned(), + DEFAULT_LOCAL_HOST, + 11_435, + "managed", + std::process::id(), + None, + None, + None, + ); + holder.status = "running".to_owned(); + holder.engine_pid = Some(std::process::id()); + holder.write()?; + + let still_reserved = resolve_managed_port_in_transaction( + &paths, + "lemonade", + "model-b", + DEFAULT_LOCAL_HOST, + serve_port::PortRequest::Explicit(11_435), + &mut always_free, + ); + // The refusal names the service AND the command that frees the port; + // the stop below is that command's own implementation, and the grant + // after it is the outcome, so advice and result are pinned together. + let stop_hint = service_hints::service_stop_hint("lemonade-a-1"); + assert!( + matches!(still_reserved, Err(ref error) if error.to_string().contains("already reserved by managed service lemonade-a-1") + && error.to_string().contains(&stop_hint)), + "unexpected result while still running: {still_reserved:?}" + ); + + // Go through the real stop path (`rocm services stop --yes`'s own + // implementation) rather than writing `status = "stopped"` to disk + // directly, so this test actually proves the advice serve-45 pins: that + // running the stop command frees the port. The termination closure is + // the same seam production injects a real signal through; here it just + // confirms every recorded process gone without touching the test + // process itself. + crate::service_control::stop_internal_managed_service_with(&paths, "lemonade-a-1", |_| { + crate::service_control::RecordedStopTermination::confirmed_stopped() + })?; + + let decision = resolve_managed_port_in_transaction( + &paths, + "lemonade", + "model-b", + DEFAULT_LOCAL_HOST, + serve_port::PortRequest::Explicit(11_435), + &mut always_free, + ); + let _ = fs::remove_dir_all(root); + + match decision? { + ManagedPortDecision::Lease(lease) => assert_eq!(lease.port(), 11_435), + ManagedPortDecision::AlreadyLive(existing) => { + panic!( + "a stopped record must not block reuse: {}", + existing.service_id + ) + } + } + Ok(()) + } + + #[test] + fn allocation_transaction_rejects_an_explicit_port_a_recovering_service_holds() -> Result<()> { + let (root, paths) = test_paths("alloc-txn-rejects-explicit-collision"); + paths.ensure()?; + let existing = write_live_service_record( + &paths, + "lemonade-recovering", + "model-recovering", + 11_435, + "recovering", + )?; + + let result = resolve_managed_port_in_transaction( + &paths, + "lemonade", + "model-other", + "127.0.0.1", + serve_port::PortRequest::Explicit(11_435), + &mut never_probe, + ); + let _ = fs::remove_dir_all(root); + + let Err(error) = result else { + panic!("an explicit port a recovering service holds must be refused") + }; + let message = error.to_string(); + assert!( + message.contains(&existing.service_id), + "the refusal must name the service holding the port: {message}" + ); + assert!( + message.contains("recovering"), + "the refusal must say why the port is unavailable: {message}" + ); + Ok(()) + } + + #[test] + fn concurrent_auto_transactions_publish_distinct_ports() -> Result<()> { + let (root, paths) = test_paths("alloc-txn-concurrent-distinct"); + paths.ensure()?; + let paths = std::sync::Arc::new(paths); + + let spawn_one = |model: &'static str| { + let paths = std::sync::Arc::clone(&paths); + thread::spawn(move || -> Result { + let _launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + let lease = match resolve_managed_port_in_transaction( + &paths, + "lemonade", + model, + "127.0.0.1", + serve_port::PortRequest::Auto, + &mut serve_port::loopback_bind_probe, + )? { + ManagedPortDecision::Lease(lease) => lease, + ManagedPortDecision::AlreadyLive(record) => { + panic!("fresh model must not already be live: {record:?}") + } + }; + let port = lease.port(); + write_live_service_record(&paths, model, model, port, "running")?; + lease.release(); + Ok(port) + }) + }; + + let first = spawn_one("model-one"); + let second = spawn_one("model-two"); + let first_port = first.join().expect("thread one panicked")?; + let second_port = second.join().expect("thread two panicked")?; + let _ = fs::remove_dir_all(root); + + assert_ne!(first_port, second_port); + for port in [first_port, second_port] { + assert!( + (serve_port::AUTO_PORT_FIRST..=serve_port::AUTO_PORT_LAST).contains(&port), + "leased port {port} must fall within the auto-select range" + ); + } + Ok(()) + } + + #[test] + fn concurrent_equivalent_transactions_converge_on_one_record() -> Result<()> { + let (root, paths) = test_paths("alloc-txn-concurrent-converge"); + paths.ensure()?; + let paths = std::sync::Arc::new(paths); + + let race_once = || { + let paths = std::sync::Arc::clone(&paths); + thread::spawn(move || -> Result { + let _launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + match resolve_managed_port_in_transaction( + &paths, + "lemonade", + "shared-model", + "127.0.0.1", + serve_port::PortRequest::Auto, + &mut serve_port::loopback_bind_probe, + )? { + ManagedPortDecision::Lease(lease) => { + let port = lease.port(); + write_live_service_record( + &paths, + "lemonade-shared", + "shared-model", + port, + "running", + )?; + lease.release(); + Ok(true) + } + ManagedPortDecision::AlreadyLive(_) => Ok(false), + } + }) + }; + + let first = race_once(); + let second = race_once(); + let first_launched = first.join().expect("thread one panicked")?; + let second_launched = second.join().expect("thread two panicked")?; + + let live_count = live_managed_services(&paths).len(); + let _ = fs::remove_dir_all(root); + + assert_ne!( + first_launched, second_launched, + "exactly one racing transaction must launch" + ); + assert_eq!(live_count, 1, "only one record may end up live"); + Ok(()) + } + + #[test] + fn abandoned_launch_leaves_no_record_log_or_endpoint_key() -> Result<()> { + let (root, paths) = test_paths("abandon-launch-cleanup"); + paths.ensure()?; + let mut record = write_live_service_record( + &paths, + "lemonade-abandon", + "abandon-model", + 11_490, + "running", + )?; + fs::create_dir_all( + record + .engine_state_path + .parent() + .expect("engine state parent"), + )?; + fs::write(&record.engine_state_path, "{}")?; + endpoint_keys::store_endpoint_api_key(&paths, &record.service_id, "secret-key")?; + // Simulate a pre-spawn failure: nothing ever ran. + record.engine_pid = None; + + abandon_failed_managed_launch(&paths, &record); + + assert!(!record.manifest_path.exists()); + assert!(!record.log_path.exists()); + assert!(!record.engine_state_path.exists()); + assert_eq!( + endpoint_keys::endpoint_api_key(&paths, &record.service_id), + None + ); + assert!(live_managed_services(&paths).is_empty()); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + #[cfg(target_os = "linux")] + fn abandoned_launch_never_signals_a_pid_that_no_longer_matches_the_spawn_token() -> Result<()> { + // The child exited and its pid now belongs to an unrelated live + // process: the start token recorded at spawn no longer matches, so the + // forced tree kill must not touch it. + let (root, paths) = test_paths("abandon-launch-recycled-pid"); + paths.ensure()?; + let mut record = write_live_service_record( + &paths, + "lemonade-abandon-recycled", + "abandon-recycled-model", + 11_493, + "running", + )?; + let current_exe = std::env::current_exe().expect("current test exe"); + let mut bystander = ProcessCommand::new(current_exe) + .arg("--exact") + .arg(sleeper_test_name()) + .arg("--ignored") + .arg("--test-threads=1") + .env("ROCM_TEST_SLEEPER_CHILD", "1") + .spawn() + .expect("spawn bystander"); + record.record_engine_identity(bystander.id()); + record.engine_start_ticks = record.engine_start_ticks.map(|ticks| ticks + 1); + + abandon_failed_managed_launch(&paths, &record); + + let mismatched_left_alone = bystander.try_wait().expect("poll bystander").is_none(); + // A child that was gone before its token could be captured leaves none: + // that must not fall back to "best-effort" signalling either. + record.engine_start_ticks = None; + abandon_failed_managed_launch(&paths, &record); + let still_running = + mismatched_left_alone && bystander.try_wait().expect("poll bystander").is_none(); + let _ = bystander.kill(); + let _ = bystander.wait(); + let _ = fs::remove_dir_all(root); + assert!( + still_running, + "a pid with a mismatched token must be left alone" + ); + Ok(()) + } + + #[test] + #[cfg(not(windows))] + fn abandoned_launch_terminates_the_spawned_child() -> Result<()> { + let (root, paths) = test_paths("abandon-launch-terminates-child"); + paths.ensure()?; + let mut record = write_live_service_record( + &paths, + "lemonade-abandon-child", + "abandon-child-model", + 11_491, + "running", + )?; + + let current_exe = std::env::current_exe().expect("current test exe"); + let mut child = ProcessCommand::new(current_exe) + .arg("--exact") + .arg(sleeper_test_name()) + .arg("--ignored") + .arg("--test-threads=1") + .env("ROCM_TEST_SLEEPER_CHILD", "1") + .spawn() + .expect("spawn sleeper child"); + let child_pid = child.id(); + record.record_engine_identity(child_pid); + + abandon_failed_managed_launch(&paths, &record); + + let wait_result = child.wait(); + + match wait_result { + Ok(status) => assert!( + !status.success(), + "the sleeper child must be terminated, not exit cleanly" + ), + Err(error) => panic!("failed to wait for sleeper child: {error}"), + } + let persisted = load_managed_services(&paths) + .expect("re-read manifests") + .into_iter() + .find(|candidate| candidate.service_id == record.service_id); + let _ = fs::remove_dir_all(root); + let persisted = persisted.expect( + "a launch that spawned a child must keep its record so `rocm services`/`rocm logs` can show the attempt", + ); + // `stopped`, not `failed`: `rocmd`'s recovery restarts a `failed` record + // that carries no stop marker, which would respawn a launch the CLI just + // reported as failed. The rocmd side of the pair is + // `abandoned_launch_record_is_not_a_recovery_candidate` in + // `apps/rocmd/src/watchers.rs`. + assert_eq!( + persisted.status, "stopped", + "a spawned child's record must be kept, but in a status recovery ignores" + ); + Ok(()) + } +} diff --git a/apps/rocm/src/service_launch/readiness.rs b/apps/rocm/src/service_launch/readiness.rs new file mode 100644 index 0000000000..1e14c85599 --- /dev/null +++ b/apps/rocm/src/service_launch/readiness.rs @@ -0,0 +1,634 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Waiting on a launched or already-live managed service: the readiness budget, +//! liveness of the tracked process, how a wait ends, and the end-to-end OOM +//! fault-injection seam that fabricates a launch that never became ready. + +// There is deliberately no `use std::fs`, `use std::thread` or +// `use rocm_core::process_is_running` here: this file names them by full path. +// Their use sites sit under different `cfg` combinations (the Linux-only zombie +// probe, the OOM fault-injection seam, one Linux-only test), so any single import +// would be unused in some build, and a gate on it would have to be kept in step +// with every use site. Keep the full paths. The tests module imports `fs` for +// itself because its tests use it on every platform. +use super::spawn::ManagedLaunchReport; +#[cfg(feature = "e2e-oom-fault-injection")] +use crate::device_policy_name; +use crate::endpoint_keys; +use crate::engine_runtime_select::wait_for_service_http_ready_with_progress; +use crate::status_for_readiness; +#[cfg(feature = "e2e-oom-fault-injection")] +use anyhow::Context; +use anyhow::{Result, anyhow}; +#[cfg(feature = "e2e-oom-fault-injection")] +use rocm_core::format_http_base_url; +use rocm_core::{AppPaths, EndpointReadiness, ManagedServiceRecord}; +#[cfg(feature = "e2e-oom-fault-injection")] +use rocm_engine_protocol::{DevicePolicy, ResolveModelResponse}; +use std::fmt::Write as _; +use std::time::Duration; + +/// How long one readiness wait polls before `rocm serve` reports `starting`. +const READINESS_WAIT: Duration = Duration::from_secs(45); + +/// `rocm serve --wait`: keep waiting past [`READINESS_WAIT`] (a first start +/// downloads the model, which can take far longer) until the server answers or +/// stops being live. A server that exited ends the wait with the last reading +/// instead of blocking forever. +fn wait_past_readiness_budget( + mut probe: impl FnMut() -> EndpointReadiness, + mut still_live: impl FnMut() -> bool, +) -> EndpointReadiness { + loop { + let readiness = probe(); + if readiness == EndpointReadiness::Serving || !still_live() { + return readiness; + } + } +} + +/// Whether `pid` is a running process that is not a zombie. The serve +/// supervisor is an unreaped child of this process, so after it dies +/// `process_is_running` alone keeps reporting it alive. +pub(super) fn service_pid_is_alive(pid: u32) -> bool { + rocm_core::process_is_running(pid) && !process_is_zombie(pid) +} + +#[cfg(target_os = "linux")] +fn process_is_zombie(pid: u32) -> bool { + let Ok(stat) = std::fs::read_to_string(format!("/proc/{pid}/stat")) else { + return false; + }; + // `comm` may contain spaces and parentheses; the state follows the last `)`. + stat.rfind(')') + .and_then(|end| stat.get(end + 1..)) + .and_then(|rest| rest.split_whitespace().next()) + == Some("Z") +} + +#[cfg(not(target_os = "linux"))] +const fn process_is_zombie(_pid: u32) -> bool { + false +} + +/// Whether the record still describes a server that can finish starting: its +/// status is a live one *and* a tracked process is alive. The status alone is +/// not enough, since a killed engine never writes a terminal state. +fn record_still_live(record: &ManagedServiceRecord, pid_alive: impl Fn(u32) -> bool) -> bool { + if !crate::managed_service_status::managed_service_is_live(record) { + return false; + } + let pids = crate::managed_service_status::recorded_service_pids(record); + pids.is_empty() || pids.into_iter().any(pid_alive) +} + +/// [`record_still_live`] over the manifest on disk, so a `rocm services stop` +/// from another terminal (which writes only the manifest) ends the wait too. +fn service_still_live(paths: &AppPaths, service_id: &str, pid_alive: impl Fn(u32) -> bool) -> bool { + let Ok(mut latest) = ManagedServiceRecord::read_manifest(paths, service_id) else { + return false; + }; + let _ = latest.refresh_from_engine_state(); + record_still_live(&latest, pid_alive) +} + +/// Record a server that died mid-start as stopped, leaving a manifest that a +/// `services stop` already settled untouched. A `failed` state the engine +/// reported is persisted rather than overwritten, so the reason survives. +fn settle_exited_service(paths: &AppPaths, service_id: &str) { + let Ok(mut latest) = ManagedServiceRecord::read_manifest(paths, service_id) else { + return; + }; + let on_disk = latest.status.clone(); + let _ = latest.refresh_from_engine_state(); + if crate::managed_service_status::managed_service_is_live(&latest) { + latest.status = "stopped".to_owned(); + } else if latest.status == on_disk { + return; + } + let _ = latest.write(); +} + +/// The failure a `--wait` ends with when the server died before it became +/// ready, after recording the death in the manifest. `None` while the wait did +/// not apply, the endpoint is serving, or the server is still alive. +pub(super) fn died_during_wait( + paths: &AppPaths, + service_id: &str, + wait_until_ready: bool, + readiness: EndpointReadiness, + pid_alive: impl Fn(u32) -> bool, +) -> Option { + if !wait_until_ready + || readiness == EndpointReadiness::Serving + || service_still_live(paths, service_id, pid_alive) + { + return None; + } + settle_exited_service(paths, service_id); + let mut message = format!( + "the server exited before it became ready (vLLM and AIM give up after 5 minutes by default; `ROCM_CLI_VLLM_READY_TIMEOUT_SECS` / `ROCM_CLI_AIM_READY_TIMEOUT_SECS` allow longer); see `rocm logs --service {service_id}` for its output" + ); + // A plain `serve` adds the memory guidance for a vLLM launch that ran out of + // VRAM; the wait's failure is the same launch, so it gets the same note. + // "starting" is the status the note keys on for "never became ready", which is + // what a `--wait` death is, whatever the settled record now says. + if let Ok(record) = ManagedServiceRecord::read_manifest(paths, service_id) { + for note in crate::serve_summary::append_oom_serve_note( + Vec::new(), + &record.engine, + false, + "starting", + Some(&record.log_path), + ) { + let _ = write!(message, "\n note: {note}"); + } + } + Some(anyhow!(message)) +} + +/// Record the outcome of a finished wait on the manifest as it is now, changing +/// only what the wait learned. The copy read at spawn time can be long stale: +/// `rocmd` may have recorded a restart or a new supervisor pid, and `rocm logs` +/// refreshes the record, so writing it back whole would revert those. +pub(super) fn record_wait_outcome( + paths: &AppPaths, + held: &mut ManagedServiceRecord, + status: &str, + verified: bool, +) -> Result<()> { + let mut latest = ManagedServiceRecord::read_manifest(paths, &held.service_id) + .unwrap_or_else(|_| held.clone()); + status.clone_into(&mut latest.status); + if verified { + // Latch the verification the wait just performed, so the readiness checks + // behind `services list` and chat read it instead of re-probing. + latest.inference_verified_at_unix_ms = Some(rocm_core::unix_time_millis() as u64); + } + latest.write()?; + *held = latest; + Ok(()) +} + +/// Test-only fault-injection switch, off (`const false`) in shipped binaries: +/// the `e2e-oom-fault-injection` feature is enabled only by the `xtask e2e` +/// mock-lane build, never by the release build. When enabled and +/// `ROCM_E2E_SIMULATE_OOM_LAUNCH=1` is set on the process, `rocm serve` fabricates +/// the on-disk state of a managed vLLM launch that spawned, wrote an +/// out-of-memory traceback to the log it owns, and never became ready — so the +/// positive OOM-guidance summary path can be verified on a GPU-less CI host, +/// where a real launch cannot run. See [`simulate_oom_managed_launch`]. +#[cfg(feature = "e2e-oom-fault-injection")] +pub(crate) fn e2e_simulate_oom_launch() -> bool { + std::env::var_os("ROCM_E2E_SIMULATE_OOM_LAUNCH").is_some_and(|value| value == "1") +} + +/// Release builds carry no fault-injection switch: the seam compiles out +/// entirely, so `rocm serve` behaves identically to a build without the feature. +#[cfg(not(feature = "e2e-oom-fault-injection"))] +pub(crate) const fn e2e_simulate_oom_launch() -> bool { + false +} + +/// Test-only (see [`e2e_simulate_oom_launch`]): fabricate the on-disk state of a +/// managed vLLM launch that spawned, OOM'd, and never became ready, then return +/// the launch report the summary path would see for it — `already_running: false` +/// (this invocation launched it), `status: "starting"` (it failed to become +/// ready), and a `log_path` whose contents carry a real allocator OOM signature. +/// [`crate::serve_summary::append_oom_serve_note`] then renders the memory-knob guidance. No process is +/// spawned. Compiled out of shipped binaries. +#[cfg(feature = "e2e-oom-fault-injection")] +#[allow(clippy::too_many_arguments)] +pub(super) fn simulate_oom_managed_launch( + paths: &AppPaths, + engine: &str, + service_id: &str, + requested_model: &str, + resolve: &ResolveModelResponse, + host: &str, + port: u16, + device_policy: &DevicePolicy, + gpu_indices: &[u32], + runtime_id: Option<&str>, + env_id: Option<&str>, +) -> Result { + paths.ensure()?; + std::fs::create_dir_all(paths.services_dir())?; + let mut record = ManagedServiceRecord::new( + paths, + service_id, + engine, + requested_model, + resolve.canonical_model_id.clone(), + host, + port, + "managed", + std::process::id(), + runtime_id.map(str::to_owned), + env_id.map(str::to_owned), + Some(device_policy_name(device_policy).to_owned()), + ); + record.gpu_indices = gpu_indices.to_vec(); + record.status = "starting".to_owned(); + record.write()?; + // A real allocator OOM signature in the log this invocation owns, so the + // summary renders memory guidance for a launch that actually OOM'd. + // + // The engine's own error block is reproduced below it, because that is what + // a real managed launch leaves on disk, and leaving it out hid a bug this + // scenario should have caught: the supervisor and the engine share one log, + // so when vLLM OOMs the engine bails with its startup-log context — which + // embeds rocm-cli's own OOM hint — and `exit_code_for` writes that whole + // error back into the same file, *after* the real line. A fixture carrying + // only the raw OOM line can never see the newest-first scan land on + // rocm-cli's own sentence instead of the user's error. + let real_line = "torch.OutOfMemoryError: HIP out of memory. Tried to allocate 7.21 GiB."; + std::fs::write( + &record.log_path, + format!( + "{real_line}\n\ + Error: vLLM server process exited before becoming ready (status: exit status: 1)\n\ + \n\ + Startup log, last {} lines as the terminal drew them:\n\ + {real_line}\n\ + \n\ + {}{real_line}). {}\n\ + For conditional remediation, run `{} 'vllm: {real_line}'`.\n", + rocm_engine_protocol::DEFAULT_LOG_TAIL_LINES, + rocm_core::VLLM_OOM_DETECTED_PREFIX, + rocm_core::VLLM_GPU_MEMORY_UTILIZATION_HINT, + rocm_core::VLLM_OOM_DIAGNOSE_COMMAND, + ), + ) + .with_context(|| format!("failed to write {}", record.log_path.display()))?; + Ok(ManagedLaunchReport { + service_id: service_id.to_owned(), + endpoint_url: format!("{}/v1", format_http_base_url(host, port)), + status: "starting".to_owned(), + already_running: false, + child_pid: None, + log_path: Some(record.log_path), + manifest_path: Some(record.manifest_path), + }) +} + +/// Probe `record`'s endpoint until it answers: once within the readiness +/// budget, or (with `wait_until_ready`) for as long as the server stays alive. +pub(super) fn wait_for_record_ready( + paths: &AppPaths, + record: &ManagedServiceRecord, + wait_until_ready: bool, + endpoint_api_key: Option<&str>, + on_wait_tick: &mut dyn FnMut(Duration), +) -> EndpointReadiness { + let probe = |on_tick: &mut dyn FnMut(Duration)| { + wait_for_service_http_ready_with_progress( + &record.engine, + &record.host, + record.port, + &|| { + let mut latest = record.clone(); + let _ = latest.refresh_from_engine_state(); + latest.served_model_name().to_owned() + }, + endpoint_api_key, + READINESS_WAIT, + on_tick, + ) + }; + if wait_until_ready { + wait_past_readiness_budget( + || probe(&mut *on_wait_tick), + || service_still_live(paths, &record.service_id, service_pid_is_alive), + ) + } else { + probe(on_wait_tick) + } +} + +/// An equivalent service is already live. Without `--wait`, or once it is +/// serving, report it as is; otherwise wait for it like a fresh launch would. +pub(super) fn wait_on_existing_service( + paths: &AppPaths, + mut report: ManagedLaunchReport, + wait_until_ready: bool, + on_wait_begin: &mut dyn FnMut(&str, u16), + on_wait_tick: &mut dyn FnMut(Duration), +) -> Result { + if !wait_until_ready || report.status == "ready" { + return Ok(report); + } + let Ok(mut record) = ManagedServiceRecord::read_manifest(paths, &report.service_id) else { + return Ok(report); + }; + // The id of the server being waited on, which is not the caller's freshly + // generated one when it attached to an existing server. + on_wait_begin(&report.service_id, record.port); + // The key the caller generated belongs to a launch that did not happen; the + // existing service answers to its own. + let stored_key = endpoint_keys::endpoint_api_key(paths, &report.service_id); + let readiness = + wait_for_record_ready(paths, &record, true, stored_key.as_deref(), on_wait_tick); + if let Some(error) = died_during_wait( + paths, + &report.service_id, + true, + readiness, + service_pid_is_alive, + ) { + return Err(error); + } + report.status = status_for_readiness(readiness).to_owned(); + record_wait_outcome( + paths, + &mut record, + &report.status, + readiness == EndpointReadiness::Serving, + )?; + Ok(report) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::service_launch::test_support::reusable_record; + use crate::tests::test_paths; + use std::fs; + + #[test] + fn wait_past_readiness_budget_keeps_probing_a_live_starting_server() { + let mut readings = vec![ + EndpointReadiness::Unreachable, + EndpointReadiness::Unreachable, + EndpointReadiness::Serving, + ] + .into_iter(); + let mut probes = 0; + let result = wait_past_readiness_budget( + || { + probes += 1; + readings.next().expect("no probe after Serving") + }, + || true, + ); + assert_eq!(result, EndpointReadiness::Serving); + assert_eq!(probes, 3, "must outlast the first (45s) budget"); + } + + #[test] + fn wait_past_readiness_budget_stops_when_the_server_exits() { + let mut probes = 0; + let result = wait_past_readiness_budget( + || { + probes += 1; + EndpointReadiness::Unreachable + }, + || false, + ); + assert_eq!(result, EndpointReadiness::Unreachable); + assert_eq!(probes, 1); + } + + #[test] + fn wait_past_readiness_budget_returns_at_once_when_already_serving() { + let result = wait_past_readiness_budget( + || EndpointReadiness::Serving, + || panic!("liveness is irrelevant once serving"), + ); + assert_eq!(result, EndpointReadiness::Serving); + } + + fn record_tracking_pid(paths: &AppPaths, pid: u32, status: &str) -> ManagedServiceRecord { + let mut record = reusable_record(paths); + record.supervisor_pid = pid; + record.status = status.to_owned(); + record + } + + #[cfg(unix)] + fn exited_child_pid() -> u32 { + let mut child = std::process::Command::new("true").spawn().expect("spawn"); + let pid = child.id(); + child.wait().expect("reap"); + pid + } + + #[test] + #[cfg(unix)] + fn record_still_live_ends_the_wait_for_a_dead_pid_with_a_live_status() { + // The engine was killed before writing any terminal state: the status + // still says `starting`, but nothing tracked is running. + let (root, paths) = test_paths("wait-dead-pid"); + paths.ensure().expect("paths"); + let dead = record_tracking_pid(&paths, exited_child_pid(), "starting"); + assert!(!record_still_live(&dead, service_pid_is_alive)); + + let alive = record_tracking_pid(&paths, std::process::id(), "starting"); + assert!(record_still_live(&alive, service_pid_is_alive)); + + // A settled status ends the wait even while the pid is still around. + let stopped = record_tracking_pid(&paths, std::process::id(), "stopped"); + assert!(!record_still_live(&stopped, service_pid_is_alive)); + let _ = fs::remove_dir_all(root); + } + + #[test] + #[cfg(target_os = "linux")] + fn service_pid_is_alive_treats_an_unreaped_child_as_dead() { + // The serve supervisor is our own unreaped child; once it dies it is a + // zombie that `kill(pid, 0)` still reports as running. + let mut child = std::process::Command::new("true").spawn().expect("spawn"); + let pid = child.id(); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while !process_is_zombie(pid) { + assert!(std::time::Instant::now() < deadline, "child never exited"); + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + rocm_core::process_is_running(pid), + "a zombie still answers kill(pid, 0)" + ); + assert!(!service_pid_is_alive(pid)); + child.wait().expect("reap"); + } + + #[test] + fn service_still_live_follows_the_manifest_on_disk() { + // `rocm services stop` from another terminal writes only the manifest. + let (root, paths) = test_paths("wait-manifest"); + paths.ensure().expect("paths"); + let mut record = record_tracking_pid(&paths, std::process::id(), "starting"); + record.write().expect("write"); + assert!(service_still_live(&paths, &record.service_id, |_| true)); + + record.status = "stopped".to_owned(); + record.write().expect("write"); + assert!(!service_still_live(&paths, &record.service_id, |_| true)); + assert!(!service_still_live(&paths, "no-such-service", |_| true)); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn settle_exited_service_demotes_a_live_manifest_and_spares_a_settled_one() { + let (root, paths) = test_paths("wait-settle"); + paths.ensure().expect("paths"); + let mut record = record_tracking_pid(&paths, 0, "starting"); + record.write().expect("write"); + settle_exited_service(&paths, &record.service_id); + let after = ManagedServiceRecord::read_manifest(&paths, &record.service_id).unwrap(); + assert_eq!(after.status, "stopped"); + + // A record that already failed keeps its reason: the guard, not the + // overwrite, is what spares it (a `stopped` -> `stopped` rewrite would + // be invisible). + record.status = "failed".to_owned(); + record.write().expect("write"); + settle_exited_service(&paths, &record.service_id); + let after = ManagedServiceRecord::read_manifest(&paths, &record.service_id).unwrap(); + assert_eq!(after.status, "failed"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn wait_on_existing_service_only_waits_for_one_that_is_not_serving_yet() { + // The probing path itself blocks for the readiness budget, so only the + // cases that must return at once are driven here; a manifest planted + // as `starting` would hang a wrong guard for 45 seconds and fail it. + let (root, paths) = test_paths("wait-existing"); + paths.ensure().expect("paths"); + let mut record = record_tracking_pid(&paths, 0, "starting"); + record.write().expect("write"); + let report = |status: &str| ManagedLaunchReport { + service_id: record.service_id.clone(), + endpoint_url: record.endpoint_url.clone(), + status: status.to_owned(), + already_running: true, + child_pid: None, + log_path: None, + manifest_path: None, + }; + let mut no_tick = |_: Duration| panic!("must not probe"); + + let untouched = wait_on_existing_service( + &paths, + report("starting"), + false, + &mut |_, _| panic!("no --wait, no banner"), + &mut no_tick, + ) + .expect("no --wait"); + assert_eq!(untouched.status, "starting"); + let ready = wait_on_existing_service( + &paths, + report("ready"), + true, + &mut |_, _| panic!("already ready, no banner"), + &mut no_tick, + ) + .expect("already ready"); + assert_eq!(ready.status, "ready"); + record.status = "starting".to_owned(); + assert_eq!( + ManagedServiceRecord::read_manifest(&paths, &record.service_id) + .unwrap() + .status, + "starting" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn died_during_wait_fails_with_a_logs_pointer_and_records_the_death() { + let (root, paths) = test_paths("wait-died"); + paths.ensure().expect("paths"); + let record = record_tracking_pid(&paths, std::process::id(), "starting"); + record.write().expect("write"); + let id = record.service_id; + let status = || { + ManagedServiceRecord::read_manifest(&paths, &id) + .unwrap() + .status + }; + + // Not waiting, already serving, or still alive: no failure, no change. + let pending = EndpointReadiness::Unreachable; + assert!(died_during_wait(&paths, &id, false, pending, |_| false).is_none()); + assert!( + died_during_wait(&paths, &id, true, EndpointReadiness::Serving, |_| false).is_none() + ); + assert!(died_during_wait(&paths, &id, true, pending, |_| true).is_none()); + assert_eq!(status(), "starting"); + + // Dead server: the error names the logs command and the record is settled. + let error = died_during_wait(&paths, &id, true, pending, |_| false).expect("died"); + let message = error.to_string(); + assert!( + message.contains("exited before it became ready"), + "{message}" + ); + assert!( + message.contains(&format!("rocm logs --service {id}")), + "{message}" + ); + assert_eq!(status(), "stopped"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn died_during_wait_adds_the_oom_guidance_for_a_vllm_launch_only() { + let (root, paths) = test_paths("wait-died-oom"); + paths.ensure().expect("paths"); + let mut record = record_tracking_pid(&paths, std::process::id(), "starting"); + record.engine = "vllm".to_owned(); + record.write().expect("write"); + fs::write( + &record.log_path, + "torch.OutOfMemoryError: HIP out of memory. Tried to allocate 7.21 GiB.\n", + ) + .expect("log"); + let id = record.service_id; + let pending = EndpointReadiness::Unreachable; + let message = died_during_wait(&paths, &id, true, pending, |_| false) + .expect("died") + .to_string(); + assert!(message.contains("--gpu-memory-utilization"), "{message}"); + + // Same log, another engine: that engine has no such flag to suggest. + let mut record = ManagedServiceRecord::read_manifest(&paths, &id).unwrap(); + record.engine = "lemonade".to_owned(); + "starting".clone_into(&mut record.status); + record.write().expect("write"); + let message = died_during_wait(&paths, &id, true, pending, |_| false) + .expect("died") + .to_string(); + assert!(!message.contains("--gpu-memory-utilization"), "{message}"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_wait_outcome_keeps_what_changed_on_disk_during_the_wait() { + let (root, paths) = test_paths("wait-outcome"); + paths.ensure().expect("paths"); + let mut held = record_tracking_pid(&paths, 111, "starting"); + held.write().expect("write"); + // While the wait ran, `rocmd` recorded a restart and a new supervisor. + let mut elsewhere = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); + elsewhere.restart_count = 3; + elsewhere.supervisor_pid = 222; + elsewhere.write().expect("write"); + + record_wait_outcome(&paths, &mut held, "ready", true).expect("outcome"); + let after = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); + assert_eq!(after.status, "ready"); + assert!(after.inference_verified_at_unix_ms.is_some()); + assert_eq!(after.restart_count, 3, "the restart count was reverted"); + assert_eq!(after.supervisor_pid, 222, "the supervisor pid was reverted"); + + record_wait_outcome(&paths, &mut held, "starting", false).expect("outcome"); + let after = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); + assert_eq!(after.status, "starting"); + let _ = fs::remove_dir_all(root); + } +} diff --git a/apps/rocm/src/service_launch/spawn.rs b/apps/rocm/src/service_launch/spawn.rs new file mode 100644 index 0000000000..0bc3d02a34 --- /dev/null +++ b/apps/rocm/src/service_launch/spawn.rs @@ -0,0 +1,1306 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Spawning the detached managed engine child shared by the background and +//! attached serve paths: process and stdio plumbing, [`ManagedLaunchReport`] and +//! [`ManagedSpawn`], and the serialized spawn transaction itself. + +use super::endpoint_auth::ensure_public_service_has_endpoint_key; +use super::launch_transaction::{ + ManagedPortDecision, ManagedReuseRefusal, abandon_failed_managed_launch, + managed_service_reuse_refusal, no_usable_gpu_refusal, resolve_managed_port_in_transaction, +}; +use crate::endpoint_keys; +use crate::engine_ipc::builtin_engine_serve_http_args; +#[cfg(not(windows))] +use crate::engine_runtime_select::apply_app_path_env; +use crate::engine_runtime_select::managed_service_launcher_path; +#[cfg(windows)] +use crate::engine_runtime_select::{app_path_env_var_refs, app_path_env_var_values}; +use crate::engines_cmd::env_root_for_service; +use crate::serve_port; +use crate::service_hints; +use crate::services_text::read_optional_tail_lines; +use crate::{device_policy_name, record_cli_audit_event}; +use anyhow::{Context, Result, bail}; +#[cfg(windows)] +use rocm_core::process_is_running; +use rocm_core::{AppPaths, ManagedServiceRecord}; +use rocm_engine_protocol::{DevicePolicy, EngineRecipeHint, ResolveModelResponse}; +use std::fs; +#[cfg(not(windows))] +use std::io; +use std::path::{Path, PathBuf}; +#[cfg(not(windows))] +use std::process::{Command as ProcessCommand, Stdio}; +use std::thread; +use std::time::Duration; + +#[cfg(not(windows))] +pub(crate) fn detach_background_command(command: &mut ProcessCommand) { + rocm_core::detach_command_session(command); +} + +#[cfg(not(windows))] +pub(crate) fn attach_background_stdio( + command: &mut ProcessCommand, + log_path: Option<&Path>, +) -> Result<()> { + if let Some(log_path) = log_path { + let log = fs::OpenOptions::new() + .create(true) + .append(true) + .open(log_path) + .with_context(|| format!("failed to open {}", log_path.display()))?; + command + .stdout(Stdio::from(log.try_clone()?)) + .stderr(Stdio::from(log)); + } else { + command.stdout(Stdio::null()).stderr(Stdio::null()); + } + Ok(()) +} + +/// Windows spawn-liveness check: wait out the startup window, then fail with the +/// startup log when the child is already gone. The wait and the liveness probe +/// are injected so a unit test runs the exit-detected path on every platform, +/// including the Windows lane that actually calls it. +/// +/// # Errors +/// Fails with the startup-failure detail when `is_running` reports the child dead. +#[cfg_attr(not(windows), allow(dead_code))] +fn ensure_spawned_child_survives_startup( + child_pid: u32, + log_path: &Path, + settle: impl FnOnce(), + is_running: impl FnOnce(u32) -> bool, +) -> Result<()> { + settle(); + if is_running(child_pid) { + Ok(()) + } else { + bail!("{}", managed_engine_startup_failure_detail("", log_path)) + } +} + +/// `status_clause` is a pre-formatted fragment inserted right after "exited +/// immediately" (e.g. `" with status exit status: 1"` on Unix, where a real +/// `ExitStatus` is available; empty on Windows, where the liveness check +/// below only has a yes/no answer, not an exit code). +pub(crate) fn managed_engine_startup_failure_detail( + status_clause: &str, + log_path: &Path, +) -> String { + let mut recent_lines = read_optional_tail_lines(log_path, 80, "service log"); + if recent_lines.is_empty() { + for _ in 0..5 { + thread::sleep(Duration::from_millis(120)); + recent_lines = read_optional_tail_lines(log_path, 80, "service log"); + if !recent_lines.is_empty() { + break; + } + } + } + if recent_lines.is_empty() { + return format!( + "managed engine exited immediately{status_clause}; inspect {}", + log_path.display() + ); + } + format!( + "managed engine exited immediately{status_clause}; inspect {}\n\nrecent startup log output:\n{}", + log_path.display(), + recent_lines.join("\n") + ) +} +#[cfg(not(windows))] +pub(crate) fn managed_service_process_command(program: &Path, args: &[String]) -> ProcessCommand { + let mut command = ProcessCommand::new(program); + command.args(args); + command +} + +/// Engine-neutral result of a managed launch. Returned rather than printed so the +/// caller can render it either as the rich deployment summary (interactive TTY) or +/// as the plain line-by-line form (piped output, chat assistant), from one code path. +pub(crate) struct ManagedLaunchReport { + pub(crate) service_id: String, + /// `http://host:port/v1`. + pub(crate) endpoint_url: String, + /// `"ready"` (inference confirmed), `"running"` (model listed but not serving + /// yet), `"starting"` (endpoint not answering), or the existing service's + /// status when nothing was spawned. + pub(crate) status: String, + /// True when an equivalent service was already live and nothing was spawned. + pub(crate) already_running: bool, + pub(crate) child_pid: Option, + pub(crate) log_path: Option, + pub(crate) manifest_path: Option, +} + +/// Either an already-live service (nothing spawned) or a freshly spawned engine +/// child that is `running` but not yet HTTP-ready. +/// +/// Split out of [`start_managed_service`](super::start_managed_service) so the +/// attached (`--verbose` / `--foreground`) serve path can spawn the very same +/// detached child and stream its log live from the first line — including +/// startup — instead of blocking on the readiness wait before any output appears. +pub(super) enum ManagedSpawn { + AlreadyRunning(ManagedLaunchReport), + // `ManagedServiceRecord` is large; box it so the two variants stay a similar + // size (clippy::large_enum_variant). + Spawned { + record: Box, + child_pid: u32, + }, +} + +/// Spawn the detached engine child shared by the managed (background) and +/// attached (`--verbose`/`--foreground`) serve paths. Returns before the HTTP +/// readiness wait; callers decide whether to block on readiness +/// ([`start_managed_service`](super::start_managed_service)) or start tailing the +/// log immediately ([`run_attached_service`](super::run_attached_service)). +/// +/// The whole body is one serialized transaction under the shared allocation +/// lock: duplicate detection, live-reservation refresh, candidate leasing, +/// manifest/log creation, argv preparation, the bounded immediate-exit check, +/// and record publication. Readiness deliberately runs after the lock is +/// released. +#[allow(clippy::too_many_arguments)] +pub(super) fn spawn_managed_engine_child( + paths: &AppPaths, + engine: &str, + service_id: &str, + requested_model: &str, + resolve: &ResolveModelResponse, + host: &str, + port_request: serve_port::PortRequest, + device_policy: &DevicePolicy, + gpu_indices: &[u32], + runtime_id: Option<&str>, + env_id: Option<&str>, + engine_recipe: Option<&EngineRecipeHint>, + require_api_key: bool, + gpu_refusal_waived_for_reuse: bool, +) -> Result { + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + // No lock is acquired here: every production caller + // ([`start_managed_service`], [`run_attached_service`]) already holds + // [`rocm_core::AppPaths::managed_launch_lock_path`] for the whole of this + // call, so no two transactions can choose the same port or publish + // competing records for one engine+model. A second acquisition in this + // frame would self-deadlock, since the lock is exclusive per open file + // description, not per process. + // + // Idempotency guard: if a managed service for this engine+model is already + // alive, surface it and spawn nothing. A second `serve --managed` (e.g. the + // chat assistant re-issuing the same request) is treated as satisfied, not + // an error. Keyed on engine+canonical model — the freshly generated + // `service_id` is timestamp-unique and would never match an existing one. + // Stale/dead services fall through and relaunch normally. + let requested_recipe_json = engine_recipe + .map(serde_json::to_string) + .transpose() + .context("failed to encode engine recipe hint")?; + let lease = match resolve_managed_port_in_transaction( + paths, + engine, + &resolve.canonical_model_id, + host, + port_request, + &mut serve_port::loopback_bind_probe, + )? { + ManagedPortDecision::AlreadyLive(existing) => { + let existing = *existing; + // Asked through the shared predicate rather than re-tested here, so this + // refusal and `serve()`'s reuse pre-gate cannot disagree about whether a + // service is reusable. When they disagreed, the pre-gate let a + // non-reusable service bypass the no-GPU refusal and the user was sent + // here to be told to stop a working server for nothing. + // + // The same device policy the serve path passes. This side used to pass + // `None` to keep the spawn's acceptance untouched, but that opted the + // spawn out of a term it has the input for, left its own + // `DevicePolicyTooWeak` bail unreachable, and made the serve path's + // argument droppable. Nothing observable widens: `parse_device_policy` + // is the only producer of a `DevicePolicy` on every path that reaches + // here and it rejects `cpu_only`, so no record this CLI writes can + // carry the policy this term refuses. + match managed_service_reuse_refusal( + &existing, + requested_recipe_json.as_deref(), + require_api_key, + device_policy, + ) { + Some(ManagedReuseRefusal::RecipeMismatch) => bail!( + "managed service `{}` is already running for engine `{engine}` and model `{}` with different serve options (recipe hint, tool-call parser, or generation defaults); stop it and run `rocm serve` again to apply the requested options", + existing.service_id, + resolve.canonical_model_id + ), + Some(ManagedReuseRefusal::MissingApiKey) => bail!( + "managed service `{}` is already running for engine `{engine}` and model `{}` \ + without authentication, and a running server cannot be given a key it did not \ + start with; stop it with `{}` and run the command again to \ + serve it with `--require-api-key`", + existing.service_id, + resolve.canonical_model_id, + service_hints::service_stop_hint(&existing.service_id) + ), + // Reachable only from a record whose `device_policy` says + // `cpu_only`, which no `rocm serve` can write today + // (`parse_device_policy` refuses that policy outright) — so this is + // a guard against a record written by some future path, not a + // refusal users see now. The advice still names a command that + // clears the condition: stopping the service removes the only thing + // this term can object to, and the rerun then launches normally + // (AGENTS.md section 3). + Some(ManagedReuseRefusal::DevicePolicyTooWeak) => bail!( + "managed service `{}` is already running under a device policy that cannot \ + satisfy this request; stop it with `{}` and run the command again", + existing.service_id, + service_hints::service_stop_hint(&existing.service_id) + ), + None => {} + } + // An explicit port is a demand, not a hint: reporting success on a + // different port than the one asked for would silently drop it. + if let Some(port) = port_request.explicit() + && port != existing.port + { + bail!( + "managed service `{}` is already serving engine `{engine}` and model `{}` on port {}, so port {port} would not be used; stop it with `{}` and run the command again to serve on that port", + existing.service_id, + resolve.canonical_model_id, + existing.port, + service_hints::service_stop_hint(&existing.service_id) + ); + } + record_cli_audit_event( + paths, + "service", + "managed_service_launch_skipped", + "info", + format!( + "skipped duplicate managed launch engine={engine} model={} existing_service_id={} status={}", + resolve.canonical_model_id, existing.service_id, existing.status + ), + Some(&existing.service_id), + ); + return Ok(ManagedSpawn::AlreadyRunning(ManagedLaunchReport { + service_id: existing.service_id, + endpoint_url: existing.endpoint_url, + status: existing.status, + already_running: true, + child_pid: None, + log_path: None, + manifest_path: None, + })); + } + ManagedPortDecision::Lease(lease) => lease, + }; + + // Nothing live to reuse — and if the only reason this invocation got past + // `serve()`'s no-usable-GPU pre-flight was a reusable service, that premise + // has just been falsified. `serve()` decided it from a snapshot taken before + // GPU selection and the launch lock; the check above is the authoritative + // one, so a service that died in between (an OOM, say) would otherwise fall + // through to a real spawn on a host with no usable GPU, and the user would + // get whatever the engine says about having no device instead of the + // actionable refusal the pre-flight exists to give. Raise it here, where the + // premise is known to be false. + if gpu_refusal_waived_for_reuse { + bail!("{}", no_usable_gpu_refusal(device_policy)); + } + + let mut record = ManagedServiceRecord::new( + paths, + service_id, + engine, + requested_model, + resolve.canonical_model_id.clone(), + host, + lease.port(), + "managed", + 0, + runtime_id.map(str::to_owned), + env_id.map(str::to_owned), + Some(device_policy_name(device_policy).to_owned()), + ); + record.gpu_indices = gpu_indices.to_vec(); + record.served_model_id = resolve.served_model_id.clone(); + record.engine_recipe_json = requested_recipe_json; + // The flag the user actually passed, carried through rather than re-derived. + // + // Deriving it from key-file presence looked equivalent and was not: + // `resolve_endpoint_auth` mints a key for *every* non-loopback bind whether or + // not auth was demanded, so a plain `--host 0.0.0.0 --allow-public-bind` + // recorded `true` here. The guard below tests this field before the bind + // address, so that service was then refused with a message naming a flag it + // never used and a relaunch command that drops `--allow-public-bind` — the + // public-bind branch, which carries the right command, became unreachable. + // + // This field means "the user demanded auth on a bind that would not otherwise + // require it". A public bind needs no such record; its address still says so. + record.requires_api_key = require_api_key; + // Publishing the record inside the lock is what reserves the leased port for + // every other transaction. From here on, any failure must remove it again. + record.write()?; + + match launch_managed_engine_child_locked( + paths, + &mut record, + lease, + engine, + service_id, + resolve, + host, + device_policy, + gpu_indices, + runtime_id, + env_id, + engine_recipe, + #[cfg(not(windows))] + None, + ) { + Ok(child_pid) => Ok(ManagedSpawn::Spawned { + record: Box::new(record), + child_pid, + }), + Err(error) => { + // The single cleanup choke point for every failure path below the + // record's first write: `record.engine_pid` tells it whether a child + // ever ran, so a pre-spawn failure deletes everything and a post-spawn + // one keeps the record and log (marked `status = "stopped"`) instead. + abandon_failed_managed_launch(paths, &record); + Err(error) + } + } +} + +// Test-only, thread-scoped spawn interceptor for callers that cannot take a +// `spawn_override` argument (`start_managed_service`, `run_attached_service`): +// lets a test observe state at the instant production would spawn the child. +#[cfg(all(test, not(windows)))] +thread_local! { + pub(super) static TEST_SPAWN_HOOK: std::cell::RefCell>>> = + const { std::cell::RefCell::new(None) }; +} + +#[cfg(all(test, not(windows)))] +fn test_spawn_hook() -> Option>> { + TEST_SPAWN_HOOK.with(|hook| hook.borrow().clone()) +} + +#[cfg(not(windows))] +type SpawnOverride<'a> = dyn Fn(&mut ProcessCommand) -> io::Result + 'a; + +/// The side-effecting half of the launch transaction: create the log, prepare +/// concrete argv, hand the leased port to the child, spawn it, reject a child +/// that has already exited, and publish the running record. +/// +/// Split out of [`spawn_managed_engine_child`] so that *every* failure below the +/// record's first write is funnelled through one cleanup path before the +/// launch lock is released. Returns the child pid. +#[allow(clippy::too_many_arguments)] +fn launch_managed_engine_child_locked( + paths: &AppPaths, + record: &mut ManagedServiceRecord, + lease: rocm_core::LoopbackPortLease, + engine: &str, + service_id: &str, + resolve: &ResolveModelResponse, + host: &str, + device_policy: &DevicePolicy, + gpu_indices: &[u32], + runtime_id: Option<&str>, + env_id: Option<&str>, + engine_recipe: Option<&EngineRecipeHint>, + // Test-only seam: lets a test assert the leased port is actually bindable + // (i.e. released) at the exact point production would spawn the child, + // without needing to fake or intercept the real spawn mechanics. + #[cfg(not(windows))] spawn_override: Option<&SpawnOverride<'_>>, +) -> Result { + let port = record.port; + if let Some(parent) = record.engine_state_path.parent() { + fs::create_dir_all(parent) + .with_context(|| format!("failed to create {}", parent.display()))?; + } + fs::File::create(&record.log_path) + .with_context(|| format!("failed to create {}", record.log_path.display()))?; + let current_exe = managed_service_launcher_path() + .context("failed to resolve current rocm executable path")?; + let serve_args = builtin_engine_serve_http_args( + engine, + service_id, + &resolve.canonical_model_id, + host, + port, + device_policy, + gpu_indices, + runtime_id, + env_id, + engine_recipe, + &record.engine_state_path, + Some(&record.log_path), + )?; + let engine_envs_root = env_root_for_service(paths, engine, runtime_id, env_id)?; + // Hand the child the *path* to the endpoint key file (public bind only) via the + // environment. A path — not the secret value — is what the detached-spawn + // primitives accept as an env override, and it keeps the key off both the argv + // and the environment block. `serve()` wrote the file before spawning. + // Validity, not mere existence: the engine adapters resolve the key with + // `endpoint_api_key_from_file` and enforce nothing when it yields `None`, so + // an empty or malformed key file would otherwise satisfy the guard below and + // still produce an unauthenticated public listener. + let endpoint_key_file = endpoint_keys::endpoint_key_file_if_present(paths, service_id) + .filter(|path| rocm_engine_protocol::endpoint_api_key_from_file(path).is_some()); + // `serve()` already resolved and stored the key for a public bind, so the + // public-bind branch cannot fire on the fresh-launch path today. It is the + // shared choke point for managed spawns, so enforce the invariant here too + // rather than relying on every future caller having done so. + // + // `record.requires_api_key` is passed, not a literal, and the two arguments + // are deliberately different things: that field is the `--require-api-key` + // flag the caller passed, `endpoint_key_file` is whether a *usable* key is on + // disk. A present but empty or malformed key file is where they disagree, and + // is exactly what the `requires_api_key` branch exists to refuse. + // + // The field is threaded, never derived from the key file. Deriving it marked + // every public bind as having demanded auth — see the assignment above. + ensure_public_service_has_endpoint_key( + host, + endpoint_key_file.is_some(), + record.requires_api_key, + )?; + // Hand the port over: close the probe socket immediately before the spawn, + // still under the caller's launch lock, so no other ROCm transaction can + // take it in the meantime. On a custom host the lease never held a socket + // and this is a no-op — the engine owns that bind. + debug_assert_eq!(lease.port(), port); + lease.release(); + #[cfg(windows)] + let child_pid = { + let env_values = app_path_env_var_values(paths, engine_envs_root.as_deref()); + let mut env_refs = app_path_env_var_refs(&env_values); + if let Some(key_file) = endpoint_key_file.as_deref() { + env_refs.push((rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file)); + } + let child_pid = rocm_core::spawn_detached_no_inherit(¤t_exe, &serve_args, &env_refs) + .context("failed to launch managed engine process")?; + // Set before the immediate-exit check below, so a bail! here still leaves + // `record.engine_pid` telling cleanup that a child was spawned. The + // start-time token is captured here too, while the child is certainly + // ours: cleanup binds its kill to it rather than to whatever owns the + // pid by then. + record.record_engine_identity(child_pid); + ensure_spawned_child_survives_startup( + child_pid, + &record.log_path, + || thread::sleep(Duration::from_millis(200)), + process_is_running, + )?; + child_pid + }; + #[cfg(not(windows))] + let child_pid = { + let mut command = managed_service_process_command(¤t_exe, &serve_args); + command.stdin(Stdio::null()); + attach_background_stdio(&mut command, Some(&record.log_path))?; + detach_background_command(&mut command); + apply_app_path_env(&mut command, paths); + if let Some(engine_envs_root) = engine_envs_root.as_deref() { + command.env("ROCM_CLI_ENGINE_ENVS_ROOT", engine_envs_root); + } + if let Some(key_file) = endpoint_key_file.as_deref() { + command.env(rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV, key_file); + } + #[cfg(all(test, not(windows)))] + let hook = test_spawn_hook(); + #[cfg(all(test, not(windows)))] + let spawn_override = spawn_override.or(hook.as_deref()); + let mut child = match spawn_override { + Some(spawn) => spawn(&mut command), + None => command.spawn(), + } + .context("failed to launch managed engine process")?; + let child_pid = child.id(); + // Set before the immediate-exit check below, so a bail! here still leaves + // `record.engine_pid` telling cleanup that a child was spawned. The + // start-time token is captured here too, while the child is certainly + // ours: cleanup binds its kill to it rather than to whatever owns the + // pid by then. + record.record_engine_identity(child_pid); + thread::sleep(Duration::from_millis(200)); + if let Some(status) = child + .try_wait() + .context("failed to check managed engine startup state")? + { + bail!( + "{}", + managed_engine_startup_failure_detail( + &format!(" with status {status}"), + &record.log_path + ) + ); + } + child_pid + }; + // The child is both roles until the engine state file names a separate + // server process. Each role gets its own token, captured while the child is + // alive: a stop pairs every recorded PID with its own token, and the two + // PIDs diverge as soon as `rocmd` restarts the service — at which point an + // engine PID recorded without one could only be signalled blind. + // The engine's token was already captured right after the spawn (so cleanup + // can bind to it); re-capturing here could replace it with a later owner's. + record.record_supervisor_identity(child_pid); + record.status = "running".to_owned(); + if let Err(error) = record.write() { + // Cleanup is the caller's job now: `record.engine_pid` (set above) is + // already enough for `abandon_failed_managed_launch` to terminate this + // child, so there is nothing left for this inner path to do beyond + // surfacing the error. + return Err(error.context("failed to publish the managed service record")); + } + + Ok(child_pid) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::managed_service_status::load_managed_services; + use crate::service_launch::launch_transaction::live_managed_services; + #[cfg(not(windows))] + use crate::service_launch::test_support::sleeper_test_name; + use crate::service_launch::test_support::write_live_service_record; + use crate::tests::test_paths; + use rocm_core::DEFAULT_LOCAL_HOST; + + #[test] + fn a_child_that_exited_during_the_startup_window_is_reported_with_its_log() -> Result<()> { + let dir = tempfile::tempdir()?; + let log = dir.path().join("engine.log"); + fs::write(&log, "boom: engine could not start\n")?; + let settled = std::cell::Cell::new(false); + let error = ensure_spawned_child_survives_startup( + 4242, + &log, + || settled.set(true), + |pid| { + assert_eq!(pid, 4242); + assert!(settled.get(), "liveness must be probed after the wait"); + false + }, + ) + .expect_err("a dead child must fail the launch"); + assert!(settled.get(), "the startup window must elapse first"); + assert!(format!("{error:#}").contains("boom"), "{error:#}"); + ensure_spawned_child_survives_startup(4242, &log, || {}, |_| true)?; + Ok(()) + } + + #[test] + fn spawn_managed_engine_child_refuses_an_explicit_port_a_live_equivalent_does_not_serve() + -> Result<()> { + let (root, paths) = test_paths("dup-managed-explicit-port-differs"); + paths.ensure()?; + let mut existing = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-1000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11510, + "managed", + std::process::id(), + None, + None, + None, + ); + existing.status = "ready".to_owned(); + existing.engine_pid = Some(std::process::id()); + existing.write()?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + let launch = |port_request| { + spawn_managed_engine_child( + &paths, + "lemonade", + "lemonade-qwen-2000", + "qwen", + &resolve, + "127.0.0.1", + port_request, + &resolve.device_policy, + &[], + None, + None, + None, + false, + false, + ) + }; + + let different = launch(serve_port::PortRequest::Explicit(11511)); + // The same port, and no port at all, still reuse the live service. + let same = launch(serve_port::PortRequest::Explicit(11510)); + let auto = launch(serve_port::PortRequest::Auto); + let _ = fs::remove_dir_all(root); + + let Err(error) = different else { + panic!("a port the live service does not serve must be refused") + }; + let message = error.to_string(); + assert!( + message.contains("on port 11510") && message.contains("port 11511 would not be used"), + "{message}" + ); + assert!(matches!(same?, ManagedSpawn::AlreadyRunning(_))); + assert!(matches!(auto?, ManagedSpawn::AlreadyRunning(_))); + Ok(()) + } + + #[test] + fn spawn_managed_engine_child_blocks_reuse_with_mismatched_recipe() -> Result<()> { + // A live service recorded with one recipe (e.g. a tool-call parser flag) + // must reject a relaunch requesting a different recipe rather than + // silently reusing the old server, and the error must not claim the + // mismatch is specifically about generation defaults when it could stem + // from any recipe field. + let (root, paths) = test_paths("dup-managed-recipe-mismatch"); + paths.ensure()?; + let mut existing = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-1000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11510, + "managed", + std::process::id(), + None, + None, + None, + ); + existing.status = "ready".to_owned(); + existing.engine_pid = Some(std::process::id()); + existing.engine_recipe_json = Some(serde_json::to_string(&EngineRecipeHint { + contract_version: rocm_engine_protocol::ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "lemonade".to_owned(), + required_flags: vec!["--tool-call-parser".to_owned(), "hermes".to_owned()], + ..EngineRecipeHint::default() + })?); + existing.write()?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + let requested_recipe = EngineRecipeHint { + contract_version: rocm_engine_protocol::ENGINE_RECIPE_CONTRACT_VERSION.to_owned(), + engine: "lemonade".to_owned(), + required_flags: vec!["--temperature".to_owned(), "0.5".to_owned()], + ..EngineRecipeHint::default() + }; + + let result = spawn_managed_engine_child( + &paths, + "lemonade", + "lemonade-qwen-2000", + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Explicit(11511), + &resolve.device_policy, + &[], + None, + None, + Some(&requested_recipe), + false, + false, + ); + let _ = fs::remove_dir_all(root); + + let Err(error) = result else { + panic!("mismatched recipe on a live service must be rejected") + }; + let message = error.to_string(); + assert!( + message.contains("different serve options"), + "message should describe the mismatch generically: {message}" + ); + assert!( + message.contains("recipe hint, tool-call parser, or generation defaults"), + "message should not single out generation defaults as the sole cause: {message}" + ); + Ok(()) + } + + #[test] + fn spawn_managed_engine_child_refuses_to_reuse_an_unauthenticated_service() -> Result<()> { + // `--require-api-key` used to be accepted and dropped on this path. The + // reuse branch returns before the flag is recorded and before the + // endpoint key is checked, so a caller demanding auth got a server that + // never had it, with no error. `rocm remote serve` then published that + // endpoint onto the tailnet and printed a freshly minted key under "the + // API key above is what stops anyone else calling it" — a false + // assurance about an endpoint the whole tailnet can reach. + // + // Asserted through `spawn_managed_engine_child` rather than against the + // guard's own arguments: the defect was the early return, so only the + // real call site can fail for it. + let (root, paths) = test_paths("dup-managed-unauthenticated-reuse"); + paths.ensure()?; + let mut existing = ManagedServiceRecord::new( + &paths, + "lemonade-qwen-3000", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + 11520, + "managed", + std::process::id(), + None, + None, + None, + ); + existing.status = "ready".to_owned(); + existing.engine_pid = Some(std::process::id()); + // The state that matters: live, matching, and serving without auth. + existing.requires_api_key = false; + existing.write()?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + + let result = spawn_managed_engine_child( + &paths, + "lemonade", + "lemonade-qwen-3001", + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Explicit(11520), + &resolve.device_policy, + &[], + None, + None, + None, + // The demand that used to be silently discarded. + true, + false, + ); + let _ = fs::remove_dir_all(root); + + let Err(error) = result else { + panic!( + "reusing an unauthenticated service must not satisfy `--require-api-key`; a \ + satisfied reuse leaves the endpoint open while the caller is told it is not" + ) + }; + let message = error.to_string(); + assert!( + message.contains("without authentication"), + "the refusal must say why it refused: {message}" + ); + // The printed hint must be the exact, directly-runnable command: + // `main.rs`'s `service_actions_require_yes_and_render_sandbox_result` + // proves this exact form (`rocm services stop --yes`) is the one + // the approval gate itself recommends. That running this form actually + // clears *this* refusal — a service reused without the key it is now + // being asked to require — is proven by + // `@id:serve-require-api-key-refuses-unauthenticated-reuse` + // (`tests/e2e-cucumber/features/model_serving.feature`, serve-24), + // which reproduces this exact condition, runs the printed hint, and + // asserts the record is left `stopped` — not by `stop_managed_services` + // in `tests/e2e-cucumber/tests/e2e.rs`, which runs this same + // invocation as best-effort teardown and never asserts success. + assert!( + message.contains("rocm services stop lemonade-qwen-3000 --yes"), + "the refusal must name the way out, with the service to stop, directly runnable with --yes: {message}" + ); + Ok(()) + } + + #[test] + fn a_managed_spawn_refuses_an_invalid_key_file_on_a_service_that_requires_one() -> Result<()> { + // Drives the real call site, not the guard's own arguments. The service + // was launched with `--require-api-key`, and its key file is present but + // invalid — so `requires_api_key` is true while `key_present` is false, + // which is the only way to reach the first branch. A test calling + // `ensure_public_service_has_endpoint_key` directly cannot catch a + // mis-wired call site, which is the defect that has occurred here twice. + // + // The flag is passed explicitly rather than inferred from the key file. + // Inferring it marked every public bind as having demanded auth, because + // a public bind always has a key file whether or not it asked for one. + let (root, paths) = test_paths("managed-spawn-invalid-key"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + // Non-empty but invalid (embedded CR/LF), so the file exists and is not + // merely empty (requires_api_key = true) but yields no usable key + // (key_present = false) — `endpoint_api_key_from_file` rejects embedded + // control characters. An empty seed here would let the post-condition + // below pass even if the real cleanup call never ran, since + // `endpoint_api_key` already reads back `None` for an empty file whether + // or not it was ever deleted. + endpoint_keys::store_endpoint_api_key( + &paths, + "lemonade-qwen-3000", + "bad-key\r\nX-Injected: value", + )?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + + let result = spawn_managed_engine_child( + &paths, + "lemonade", + "lemonade-qwen-3000", + "qwen", + &resolve, + // Loopback on purpose: the public-bind branch must not be what + // refuses this, or the test would pass with the guard disabled. + "127.0.0.1", + serve_port::PortRequest::Explicit(11512), + &resolve.device_policy, + &[], + None, + None, + None, + true, + false, + ); + + // This guard fires before any child is spawned, so `record.engine_pid` + // is still `None` when `spawn_managed_engine_child`'s cleanup arm calls + // `abandon_failed_managed_launch` — proving its pre-spawn branch + // (delete everything) runs from the real call site, not just from the + // `abandoned_launch_leaves_no_record_log_or_endpoint_key` test that + // calls the helper directly. + let manifest_path = paths.service_manifest_path("lemonade-qwen-3000"); + let log_path = paths.service_log_path("lemonade-qwen-3000"); + let engine_state_path = paths.service_engine_state_path("lemonade", "lemonade-qwen-3000"); + let key_file_path = endpoint_keys::endpoint_key_file_path(&paths, "lemonade-qwen-3000"); + let live = live_managed_services(&paths); + let _ = fs::remove_dir_all(root); + + let Err(error) = result else { + panic!("a service requiring a key must not spawn with an unusable key file") + }; + let message = error.to_string(); + assert!( + message.contains("--require-api-key"), + "the refusal must name the flag the service was launched with: {message}" + ); + assert!( + message.contains("without authentication"), + "the refusal must say what the risk is: {message}" + ); + assert!( + !manifest_path.exists(), + "a pre-spawn failure must not leave a record" + ); + assert!( + !log_path.exists(), + "a pre-spawn failure must not leave a log" + ); + assert!( + !engine_state_path.exists(), + "a pre-spawn failure must not leave engine state" + ); + assert!( + !key_file_path.exists(), + "a pre-spawn failure must delete the unusable key file from disk, not merely \ + leave it unreadable as a key" + ); + assert!( + live.is_empty(), + "a pre-spawn failure must leave no reservation" + ); + Ok(()) + } + + #[test] + fn a_failed_spawn_past_the_first_write_terminates_the_child_and_keeps_an_abandoned_record() + -> Result<()> { + // Drives the real call site's *post*-spawn cleanup branch, which + // neither `abandoned_launch_*` test reaches (they call + // `abandon_failed_managed_launch` directly). An engine name this + // binary does not build in makes the real spawned child (a second + // `rocm` process) exit almost immediately via + // `run_builtin_engine_serve_http`'s `other => bail!(...)` arm, after + // `launch_managed_engine_child_locked` has already set + // `record.engine_pid` — so cleanup must terminate (already-dead) the + // child, keep the record and log for inspection, mark the record + // `stopped` (a `failed` one would be respawned by rocmd recovery), and + // clear only the endpoint key. + // + // The child always dies near-instantly, but `spawn_managed_engine_child` + // only reports that within a fixed startup-check window. On a loaded or + // antivirus-scanning Windows CI box, a fresh debug `rocm.exe` can + // occasionally be slow enough to start that the window is missed even + // though the child is already doomed — so poll by retrying a fresh + // attempt (its own service id/port, so no stale manifest collides) up + // to a deadline instead of asserting on one race-prone attempt. + let deadline = std::time::Instant::now() + Duration::from_secs(10); + let mut attempt: u16 = 0; + let (root, paths, service_id, result) = loop { + attempt += 1; + let (root, paths) = + test_paths(&format!("managed-spawn-unknown-engine-cleanup-{attempt}")); + paths.ensure()?; + let service_id = format!("no-such-engine-qwen-{}", 4000 + attempt); + endpoint_keys::store_endpoint_api_key(&paths, &service_id, "test-key")?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + + let result = spawn_managed_engine_child( + &paths, + "no-such-engine", + &service_id, + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Explicit(11_514 + attempt), + &resolve.device_policy, + &[], + None, + None, + None, + false, + false, + ); + + if result.is_ok() && std::time::Instant::now() < deadline { + let _ = fs::remove_dir_all(&root); + continue; + } + break (root, paths, service_id, result); + }; + + let manifest_path = paths.service_manifest_path(&service_id); + let log_path = paths.service_log_path(&service_id); + let persisted = load_managed_services(&paths) + .expect("re-read manifests") + .into_iter() + .find(|candidate| candidate.service_id == service_id); + let key = endpoint_keys::endpoint_api_key(&paths, &service_id); + let live = live_managed_services(&paths); + + let Err(error) = result else { + panic!("an engine this binary does not build in must not report success") + }; + let message = error.to_string(); + assert!( + message.contains("managed engine exited immediately"), + "the failure must come from the spawned child dying, not an earlier guard: {message}" + ); + assert!( + manifest_path.exists(), + "a post-spawn failure must keep the record for inspection" + ); + assert!( + log_path.exists(), + "a post-spawn failure must keep the log for inspection" + ); + let persisted = + persisted.expect("a post-spawn failure must leave the record on disk, not delete it"); + // `stopped`, not `failed`: rocmd's recovery restarts a `failed` record + // without a stop marker, which would respawn a launch the CLI just + // reported as failed (pinned on the rocmd side by + // `abandoned_launch_record_is_not_a_recovery_candidate`). + assert_eq!( + persisted.status, "stopped", + "a post-spawn failure must mark the record abandoned, not leave it stuck" + ); + assert_eq!( + key, None, + "a post-spawn failure must still clear the endpoint key" + ); + assert!( + live.is_empty(), + "an abandoned record must not still reserve its port" + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn select_gpu_indices_under_launch_lock_blocks_until_the_holder_releases() -> Result<()> { + // Exercises the real lock-acquisition function production callers use + // (`select_gpu_indices_under_launch_lock`) against a lock another thread + // actually holds, so a refactor that dropped the lock early would fail + // here. `spawn_managed_engine_child` is then run for real, but only via + // its `AlreadyLive` short-circuit (a pre-seeded duplicate record), so no + // child process is ever launched. The holder writes nothing: this does + // not show a write made under the lock being observed. + use std::sync::{Arc, Barrier}; + use std::time::Instant; + + let (root, paths) = test_paths("launch-lock-holds-across-spawn"); + paths.ensure()?; + write_live_service_record(&paths, "lemonade-held-1", "held-canonical", 11_520, "ready")?; + + let barrier = Arc::new(Barrier::new(2)); + let hold_for = Duration::from_millis(250); + + let holder_paths = paths.clone(); + let holder_barrier = Arc::clone(&barrier); + let holder = thread::spawn(move || -> Result<()> { + let _lock = rocm_core::FileLock::acquire(holder_paths.managed_launch_lock_path())?; + holder_barrier.wait(); + thread::sleep(hold_for); + Ok(()) + }); + + barrier.wait(); + let started = Instant::now(); + let (gpu_indices, launch_lock) = crate::gpu_policy::select_gpu_indices_under_launch_lock( + &paths, + true, + None, + || None, + None, + None, + )?; + let waited = started.elapsed(); + + let resolve = ResolveModelResponse { + canonical_model_id: "held-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + let spawn = spawn_managed_engine_child( + &paths, + "lemonade", + "lemonade-held-2", + "held-model", + &resolve, + DEFAULT_LOCAL_HOST, + serve_port::PortRequest::Auto, + &resolve.device_policy, + &gpu_indices, + None, + None, + None, + false, + false, + ); + drop(launch_lock); + holder.join().expect("holder thread")?; + let _ = fs::remove_dir_all(root); + + // `thread::sleep` only guarantees *at least* `hold_for`, but under a + // loaded host (e.g. a full parallel `cargo test` run on WSL2) timer + // coalescing can make the measured wait a hair short of the nominal + // hold. Allow a small margin so the test still fails loudly if the + // call returns early, without being flaky on actual block duration. + let margin = Duration::from_millis(20); + assert!( + waited + margin >= hold_for, + "select_gpu_indices_under_launch_lock returned before the holder released the lock: waited {waited:?}, held {hold_for:?}" + ); + match spawn? { + ManagedSpawn::AlreadyRunning(report) => { + assert_eq!(report.service_id, "lemonade-held-1"); + } + ManagedSpawn::Spawned { .. } => { + panic!( + "expected the pre-seeded AlreadyLive record to short-circuit, not a real spawn" + ) + } + } + Ok(()) + } + + #[test] + #[cfg(not(windows))] + fn launch_releases_the_leased_port_before_spawning_the_child() -> Result<()> { + // `lease.release()` must run before the child is spawned, not after, + // or the engine is handed a port the CLI is still holding open. + // Proven via the injected `spawn_override` seam: it asserts a fresh + // bind on the leased port succeeds at the exact instant production + // would spawn, which only holds if the lease was already released. + let (root, paths) = test_paths("launch-releases-port-before-spawn"); + paths.ensure()?; + fs::create_dir_all(paths.services_dir())?; + + // Ephemeral, not a fixed number in the automatic range, which a real + // `rocm serve` on the developer's machine could be holding. + let port = std::net::TcpListener::bind(("127.0.0.1", 0))? + .local_addr()? + .port(); + let lease = rocm_core::lease_loopback_port(port)?; + let mut record = ManagedServiceRecord::new( + &paths, + "lemonade-release-before-spawn", + "lemonade", + "qwen", + "qwen-canonical", + "127.0.0.1", + port, + "managed", + 0, + None, + None, + None, + ); + record.write()?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + + let bind_succeeded = std::cell::Cell::new(false); + let spawn_override: &dyn Fn(&mut ProcessCommand) -> io::Result = + &|_command| { + // Retry briefly: a sibling test's concurrent fork can hold a copy + // of the just-released socket until its exec closes it. + let deadline = std::time::Instant::now() + Duration::from_secs(2); + while !bind_succeeded.get() && std::time::Instant::now() < deadline { + bind_succeeded.set(std::net::TcpListener::bind(("127.0.0.1", port)).is_ok()); + } + // A long-lived no-op child stands in for the engine: `_command` + // carries real serve args this test binary cannot execute as + // itself (see `managed_launch_cleanup_sleeper_child`). + ProcessCommand::new(std::env::current_exe().expect("test binary")) + .arg(sleeper_test_name()) + .arg("--exact") + .arg("--ignored") + .env("ROCM_TEST_SLEEPER_CHILD", "1") + .stdin(Stdio::null()) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + }; + + let result = launch_managed_engine_child_locked( + &paths, + &mut record, + lease, + "lemonade", + "lemonade-release-before-spawn", + &resolve, + "127.0.0.1", + &resolve.device_policy, + &[], + None, + None, + None, + Some(spawn_override), + ); + if let Ok(child_pid) = result.as_ref() { + let _ = rocm_core::terminate_process(*child_pid); + } + let _ = fs::remove_dir_all(root); + + assert!( + bind_succeeded.get(), + "the port must already be bindable (released) by the time the child is spawned" + ); + result?; + Ok(()) + } +} diff --git a/apps/rocm/src/service_launch/start.rs b/apps/rocm/src/service_launch/start.rs new file mode 100644 index 0000000000..06cb93697e --- /dev/null +++ b/apps/rocm/src/service_launch/start.rs @@ -0,0 +1,487 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The background launch path: [`start_managed_service`] and the plain, +//! line-by-line rendering of its [`ManagedLaunchReport`]. + +use super::endpoint_auth::drop_orphaned_endpoint_key_on_already_running; +use super::readiness::{ + died_during_wait, record_wait_outcome, service_pid_is_alive, wait_for_record_ready, + wait_on_existing_service, +}; +#[cfg(feature = "e2e-oom-fault-injection")] +use super::readiness::{e2e_simulate_oom_launch, simulate_oom_managed_launch}; +use super::spawn::{ManagedLaunchReport, ManagedSpawn, spawn_managed_engine_child}; +use crate::serve_port; +use crate::{record_cli_audit_event, status_for_readiness}; +use anyhow::Result; +use rocm_core::format_http_base_url; +use rocm_core::{AppPaths, EndpointReadiness}; +use rocm_engine_protocol::{DevicePolicy, EngineRecipeHint, ResolveModelResponse}; +use std::fmt::Write as _; +#[cfg(windows)] +use std::thread; +use std::time::Duration; + +/// Launch a managed service in the background: run the locked allocation and +/// spawn transaction, release the launch lock once the record is published, +/// then wait for HTTP readiness. +/// +/// Drops the endpoint key `serve()` freshly staged for this attempt if the +/// launch fails before any `ManagedServiceRecord` exists to own it (port +/// reserved/in use, range exhausted, other bind error) — such a failure +/// returns before `abandon_failed_managed_launch` ever runs, so nothing else +/// would clear it. Scoped to the spawn call only, mirroring +/// [`run_attached_service`](super::run_attached_service)'s identical guard: a later +/// failure (the readiness wait, `record.write()`) leaves a record a live child actually owns, so it +/// must not also wipe that key. +#[allow(clippy::too_many_arguments)] +pub(crate) fn start_managed_service( + paths: &AppPaths, + engine: &str, + service_id: &str, + requested_model: &str, + resolve: &ResolveModelResponse, + host: &str, + port_request: serve_port::PortRequest, + device_policy: &DevicePolicy, + gpu_indices: &[u32], + runtime_id: Option<&str>, + env_id: Option<&str>, + engine_recipe: Option<&EngineRecipeHint>, + endpoint_api_key: Option<&str>, + launch_lock: rocm_core::FileLock, + require_api_key: bool, + gpu_refusal_waived_for_reuse: bool, + wait_until_ready: bool, + on_wait_begin: &mut dyn FnMut(&str, u16), + on_wait_tick: &mut dyn FnMut(Duration), +) -> Result { + #[cfg(feature = "e2e-oom-fault-injection")] + if e2e_simulate_oom_launch() { + return simulate_oom_managed_launch( + paths, + engine, + service_id, + requested_model, + resolve, + host, + port_request + .explicit() + .unwrap_or(rocm_core::DEFAULT_LOCAL_PORT), + device_policy, + gpu_indices, + runtime_id, + env_id, + ); + } + let (mut record, child_pid) = match spawn_managed_engine_child( + paths, + engine, + service_id, + requested_model, + resolve, + host, + port_request, + device_policy, + gpu_indices, + runtime_id, + env_id, + engine_recipe, + require_api_key, + gpu_refusal_waived_for_reuse, + ) + .inspect_err(|_| { + drop_orphaned_endpoint_key_on_already_running(paths, service_id, endpoint_api_key); + })? { + ManagedSpawn::AlreadyRunning(report) => { + // `--wait` applies to a service that is still starting too: the + // natural retry after `rocm serve m` printed `starting`. + drop(launch_lock); + return wait_on_existing_service( + paths, + report, + wait_until_ready, + on_wait_begin, + on_wait_tick, + ); + } + ManagedSpawn::Spawned { record, child_pid } => (*record, child_pid), + }; + // The transaction resolved the request to one concrete port; everything + // below — readiness, the endpoint URL, the audit line — uses that. + let port = record.port; + // The claiming service record is now persisted, so the selected GPU is + // visible to any concurrent auto-selection. Release the launch lock before + // the readiness wait below, which can block for many seconds — holding it + // that long would needlessly serialize unrelated serves. + drop(launch_lock); + + #[cfg(windows)] + thread::sleep(Duration::from_millis(200)); + + if wait_until_ready { + on_wait_begin(service_id, port); + } + let readiness = wait_for_record_ready( + paths, + &record, + wait_until_ready, + endpoint_api_key, + on_wait_tick, + ); + if let Some(error) = died_during_wait( + paths, + service_id, + wait_until_ready, + readiness, + service_pid_is_alive, + ) { + return Err(error); + } + let launch_status = status_for_readiness(readiness); + record_wait_outcome( + paths, + &mut record, + launch_status, + readiness == EndpointReadiness::Serving, + )?; + let endpoint_url = format!("{}/v1", format_http_base_url(host, port)); + record_cli_audit_event( + paths, + "service", + "managed_service_launch", + "info", + format!( + "launched managed service engine={} model={} endpoint={} readiness={}", + engine, resolve.canonical_model_id, endpoint_url, launch_status + ), + Some(service_id), + ); + Ok(ManagedLaunchReport { + service_id: service_id.to_owned(), + endpoint_url, + status: launch_status.to_owned(), + already_running: false, + child_pid: Some(child_pid), + log_path: Some(record.log_path), + manifest_path: Some(record.manifest_path), + }) +} + +/// Reproduce the original plain, line-by-line managed-launch output. Used for +/// non-interactive output (piped, CI, the chat assistant's `serve --managed`), +/// where the animated summary is inappropriate. The interactive path renders the +/// summary table via [`crate::serve_summary`] instead. +pub(crate) fn print_managed_launch_plain( + report: &ManagedLaunchReport, + endpoint_api_key: Option<&str>, +) { + if report.already_running { + println!("managed service already running"); + println!(" service_id: {}", report.service_id); + println!(" endpoint: {}", report.endpoint_url); + println!(" status: {}", report.status); + println!(" note: existing service detected; no second process spawned"); + return; + } + println!("managed service launched"); + println!(" service_id: {}", report.service_id); + if let Some(child_pid) = report.child_pid { + println!(" process_pid: {child_pid}"); + } + println!(" endpoint: {}", report.endpoint_url); + if let Some(key) = endpoint_api_key { + // Intentional one-time display of a freshly generated API key to the + // terminal so the user can copy it — the designed delivery channel + // documented on `render_endpoint_client_config`, not a log. The tag below + // is currently inert (Rust's CodeQL pack has no AlertSuppression.ql yet — + // github/codeql#21637) but will start working once that lands, since the + // tag must be the single line immediately before the flagged code. + print!( + // codeql[rust/cleartext-logging] + "{}", + render_endpoint_client_config(&report.endpoint_url, key) + ); + } + if let Some(log_path) = report.log_path.as_deref() { + println!(" log_path: {}", log_path.display()); + } + if let Some(manifest_path) = report.manifest_path.as_deref() { + println!(" manifest_path: {}", manifest_path.display()); + } + println!(" readiness: {}", report.status); +} + +/// The client configuration to show when a `--wait` begins, so a key generated for +/// a public bind reaches the user even if the wait is interrupted or fails. +/// `None` unless the wait is on the launch that key belongs to: an attach to an +/// already-starting server (`waiting_on` differs from `service_id`) answers to +/// its own key, which is not shown here. +pub(crate) fn wait_begin_client_config( + waiting_on: &str, + service_id: &str, + host: &str, + port: u16, + api_key: Option<&str>, +) -> Option { + let key = api_key.filter(|_| waiting_on == service_id)?; + let endpoint_url = format!("{}/v1", format_http_base_url(host, port)); + Some(render_endpoint_client_config(&endpoint_url, key)) +} + +/// Render the one-time secure client configuration for a public, authenticated +/// endpoint. This is the *intended* channel for delivering the key to the user +/// (unlike logs/status, which must never contain it) — it prints the key once at +/// launch alongside a ready-to-use example. Callers only invoke this for a +/// non-loopback bind that generated/received a key. +pub(super) fn render_endpoint_client_config(endpoint_url: &str, api_key: &str) -> String { + let mut out = String::new(); + let _ = writeln!(out, " api key: {api_key}"); + let _ = writeln!( + out, + " note: this key is shown only now - clients must send `Authorization: Bearer `" + ); + let _ = writeln!( + out, + " example: curl -H \"Authorization: Bearer {api_key}\" {endpoint_url}/models" + ); + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::endpoint_keys; + #[cfg(not(windows))] + use crate::service_launch::run_attached_service; + #[cfg(not(windows))] + use crate::service_launch::spawn::TEST_SPAWN_HOOK; + use crate::tests::test_paths; + use rocm_core::ManagedServiceRecord; + use std::fs; + #[cfg(not(windows))] + use std::io; + + #[test] + fn wait_begin_shows_the_key_only_for_the_launch_it_belongs_to() { + let own = wait_begin_client_config("svc-new", "svc-new", "0.0.0.0", 8000, Some("k-secret")) + .expect("own launch shows its key"); + // No `{own}` in the failure messages: the text carries the dummy key, + // and formatting it into an assert is what CodeQL flags as logging it. + assert!(own.contains("api key: k-secret"), "key line missing"); + assert!( + own.contains("http://0.0.0.0:8000/v1/models"), + "models url missing" + ); + // Attached to an already-starting server: its key is not this one. + assert!( + wait_begin_client_config("svc-old", "svc-new", "0.0.0.0", 8000, Some("k-secret")) + .is_none() + ); + // Loopback binds have no key to show. + assert!(wait_begin_client_config("svc-new", "svc-new", "127.0.0.1", 8000, None).is_none()); + } + + #[test] + fn endpoint_client_config_shows_key_once_with_bearer_guidance() { + let rendered = render_endpoint_client_config("http://0.0.0.0:11435/v1", "secret-123"); + // Dummy literal exercising the intentional one-time key display documented + // on `render_endpoint_client_config`; not a real credential or a log write. + // codeql[rust/cleartext-logging] + assert!(rendered.contains("secret-123"), "{rendered}"); + // codeql[rust/cleartext-logging]: see rationale above. + assert!(rendered.contains("Authorization: Bearer"), "{rendered}"); + // codeql[rust/cleartext-logging]: see rationale above. + assert!(rendered.contains("shown only now"), "{rendered}"); + } + + #[test] + fn failed_port_reservation_leaves_no_orphaned_endpoint_key() -> Result<()> { + // serve() stores the new attempt's endpoint key before calling + // start_managed_service. If spawn_managed_engine_child fails inside + // resolve_managed_port_in_transaction (explicit port already reserved + // by another live service), it returns before any + // ManagedServiceRecord exists for this attempt, so + // abandon_failed_managed_launch never runs. Driving the real function + // end to end (not a simulated call to + // drop_orphaned_endpoint_key_on_already_running) proves + // `start_managed_service`'s own `.inspect_err` actually wires the cleanup, mirroring how + // attached_launch_failing_before_spawn_leaves_no_orphaned_endpoint_key + // drives run_attached_service for the foreground path. + let (root, paths) = test_paths("orphaned-key-on-reserved-port"); + paths.ensure()?; + let mut other = ManagedServiceRecord::new( + &paths, + "lemonade-llama-1000", + "lemonade", + "llama", + "llama-canonical", + "127.0.0.1", + 11513, + "managed", + std::process::id(), + None, + None, + None, + ); + other.status = "ready".to_owned(); + other.engine_pid = Some(std::process::id()); + other.write()?; + + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + + let service_id = "lemonade-qwen-2000"; + endpoint_keys::store_endpoint_api_key(&paths, service_id, "test-key")?; + let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + + let result = start_managed_service( + &paths, + "lemonade", + service_id, + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Explicit(11513), + &resolve.device_policy, + &[], + None, + None, + None, + Some("test-key"), + launch_lock, + false, + false, + false, + &mut |_, _| {}, + &mut |_elapsed| {}, + ); + let Err(error) = result else { + let _ = fs::remove_dir_all(root); + panic!("explicit port already reserved by a live service must be rejected") + }; + assert!( + error + .to_string() + .contains("already reserved by managed service"), + "unexpected error: {error}" + ); + + assert!( + endpoint_keys::endpoint_api_key(&paths, service_id).is_none(), + "cleanup must drop the key this failed attempt staged before launching" + ); + + let _ = fs::remove_dir_all(root); + Ok(()) + } + + /// The launch lock must still be held when the child is spawned: it is what + /// serializes allocation against publication. Observed through the spawn + /// hook, since releasing it early changes nothing else a test can see. + #[cfg(not(windows))] + #[test] + fn start_and_attached_launches_hold_the_launch_lock_until_the_spawn() -> Result<()> { + let resolve = ResolveModelResponse { + canonical_model_id: "qwen-canonical".to_owned(), + served_model_id: None, + task: "chat".to_owned(), + source: "hf".to_owned(), + revision: "main".to_owned(), + loader: "llama.cpp".to_owned(), + trust_remote_code: false, + chat_template_mode: "auto".to_owned(), + dtype: "auto".to_owned(), + device_policy: DevicePolicy::GpuPreferred, + estimated_memory: "unknown".to_owned(), + launch_defaults: serde_json::json!({}), + engine_recipe: None, + warnings: Vec::new(), + }; + for attached in [false, true] { + let (root, paths) = test_paths("launch-lock-held-until-spawn"); + paths.ensure()?; + let lock_path = paths.managed_launch_lock_path(); + let lock_was_held = std::rc::Rc::new(std::cell::Cell::new(false)); + let observed = lock_was_held.clone(); + TEST_SPAWN_HOOK.with(|hook| { + *hook.borrow_mut() = Some(std::rc::Rc::new(move |_command| { + observed.set( + rocm_core::FileLock::acquire_bounded(&lock_path, Duration::ZERO).is_err(), + ); + Err(io::Error::other("spawn intercepted by test")) + })); + }); + let launch_lock = rocm_core::FileLock::acquire(paths.managed_launch_lock_path())?; + let result = if attached { + run_attached_service( + &paths, + "lemonade", + "lemonade-lock-attached", + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Auto, + &[], + None, + None, + None, + launch_lock, + false, + false, + |_, _| unreachable!("the spawn fails before a Ctrl-C could stop it"), + ) + } else { + start_managed_service( + &paths, + "lemonade", + "lemonade-lock-start", + "qwen", + &resolve, + "127.0.0.1", + serve_port::PortRequest::Auto, + &resolve.device_policy, + &[], + None, + None, + None, + None, + launch_lock, + false, + false, + false, + &mut |_, _| {}, + &mut |_elapsed| {}, + ) + .map(|_| ()) + }; + TEST_SPAWN_HOOK.with(|hook| *hook.borrow_mut() = None); + let _ = fs::remove_dir_all(root); + assert!( + format!("{:#}", result.expect_err("the intercepted spawn fails")) + .contains("spawn intercepted by test") + ); + assert!( + lock_was_held.get(), + "the launch lock must be held at spawn (attached: {attached})" + ); + } + Ok(()) + } +} diff --git a/apps/rocm/src/service_launch/test_support.rs b/apps/rocm/src/service_launch/test_support.rs new file mode 100644 index 0000000000..70a5504e09 --- /dev/null +++ b/apps/rocm/src/service_launch/test_support.rs @@ -0,0 +1,103 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Fixtures shared by the tests of more than one launch submodule. + +use anyhow::Result; +use rocm_core::{AppPaths, ManagedServiceRecord}; +#[cfg(not(windows))] +use std::thread; +#[cfg(not(windows))] +use std::time::Duration; + +/// A record with every reuse-relevant field set to the "reusable" value, so +/// each test that uses it breaks exactly one term and nothing else. +pub(super) fn reusable_record(paths: &AppPaths) -> ManagedServiceRecord { + let mut record = ManagedServiceRecord::new( + paths, + "vllm-qwen-1000", + "vllm", + "qwen", + "qwen-canonical".to_owned(), + "127.0.0.1", + 8000, + "managed", + 0, + None, + None, + Some("gpu_required".to_owned()), + ); + record.engine_recipe_json = None; + record.requires_api_key = true; + record +} + +pub(super) fn write_live_service_record( + paths: &AppPaths, + service_id: &str, + canonical_model_id: &str, + port: u16, + status: &str, +) -> Result { + let mut record = ManagedServiceRecord::new( + paths, + service_id, + "lemonade", + canonical_model_id, + canonical_model_id.to_owned(), + rocm_core::DEFAULT_LOCAL_HOST, + port, + "managed", + std::process::id(), + None, + None, + None, + ); + record.status = status.to_owned(); + record.engine_pid = Some(std::process::id()); + record.write()?; + Ok(record) +} + +/// The helper the cleanup tests re-run this test binary as, to get a live child +/// process that outlives the spawn they are checking. It is `#[ignore]`d so a +/// normal run skips it, and it only sleeps when the tests that spawn it set +/// `ROCM_TEST_SLEEPER_CHILD`. They select it by [`sleeper_test_name`]. +#[test] +#[cfg(not(windows))] +#[ignore = "spawned as a child process by abandoned_launch_terminates_the_spawned_child"] +fn managed_launch_cleanup_sleeper_child() { + if std::env::var_os("ROCM_TEST_SLEEPER_CHILD").is_some() { + thread::sleep(Duration::from_mins(2)); + } +} + +/// The name libtest lists [`managed_launch_cleanup_sleeper_child`] under, which +/// is what `--exact` has to be given to run it alone. +/// +/// It is read off the function item itself, so renaming the function or moving +/// this module changes it with them and the three tests that spawn the helper +/// cannot be left naming a test that no longer exists. libtest names a test by +/// its path inside the crate, while the item path starts with the crate name, +/// which is dropped. The assertions check that against the module path as well. +#[cfg(not(windows))] +pub(super) fn sleeper_test_name() -> String { + let item_path = std::any::type_name_of_val(&managed_launch_cleanup_sleeper_child); + let name = item_path + .split_once("::") + .map_or(item_path, |(_krate, rest)| rest); + let module = module_path!() + .split_once("::") + .map_or("", |(_krate, rest)| rest); + assert_eq!( + name, + format!("{module}::managed_launch_cleanup_sleeper_child"), + "the sleeper helper is no longer where its name is derived from" + ); + assert!( + name.starts_with("service_launch::"), + "libtest names tests by their path inside the crate: {name}" + ); + name.to_owned() +} diff --git a/apps/rocm/src/service_records.rs b/apps/rocm/src/service_records.rs index 7be31d9579..5fe013a611 100644 --- a/apps/rocm/src/service_records.rs +++ b/apps/rocm/src/service_records.rs @@ -350,7 +350,7 @@ fn describe_hours(hours: u64) -> String { /// /// "No `.json` beside it" is also exactly what a *launch in progress* looks /// like: `serve` writes the 0600 `.endpoint-key` before -/// `spawn_managed_engine_child` (in `service_launch.rs`) writes the first +/// `spawn_managed_engine_child` (in `service_launch/spawn.rs`) writes the first /// `.json`. Nothing about /// the file distinguishes the two cases — under `--any-age` there is no age gate /// left to ask — so this function does not try. It is only ever reached with the @@ -633,9 +633,9 @@ fn apply_service_prune_plan( /// leftover, and that is also what a launch looks like mid-flight: `serve` /// writes the 0600 `.endpoint-key` at the top of the managed path, and the /// first `.json` only lands at `record.write()` inside -/// `spawn_managed_engine_child` (in `service_launch.rs`). Between those two writes a live server's -/// secret is indistinguishable from a leftover, and `--any-age` removes the age -/// gate that used to hide the window, so a concurrent +/// `spawn_managed_engine_child` (in `service_launch/spawn.rs`). Between those two +/// writes a live server's secret is indistinguishable from a leftover, and +/// `--any-age` removes the age gate that used to hide the window, so a concurrent /// `rocm services prune --any-age --yes` deleted it. /// /// `serve` already holds `managed_launch_lock_path` across the whole of that diff --git a/apps/rocm/src/storage.rs b/apps/rocm/src/storage.rs index 3983cca0ba..916b2b19a1 100644 --- a/apps/rocm/src/storage.rs +++ b/apps/rocm/src/storage.rs @@ -484,7 +484,7 @@ pub(crate) fn build_report(paths: &AppPaths, config: &RocmCliConfig) -> Result.log` beside the manifest. `ManagedServiceRecord::new` // only *computes* that path (via `AppPaths::service_log_path`) - it // writes nothing. The file is created by the launch site, - // `spawn_managed_engine_child` (in `service_launch.rs`) for + // `spawn_managed_engine_child` (in `service_launch/spawn.rs`) for // `rocm serve --managed` and `supervise_service` in `rocmd` on the // supervised/recovery path. What *fills* it differs by platform, so the // redirect cannot be stated unqualified: `supervise_service` redirects diff --git a/apps/rocm/src/test_support.rs b/apps/rocm/src/test_support.rs index 1987de3679..b784808e6d 100644 --- a/apps/rocm/src/test_support.rs +++ b/apps/rocm/src/test_support.rs @@ -389,7 +389,7 @@ pub(crate) fn owned_argv(args: &[&str]) -> Vec { /// Build an `AutomationRuntimeState` for the background-helper tests (the /// no-double-spawn guard in `background_helper.rs` and the autostart tests in -/// `service_launch.rs`). +/// `service_launch/helper_autostart.rs`). pub(crate) fn runtime_state(running: bool, daemon_pid: u32) -> AutomationRuntimeState { AutomationRuntimeState { running, diff --git a/apps/rocmd/src/supervise.rs b/apps/rocmd/src/supervise.rs index eb594e2167..70e7b23a0f 100644 --- a/apps/rocmd/src/supervise.rs +++ b/apps/rocmd/src/supervise.rs @@ -1852,7 +1852,7 @@ mod tests { } /// Mirrors `launch_releases_the_leased_port_before_spawning_the_child` in - /// `apps/rocm/src/service_launch.rs`: the probe lease on the recorded port + /// `apps/rocm/src/service_launch/spawn.rs`: the probe lease on the recorded port /// must already be released by the time the recovery supervisor is /// spawned, or the supervisor's own bind of that port would fail. #[test] diff --git a/apps/rocmd/src/watchers/server_recover.rs b/apps/rocmd/src/watchers/server_recover.rs index f43402a4e1..aac0a9e9d5 100644 --- a/apps/rocmd/src/watchers/server_recover.rs +++ b/apps/rocmd/src/watchers/server_recover.rs @@ -798,8 +798,8 @@ mod tests { /// The record `rocm serve` keeps when it abandons a launch whose child /// already ran (`abandon_failed_managed_launch` in - /// `apps/rocm/src/service_launch.rs`) must not be respawned: the CLI has just - /// reported that launch as failed. The producing side is + /// `apps/rocm/src/service_launch/launch_transaction.rs`) must not be respawned: + /// the CLI has just reported that launch as failed. The producing side is /// `abandoned_launch_terminates_the_spawned_child` in that file; both go /// through `ManagedServiceRecord::mark_launch_abandoned`. #[test] diff --git a/docs/architecture.md b/docs/architecture.md index ad51f2aaf6..f258367915 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -17,7 +17,7 @@ A second check runs the other way: `cargo xtask check-architecture-doc` ([`archi New subcommands and subsystems default to their own file from day one — they should not grow inside `main.rs`/`lib.rs` waiting for a future extraction pass. Two extraction patterns already exist in the codebase; use whichever fits: - **Full domain extraction** — a subsystem's domain implementation moves into its own file that owns its own types (structs/enums), not just relocated functions; that ownership is usually what distinguishes this pattern from mechanical relocation below, though the Phase 6a variant described next shows ownership alone doesn't fully tell them apart — every `apps/rocm` module is a private `mod x;` regardless of pattern, so module privacy alone doesn't tell the two apart either. Full-domain-extraction modules are typically reached via qualified paths (e.g. `comfyui::render_status(...)`); mechanically relocated modules with no owned types are reached the same way (e.g. [`endpoint_keys.rs`](../apps/rocm/src/endpoint_keys.rs), [`logging.rs`](../apps/rocm/src/logging.rs)). The exception — reached unqualified through `use crate::x::{...}` re-imports, matching their dispatch call sites' pre-extraction form — is limited to the clusters that moved as a dispatch-adjacent group; the block of `use crate::x::{...}` re-imports near the top of [`main.rs`](../apps/rocm/src/main.rs) is the current list of the modules it reaches that way (a few of them re-imported for reasons other than dispatch, as the comment above the block explains). [`service_launch.rs`](../apps/rocm/src/service_launch.rs) (as of Phase 6b, ROCMAI-91) uses the same unqualified re-import form for its cross-module callers, but it is not a dispatch-adjacent cluster in the same sense — one of its callers, `automations.rs`, is not a dispatch call site, and its pre-extraction form was a qualified `crate::` call rather than an unqualified import, so the re-import is new rather than kept. This call-site form, not type ownership or dispatch-fn placement, is what actually separates the Phase 6a variant below from full domain extraction. Where a subsystem has a dedicated clap subcommand, its command *enum* usually lives in [`cli.rs`](../apps/rocm/src/cli.rs) as part of the `Command`/`Cli` definition (e.g. `ComfyuiCommand`, `RuntimesCommand`), the way [`apps/rocmd`](../apps/rocmd/src/cli.rs) keeps its command tree in a `cli.rs` of its own — but the dispatch function, the match over that tree, does not stay in `main.rs` with it: `comfyui::run` lives in `comfyui.rs`, reached by the qualified path full domain extraction uses, while `engines()` and `runtimes()` moved out to `engines_cmd.rs`/`runtimes_cmd.rs` respectively, following the mechanical-relocation re-import form below. Only the top-level `dispatch()`, the match that sends each `Command` variant to its handler, stays in `main.rs`. Not every domain-extracted module has a dedicated command enum at all (`providers.rs` has none; it's invoked from the existing chat/config command flows). In library crates (`crates/rocm-core`) the module is `pub mod x;` plus a `pub use x::{...};` re-export, since it's part of the crate's public API. This is the default for new subsystems. Examples: [`apps/rocm/src/therock.rs`](../apps/rocm/src/therock.rs) (TheRock SDK install orchestration, with its update planning and other clusters in `therock/` child modules and no command enum of its own), [`comfyui.rs`](../apps/rocm/src/comfyui.rs) (`ComfyuiCommand`), [`providers.rs`](../apps/rocm/src/providers.rs) (no dedicated command enum); `crates/rocm-core`'s [`diagnose.rs`](../crates/rocm-core/src/diagnose.rs)/[`examine.rs`](../crates/rocm-core/src/examine.rs). -- **Mechanical relocation** — a `pub(crate) fn` moves out verbatim, with shared types/config staying at the crate root and reached via `crate::`. Used for dispatch-adjacent clusters where a minimal, easy-to-review diff matters more than full extraction. Example: [`apps/rocm/src/update_cmd.rs`](../apps/rocm/src/update_cmd.rs). Variant: when the cluster is large enough to own types (part of `main.rs`'s Phase 6a split, ROCMAI-82, `main.rs`'s Phase 6b split, ROCMAI-91, `main.rs`'s Phase 6c split, ROCMAI-89, and Phase 6e's follow-up, ROCMAI-726), those types move with it instead of staying in `main.rs` — [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs)'s types are used nowhere else, but [`driver_install.rs`](../apps/rocm/src/driver_install.rs)'s `DriverInstallResult`/`DriverInstallError` are `pub(crate)` and read from [`install_cmd.rs`](../apps/rocm/src/install_cmd.rs)'s `install()` (`result.output`/`result.executed`, `error.source`/`error.executed`); [`freeform_execution.rs`](../apps/rocm/src/freeform_execution.rs)'s `FreeformExecution` is `pub(crate)` and read from `main.rs`'s `execute_freeform_next_action()` (`execution.action`) — it was already `pub(crate)` before the split, unlike the two functions that return/consume it, which widened from module-private; [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs)'s `ServeEngineSelection`/`ServeArgs` both stay `pub(crate)` and are used from `main.rs` (`ServeArgs`) and [`model_fit.rs`](../apps/rocm/src/model_fit.rs) (`assess_model_for_host` calls `select_serve_engine` and reads the resulting `ServeEngineSelection`'s `.engine`/`.source` fields) — and `ServeGenerationDefaults` moved out with the engine-recipe overrides to [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs), where it and its fields are `pub(crate)` because `serve()` builds it; [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s `AttachOutcome`/`RawModeGuard`/`ManagedSpawn`/`AutostartClaim` are private, and its `ManagedLaunchReport` is `pub(crate)` and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) for the interactive summary; [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)'s `RuntimeActivationResult` and `ActiveRuntimeMarker` are `pub(crate)` and read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs)'s `apply_runtime_update`/`append_update_activate_summary` and from [`storage.rs`](../apps/rocm/src/storage.rs) respectively; its sibling [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)'s `SdkInstallConsents` is `pub(crate)` and read from `install_cmd.rs`'s `install()`. The pattern is still "mechanical relocation" rather than full domain extraction because call sites keep their pre-extraction unqualified-re-import form — the exception named above — rather than switching to the qualified paths full-domain-extraction modules use. Where a subcommand's dispatch fn lives is not what makes it so: a full-domain-extraction module can host its own dispatch fn too (see `comfyui::run` above). +- **Mechanical relocation** — a `pub(crate) fn` moves out verbatim, with shared types/config staying at the crate root and reached via `crate::`. Used for dispatch-adjacent clusters where a minimal, easy-to-review diff matters more than full extraction. Example: [`apps/rocm/src/update_cmd.rs`](../apps/rocm/src/update_cmd.rs). Variant: when the cluster is large enough to own types (part of `main.rs`'s Phase 6a split, ROCMAI-82, `main.rs`'s Phase 6b split, ROCMAI-91, `main.rs`'s Phase 6c split, ROCMAI-89, and Phase 6e's follow-up, ROCMAI-726), those types move with it instead of staying in `main.rs` — [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs)'s types are used nowhere else, but [`driver_install.rs`](../apps/rocm/src/driver_install.rs)'s `DriverInstallResult`/`DriverInstallError` are `pub(crate)` and read from [`install_cmd.rs`](../apps/rocm/src/install_cmd.rs)'s `install()` (`result.output`/`result.executed`, `error.source`/`error.executed`); [`freeform_execution.rs`](../apps/rocm/src/freeform_execution.rs)'s `FreeformExecution` is `pub(crate)` and read from `main.rs`'s `execute_freeform_next_action()` (`execution.action`) — it was already `pub(crate)` before the split, unlike the two functions that return/consume it, which widened from module-private; [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs)'s `ServeEngineSelection`/`ServeArgs` both stay `pub(crate)` and are used from `main.rs` (`ServeArgs`) and [`model_fit.rs`](../apps/rocm/src/model_fit.rs) (`assess_model_for_host` calls `select_serve_engine` and reads the resulting `ServeEngineSelection`'s `.engine`/`.source` fields) — and `ServeGenerationDefaults` moved out with the engine-recipe overrides to [`serve_recipe.rs`](../apps/rocm/src/serve_recipe.rs), where it and its fields are `pub(crate)` because `serve()` builds it; the types in the child modules of [`service_launch.rs`](../apps/rocm/src/service_launch.rs) are private to the child that owns them (`AttachOutcome`, `RawModeGuard`, `AutostartClaim`) or `pub(super)` where a sibling child reads them (`ManagedSpawn`, `ManagedPortDecision`), except `ManagedLaunchReport`, which is `pub(crate)` and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) for the interactive summary, and `ManagedReuseRefusal`, which is `pub(crate)` because `managed_service_reuse_refusal`, a function `serve_cmd.rs` calls, returns it; [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs)'s `RuntimeActivationResult` and `ActiveRuntimeMarker` are `pub(crate)` and read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs)'s `apply_runtime_update`/`append_update_activate_summary` and from [`storage.rs`](../apps/rocm/src/storage.rs) respectively; its sibling [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)'s `SdkInstallConsents` is `pub(crate)` and read from `install_cmd.rs`'s `install()`. The pattern is still "mechanical relocation" rather than full domain extraction because call sites keep their pre-extraction unqualified-re-import form — the exception named above — rather than switching to the qualified paths full-domain-extraction modules use. Where a subcommand's dispatch fn lives is not what makes it so: a full-domain-extraction module can host its own dispatch fn too (see `comfyui::run` above). Moving a `proptest!` block to another file leaves its saved failure cases behind unless they are carried over: proptest keys them by the test's source file (`/src/.rs` reads `/proptest-regressions/.txt`, where `src/` is the directory holding `main.rs` or `lib.rs`) and does not warn when the file stops matching. Carry them in the same commit, and how depends on how much moved. When every property test of the file moves and the new file has no regression file yet, `git mv` the old one to the name the new file needs. When only some move, the tests left behind still read the old file, so keep it, and copy the saved cases to the new file's regression file (appending to it when it already exists, so its own cases stay). An `xtask` test fails the build on a regression file no test reads; it cannot notice a regression file that was moved away from tests that still need it, so check a split's diff for that. See [Property tests](testing.md#property-tests). @@ -34,7 +34,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every [`main.rs`](../apps/rocm/src/main.rs) is the crate root. Besides smaller helpers, it holds `main()`, `run()` and `dispatch()`, the match over the clap `Cli`/`Command` tree that [`cli.rs`](../apps/rocm/src/cli.rs) defines; the builders that turn that tree into a `clap::Command` (`cli_command()`, `completion_command()`); the command handlers still defined here (`version()` and `launch_default()`, the no-subcommand arm); freeform-invocation routing (`parse_freeform_invocation`, `should_treat_as_freeform`, `run_freeform`, `execute_freeform_next_action`, `treat_as_natural_language`); a few crate-root items several modules share (`status_for_readiness`, `device_policy_name`, `engine_inventory`, `ROLLBACK_RECOVERY_HINT`); the `use crate::x::{...}` re-import block described above, which also re-imports the `cli.rs` types so other modules keep reaching them as `crate::Command`, `crate::Provider` and so on; and the crate's shared `mod tests` block. - [`automations.rs`](../apps/rocm/src/automations.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-210): the `rocm automations` dispatch handler (`automations()`) plus the `rocm automations` status report it prints (`render_automations_text`), the `ProposalKind` classification that report renders through (`proposal_kind`, `fallback_tool_for_automation_action`, `proposal_plain_summary`, `proposal_plain_reason`), the small plain-language label helpers (`proposal_status_label`, `proposal_bool_argument`, `automation_event_plain_summary`, `audit_event_plain_summary`, `watcher_mode_plain_label`, `watcher_last_check`, `watcher_plain_name`, `watcher_plain_trigger`, `watcher_plain_action`, `watcher_policy_note`), and the unrelated `rocm daemon --status` summary (`render_daemon_text`). `render_automations_text` and `render_daemon_text` are `pub(crate)`: [`main.rs`](../apps/rocm/src/main.rs)'s `dispatch()` calls `render_daemon_text` directly (re-imported alongside `automations` via `use crate::automations::{automations, render_daemon_text};`; `render_automations_text` has no `dispatch()` call site of its own, reached only through `automations()` above, now local to this file), and [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs)'s read-only command runner calls both. Every other item in this list, including `watcher_policy_note` (called from `automations()`'s `Enable` arm and from `render_automations_text`, both in this same file), has no caller outside this module and stays private. The cluster's regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` the same way other mechanically relocated modules do. -- [`background_helper.rs`](../apps/rocm/src/background_helper.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), owns one `pub(crate)` type: what the CLI can tell about the background automation helper (`rocm daemon`). It holds `background_helper_already_running` (the file-based liveness check behind the no-double-spawn decision), the `HelperNote` type and `background_helper_note` with its `helper_note_for` wording, which [`services_text.rs`](../apps/rocm/src/services_text.rs) prints after a stop and [`uninstall_plan.rs`](../apps/rocm/src/uninstall_plan.rs) turns into an uninstall warning. Spawning the helper (`ensure_background_helper_running` and its autostart lock) stays in [`service_launch.rs`](../apps/rocm/src/service_launch.rs), which reads `background_helper_already_running` from here. The module is a leaf: it imports nothing from the service modules, so the note can be read from [`services_text.rs`](../apps/rocm/src/services_text.rs) without that module importing the one that spawns services. The cluster's regression tests live in this module's own `mod tests`, which stays private: the `runtime_state` and `set_helper_facts` fixtures they share with the helper-autostart and stop-output tests in [`service_launch.rs`](../apps/rocm/src/service_launch.rs) live in the crate-level [`test_support.rs`](../apps/rocm/src/test_support.rs). +- [`background_helper.rs`](../apps/rocm/src/background_helper.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), owns one `pub(crate)` type: what the CLI can tell about the background automation helper (`rocm daemon`). It holds `background_helper_already_running` (the file-based liveness check behind the no-double-spawn decision), the `HelperNote` type and `background_helper_note` with its `helper_note_for` wording, which [`services_text.rs`](../apps/rocm/src/services_text.rs) prints after a stop and [`uninstall_plan.rs`](../apps/rocm/src/uninstall_plan.rs) turns into an uninstall warning. Spawning the helper (`ensure_background_helper_running` and its autostart lock) stays in [`service_launch/helper_autostart.rs`](../apps/rocm/src/service_launch/helper_autostart.rs), a child of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), which reads `background_helper_already_running` from here. The module is a leaf: it imports nothing from the service modules, so the note can be read from [`services_text.rs`](../apps/rocm/src/services_text.rs) without that module importing the one that spawns services. The cluster's regression tests live in this module's own `mod tests`, which stays private: the `runtime_state` and `set_helper_facts` fixtures they share with the helper-autostart and stop-output tests in [`service_launch/helper_autostart.rs`](../apps/rocm/src/service_launch/helper_autostart.rs) live in the crate-level [`test_support.rs`](../apps/rocm/src/test_support.rs). - [`bootstrap.rs`](../apps/rocm/src/bootstrap.rs) — a further-extracted variant of full domain extraction: it owns its clap command enum (`BootstrapCommand`) and dispatch function too, rather than leaving the enum in [`cli.rs`](../apps/rocm/src/cli.rs) and the dispatch in [`main.rs`](../apps/rocm/src/main.rs). - [`bridge_snapshot.rs`](../apps/rocm/src/bridge_snapshot.rs) — mechanical relocation, no owned types: the host snapshot behind the internal bridge surface (`build_codex_bridge_snapshot`, `build_codex_bridge_gpu_snapshot`, `builtin_codex_bridge_engine_inventory`), whose engine list is derived from `main.rs`'s `engine_inventory`, the single table of built-in engine ids that `rocm examine`, `rocm update` and the `--engine` help test read as well. All three are `pub(crate)`: [`main.rs`](../apps/rocm/src/main.rs)'s `dispatch()` calls `build_codex_bridge_snapshot` through a re-import, [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs)'s internal MCP tools reach them as `crate::` through the same re-import. It has no tests of its own. - [`chat.rs`](../apps/rocm/src/chat.rs) — the chat assistant's entry points and the module that ties its pieces together, a file beside a `chat/` directory of child modules (as [`providers.rs`](../apps/rocm/src/providers.rs) is beside `providers/`). It keeps `read_piped_prompt` and `piped_prompt_from_input` (the `echo "…" | rocm chat` path), `render_chat_text` (the status screen), `render_chat_prompt_text` (the one-shot reply text, which calls into `chat/turn.rs`), `plain_provider_auth_status` and the clap value parsers for the sampling flags (`parse_non_negative_temperature`, `parse_unit_interval`, `parse_positive_u32`), plus the re-exports below. The work behind them lives in four leaf children: [`chat/turn.rs`](../apps/rocm/src/chat/turn.rs) (the turn loop, the system prompt and the transcript), [`chat/intent.rs`](../apps/rocm/src/chat/intent.rs) (the prompt-intent heuristics), [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs) (the MCP tool bridge and in-process command execution) and [`chat/summaries.rs`](../apps/rocm/src/chat/summaries.rs) (the deterministic tool-output summaries); `turn` is the top of that call graph and calls the other three, none of which calls back. The tool-call validation and approval policy (`validate_chat_tool_call`, the `validate_chat_*_tool_call` family, `ChatRocmCommandAction`, `ChatToolApprovalRequest`) lives in [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs), which the children call into. Callers in other modules, production and test-only, reach the moved items through the `chat::` module boundary, so `chat.rs` re-exports each one another module reads: `run_internal_mcp_call` (`main.rs`, `dash_seam.rs`), `rocm_chat_tool_system_prompt_for_host` (`dash.rs`), `chat_tool_approval_request`, `chat_tool_call_is_read_only` and `validate_chat_tool_call` (`dash_seam.rs`), `any_substring` and `ensure_flag` (from `prompt_text.rs` and `rocm_argv.rs`, for `freeform_answer.rs` and `freeform_execution.rs`), and, only under `#[cfg(test)]`, `deterministic_comfyui_tool_summary` (`comfyui.rs`'s tests, unix only), `ChatRocmCommandAction`, `rocm_chat_tool_requested_args`, `chat_install_driver_action` and `chat_rocm_command_action_from_args` (`driver_plan.rs`'s and `system_packages.rs`'s tests). Each child's tests live in its own `mod tests`, with their original names; the two piped-prompt tests and the command-preview property tests (which drive the tool-call guards, the planner's guards and the rendered command lines across the policy, the bridge and the turn loop) stay in `chat.rs`. `serve_negative_sampling_space_form_reaches_range_validator`, a shared clap value-parser wiring test living alongside its sibling command-parsing tests, has moved with them to the test module in [`cli.rs`](../apps/rocm/src/cli.rs); the four structural checks that read [`chat_cmd.rs`](../apps/rocm/src/chat_cmd.rs)'s source for the `Command::Chat` handler's wiring (`command_chat_interactive_routes_to_dash_run_chat_not_tui`, `command_chat_interactive_notes_dropped_provider_flag`, `command_chat_honors_chat_mock_and_keeps_prompt_passthrough`, `command_chat_reads_prompt_from_piped_stdin`) live in that module's own `mod tests`. @@ -93,7 +93,14 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`service_control.rs`](../apps/rocm/src/service_control.rs) — mechanical relocation, owns private types: stopping and restarting a managed local server, the process control behind `rocm services stop|restart`, the chat/LLM sandbox tool-calling path and `runtime_services.rs`'s moves onto a new runtime. It holds the sandbox-tool runner (`run_internal_sandbox_tool`), the stop path (`terminate_recorded_service_pids`, the `MANAGED_STOP_GRACE` constant, the `RecordedPidOutcome` and `RecordedStopTermination` types, `scripted_stop_outcome`, the end-to-end hook behind the `e2e-test-hooks` feature, `stop_internal_managed_service`, `stop_internal_managed_service_with`, `stop_internal_managed_service_confirmed_with`, `unload_lemonade_service_model`) and the restart path (`apply_restart_runtime_pin`, `load_service_for_restart`, `restart_child_argv`, `restart_internal_managed_service`, `restart_internal_managed_service_with`, `update_restarting_record`, `record_restarted_child`, `record_restart_readiness`). Only `run_internal_sandbox_tool` and `restart_internal_managed_service` are `pub(crate)`: the first is read from `dispatch()` in [`main.rs`](../apps/rocm/src/main.rs), from [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs)'s `run_approved_service_action` and from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), which passes it to [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s `run_attached_service` as the stop for a Ctrl-C, the second from [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs); each importer reaches them through `crate::service_control` rather than a re-export. Every other item has no caller outside this module and stays private — see [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs)'s and [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s entries above for how those two modules' own doc comments describe this module as the caller of their helpers. The module reads managed-service records through [`services_text.rs`](../apps/rocm/src/services_text.rs) (`load_managed_service`, `read_managed_service_manifest`) and shapes the `stop_server` answer with its `internal_stop_server_value`; `services_text.rs` does not call back into it. This module takes `ensure_public_service_has_endpoint_key` and the spawn helpers from [`service_launch.rs`](../apps/rocm/src/service_launch.rs), which does not import it back. The cluster's regression tests live in this module's own `mod tests`, reaching shared fixtures (`test_paths`, `ScopedTestEnv`, `plant_service_record_on_runtime`, `OLD_RUNTIME_KEY`, `NEW_RUNTIME_KEY`) through `crate::tests::` the same way other mechanically relocated modules do, and `runtime_services::` through its own `use crate::runtime_services;` for the handful of tests that drive a restart through `runtime_services`'s public entry points rather than calling it directly. - [`serve_port.rs`](../apps/rocm/src/serve_port.rs) — full domain extraction: the central `rocm serve` port policy (automatic vs explicit request, reservation by live managed services, OS preflight lease), used by [`service_launch.rs`](../apps/rocm/src/service_launch.rs); the CLI restart path shares the `rocm-core` lease primitives (`lease_loopback_port_or_bind_error`, `is_canonical_loopback_host`) rather than this module. - [`service_hints.rs`](../apps/rocm/src/service_hints.rs) — shared CLI-output component: the `rocm services stop|restart|remove --yes` hint text, shared by [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs), [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs), [`service_launch.rs`](../apps/rocm/src/service_launch.rs), [`service_records.rs`](../apps/rocm/src/service_records.rs), [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs), and [`services_text.rs`](../apps/rocm/src/services_text.rs) so they cannot go out of sync with each other, as `serve_summary.rs`'s stop row did from the day it was added — #417. It also holds the helpers that map a service tool name to the command word, verb and past tense those hints and messages use (`service_action_command`, `service_action_verb`, `service_action_past_tense`) and to its `SandboxToolArg` (`sandbox_tool_arg_from_service_tool`); [`services_text.rs`](../apps/rocm/src/services_text.rs) reads `service_action_past_tense` from here rather than through the crate root. It quotes the service id with `shell_quote`, so a hint names exactly the argv it runs. -- [`service_launch.rs`](../apps/rocm/src/service_launch.rs) — mechanical relocation, owns types (Phase 6b, ROCMAI-91): spawning/attaching to a managed service process and streaming its log, the port allocation transaction and failed-launch cleanup (under the shared launch lock), plus the endpoint-key/auth checks that gate a public bind — `AttachOutcome`/`RawModeGuard`/`ManagedSpawn`/`AutostartClaim` are its private types; `ManagedLaunchReport` is `pub(crate)`, since [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) reads its fields directly for the interactive summary. Most of this module's functions are `pub(crate)` for the same reason — other modules, including [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`automations.rs`](../apps/rocm/src/automations.rs) and [`service_control.rs`](../apps/rocm/src/service_control.rs), call into it; `main.rs` does not. `ServicesCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs) rather than moving alongside, and `services()` lives in [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs); the record removal and pruning it dispatches to now lives in [`service_records.rs`](../apps/rocm/src/service_records.rs), and only stop/restart are also reachable from the chat/LLM sandbox tool-calling path, out of scope for Phase 6b. This module does not import [`service_control.rs`](../apps/rocm/src/service_control.rs), which imports it: `run_attached_service` takes the stop for a Ctrl-C as a parameter, and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) passes `run_internal_sandbox_tool`. +- [`service_launch.rs`](../apps/rocm/src/service_launch.rs) — mechanical relocation, since split into a directory of child modules, owns no code of its own beyond its declarations: spawning/attaching to a managed service process and streaming its log, the port allocation transaction and failed-launch cleanup (under the shared launch lock), plus the endpoint-key/auth checks that gate a public bind, divided among the children listed below. The file declares the children, a `test_support` module compiled only for tests (the fixtures more than one child's tests share), and `pub(crate) use` re-exports of the items other modules call, so [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`automations.rs`](../apps/rocm/src/automations.rs), [`service_control.rs`](../apps/rocm/src/service_control.rs) and [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs) still reach them as `crate::service_launch::name`; [`main.rs`](../apps/rocm/src/main.rs) does not. An item that sibling children share is `pub(super)` and reached through `super::`; everything else is private to the child that owns it. `ServicesCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs) rather than moving alongside, and `services()` lives in [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs); the record removal and pruning it dispatches to now lives in [`service_records.rs`](../apps/rocm/src/service_records.rs), and only stop/restart are also reachable from the chat/LLM sandbox tool-calling path. This module does not import [`service_control.rs`](../apps/rocm/src/service_control.rs), which imports it: `run_attached_service` takes the stop for a Ctrl-C as a parameter, and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) passes `run_internal_sandbox_tool`. +- [`service_launch/endpoint_auth.rs`](../apps/rocm/src/service_launch/endpoint_auth.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, no owned types: bind-host validation and the endpoint API key policy. `validate_bind_host` refuses a non-loopback host without `--allow-public-bind` (`is_loopback_host` is the inverse of `rocm_engine_protocol::is_public_bind_host`, which `rocmd` shares); `resolve_endpoint_auth` decides the key a launch uses (none for a loopback bind unless a key is required, otherwise the supplied key trimmed or a generated one, refusing an empty key or one with a control character); `drop_orphaned_endpoint_key_on_already_running` clears a key staged for an attempt that spawned nothing; `ensure_public_bind_engine_supported` refuses a public bind with Windows managed Lemonade, whose key cannot be enforced; and `ensure_public_service_has_endpoint_key` is the guard a respawn runs so a public (or `--require-api-key`) service whose key file is gone is not reopened without authentication. All six are `pub(crate)` and re-exported by the parent: [`service_control.rs`](../apps/rocm/src/service_control.rs) reads `ensure_public_service_has_endpoint_key`, [`tool_policy.rs`](../apps/rocm/src/tool_policy.rs) reads `is_loopback_host`, and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) reads the rest. +- [`service_launch/launch_transaction.rs`](../apps/rocm/src/service_launch/launch_transaction.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, owns types: the part of a managed launch that decides before anything spawns. `resolve_managed_port_in_transaction` refreshes liveness, answers an equivalent live service before leasing any port (`ManagedPortDecision`), and otherwise leases a concrete port against both the ports live records still reserve (`live_managed_services`, `reserved_service_ports`) and the real OS through [`serve_port.rs`](../apps/rocm/src/serve_port.rs); a caller must hold the launch lock across it and the record write that follows. `abandon_failed_managed_launch` undoes a failed attempt before that lock is released: a spawned child is terminated first and its record kept, marked stopped, while an attempt that never spawned has every artifact removed. `managed_service_reuse_refusal` (with `ManagedReuseRefusal`) is the one answer to whether a live matching service can serve a request (recipe, `--require-api-key`, device policy), and `no_usable_gpu_refusal` words the no-usable-GPU refusal once for `serve()`'s pre-flight and for the spawn. `managed_service_reuse_refusal` and `no_usable_gpu_refusal` are `pub(crate)`, re-exported, and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), and `ManagedReuseRefusal` is `pub(crate)` too because the first returns it (only [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs) matches on it); `ManagedPortDecision`, `live_managed_services`, `resolve_managed_port_in_transaction` and `abandon_failed_managed_launch` are `pub(super)` for [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs). +- [`service_launch/spawn.rs`](../apps/rocm/src/service_launch/spawn.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, owns types: the detached engine child that the background and attached serve paths share. The process and stdio plumbing (`detach_background_command`, `attach_background_stdio`, `managed_service_process_command`, `managed_engine_startup_failure_detail`) is `pub(crate)` and, off Windows, re-exported: [`service_control.rs`](../apps/rocm/src/service_control.rs) reads all four for a restart and [`service_launch/helper_autostart.rs`](../apps/rocm/src/service_launch/helper_autostart.rs) reads the first three. `ManagedLaunchReport` (`pub(crate)`) is the engine-neutral result of a launch, and `ManagedSpawn` (`pub(super)`) says whether the launch found a live service or spawned a child. `spawn_managed_engine_child` is the whole serialized transaction under the launch lock (duplicate detection, port lease, manifest and log creation, argv preparation, the bounded immediate-exit check and record publication) and returns before any readiness wait; `launch_managed_engine_child_locked` is its side-effecting half, split out so that every failure after the record's first write goes through `abandon_failed_managed_launch`. The tests reach the spawn through a test-only, thread-scoped hook (`TEST_SPAWN_HOOK`, `pub(super)` so the tests in [`service_launch/start.rs`](../apps/rocm/src/service_launch/start.rs) can observe the state at the instant of the spawn). +- [`service_launch/readiness.rs`](../apps/rocm/src/service_launch/readiness.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, owns private items: waiting on a service after it is launched or found live. `wait_for_record_ready` and `wait_on_existing_service` run the wait (one wait is bounded by `READINESS_WAIT`, and `wait_past_readiness_budget` is what `rocm serve --wait` uses to keep polling past it until the server answers or stops being live), `service_pid_is_alive`, `record_still_live` and `service_still_live` decide whether the tracked process is still there, `settle_exited_service` demotes the manifest of one that is not, and `died_during_wait` and `record_wait_outcome` end a wait, writing its outcome back without discarding what changed on disk meanwhile. `e2e_simulate_oom_launch` and `simulate_oom_managed_launch` are the end-to-end fault-injection seam behind the `e2e-oom-fault-injection` feature: the first is a `const fn` returning false without the feature, is `pub(crate)` and re-exported, and is read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs); the second fabricates the on-disk state of a launch that ran out of memory. `wait_for_record_ready`, `wait_on_existing_service`, `service_pid_is_alive`, `died_during_wait`, `record_wait_outcome` and `simulate_oom_managed_launch` are `pub(super)` for [`service_launch/start.rs`](../apps/rocm/src/service_launch/start.rs); the rest are private. +- [`service_launch/start.rs`](../apps/rocm/src/service_launch/start.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, no owned types: the background launch path. `start_managed_service` runs the spawn transaction, releases the launch lock once the record is published, waits for HTTP readiness and returns a `ManagedLaunchReport`; if the launch fails before a record exists it drops the endpoint key `serve()` staged for the attempt. `print_managed_launch_plain` renders the report line by line for output that is not interactive, and `wait_begin_client_config` returns the one-time client configuration for a generated key when a wait begins, built by `render_endpoint_client_config` (`pub(super)`, since [`service_launch/attach.rs`](../apps/rocm/src/service_launch/attach.rs) prints it too). The three functions other modules call are `pub(crate)`, re-exported, and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs). +- [`service_launch/helper_autostart.rs`](../apps/rocm/src/service_launch/helper_autostart.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, owns a private type: starting the background automation helper (`rocm daemon`) when a watcher needs it. `ensure_background_helper_running` and `ensure_background_helper_running_quiet` do nothing unless an automation is enabled and [`background_helper.rs`](../apps/rocm/src/background_helper.rs)'s `background_helper_already_running` says no helper is running; `ensure_background_helper_running_with` takes the spawn as a parameter so the decision is testable, and `spawn_background_helper` does the detached spawn. Concurrent callers queue on an autostart lock, and `AutostartClaim` (with `AUTOSTART_CLAIM_TTL_MS`, `read_autostart_claim`, `write_autostart_claim` and `autostart_spawn_in_flight`) records a spawn in flight so the next holder defers instead of starting a second helper. Both entry points are `pub(crate)` and re-exported: [`automations.rs`](../apps/rocm/src/automations.rs) calls the first, and [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) and [`service_launch/attach.rs`](../apps/rocm/src/service_launch/attach.rs) call the quiet one. +- [`service_launch/attach.rs`](../apps/rocm/src/service_launch/attach.rs) — mechanical relocation out of [`service_launch.rs`](../apps/rocm/src/service_launch.rs), a leaf child of it, owns private types: the attached serve path (`--verbose`/`--foreground`). `run_attached_service` spawns the same detached child the background path does and follows its log in the terminal until the user detaches with Ctrl-D (the server keeps running) or stops it with Ctrl-C, or the engine exits; `AttachOutcome` is that result and `detach_key_outcome` maps a key press to it. `stream_attached_logs` reads keys in crossterm raw mode (`RawModeGuard` restores the terminal on every exit path) and `stream_attached_logs_no_tty` follows the log until the process exits when stdin is not a terminal. `finish_attached_stop` turns the answer of the stop that Ctrl-C ran into output: a confirmed stop is reported on stdout, an unconfirmed one becomes the error, and a failed stop signals the child this session launched. `run_attached_service` is `pub(crate)`, re-exported, and read from [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs). - [`service_records.rs`](../apps/rocm/src/service_records.rs) — mechanical relocation, owns types: removing one managed-service record and every file it owns (`remove_managed_service_record`, `service_record_artifacts`) and the `rocm services prune` plan/render/apply steps (`build_service_prune_plan`, `render_service_prune_plan`, `apply_service_prune_plan`, `prune_managed_service_records`) with the age gate behind them (`DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS`, `service_prune_min_age_hours`). It reads record state through [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) and [`services_text.rs`](../apps/rocm/src/services_text.rs) and builds its advice text with [`service_hints.rs`](../apps/rocm/src/service_hints.rs); none of the three depends on it. `remove_managed_service_record`, `prune_managed_service_records`, `service_prune_min_age_hours` and `DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS` are `pub(crate)` (`services()` in [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs) calls the first three; the `ServicesCommand::Prune` clap definition in [`cli.rs`](../apps/rocm/src/cli.rs) uses the constant), and `ServicePruneOutcome`, the return type of `prune_managed_service_records`, is `pub(crate)` with `pub(crate)` `text`, `removed_records`, `removed_files` and `failures` fields that `services()` reads. Every other item is private. The cluster's regression tests live in this module's own `mod tests`, reaching the shared `test_paths`, `plant_service_record` and `plant_service_record_without_its_key` fixtures through `crate::tests::` (the last two widened to `pub(crate)` for that). - [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm services`' command handler (`services()`, which dispatches `list`, `logs`, `stop`, `restart`, `remove` and `prune`) and `run_approved_service_action`, the approval-gated stop/restart path. `ServicesCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs). Both functions are `pub(crate)`: `dispatch()` calls `services()` through a re-import, and [`advised_commands.rs`](../apps/rocm/src/advised_commands.rs)'s test reaches `run_approved_service_action` through a test-only re-import in `main.rs`. The module reads from [`services_text.rs`](../apps/rocm/src/services_text.rs), [`service_control.rs`](../apps/rocm/src/service_control.rs), [`service_records.rs`](../apps/rocm/src/service_records.rs), [`service_hints.rs`](../apps/rocm/src/service_hints.rs) and [`cli_audit.rs`](../apps/rocm/src/cli_audit.rs); none of them calls back into it. The cluster's one regression test lives in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::`. - [`services_text.rs`](../apps/rocm/src/services_text.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-209): the CLI/TUI logs browser (`render_logs_text`, `render_logs_browser_text`, `render_logs_browser_page_text`, `render_logs_browser_page_text_with_options`, `logs_browser_total_pages`, the `LogBrowserEntry` type, `collect_log_browser_entries`, `filter_log_browser_entries`, `log_browser_source_label`, `write_past_attempts_hint`), the `rocm services`/`rocm services logs` status surfaces (`render_services_text`, `render_services_json`, `render_service_logs_text`, `render_service_logs_text_with_options`, `render_service_action_result`, `read_managed_service_manifest`, `load_managed_service`), the `--follow` tail behind `rocm logs --service --follow` and `rocm services logs --follow` (`follow_service_logs`, `follow_service_logs_with`, the `LogTail`, `LogSegment` and `FollowFooter` types, `write_log_segments`, `newly_ready`; new code written in this module, not relocated), the internal `rocmd`-compatible status surface (`render_internal_status_text`), the `stop_server` answer shape and its renderer (`internal_stop_server_value`, `service_stop_unconfirmed`, `render_unconfirmed_service_stop`, `process_check_command`), plus `validate_service_id`, `list_log_files`, `read_optional_tail_lines`, and the CLI-lifecycle-log tail formatting helpers (`format_cli_lifecycle_tail_line`, `lifecycle_event_label`, `humanize_log_label`, `lifecycle_field`, `read_tail_lines`). The managed-service stop and restart implementation that used to live here is in [`service_control.rs`](../apps/rocm/src/service_control.rs). The `LogBrowserEntry` struct moved here from `main.rs` with the logs browser and stays private. `dispatch()` calls directly into this module at several points — `render_internal_status_text`, `render_service_logs_text`, `render_logs_browser_text`, and `render_logs_text` — all re-imported via `use crate::services_text::{...};`; [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs)'s `services()` calls `render_services_text`, `render_services_json` and `render_service_logs_text` through its own imports. `render_service_action_result`, `service_stop_unconfirmed` and `validate_service_id` are also `pub(crate)` and read from `services_cmd.rs`'s `run_approved_service_action`; `validate_service_id` is also read from [`service_records.rs`](../apps/rocm/src/service_records.rs)'s `remove_managed_service_record`/`service_record_artifacts`, and `load_managed_service` from its `remove_managed_service_record`/`apply_service_prune_plan`. `load_managed_service` is `pub(crate)`, read from [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) (reconciling live servers against a runtime activation), from [`service_control.rs`](../apps/rocm/src/service_control.rs) and from [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs)'s own test module; `read_managed_service_manifest` and `internal_stop_server_value` are `pub(crate)` for `service_control.rs`, and `process_check_command` for its tests; `read_tail_lines`, `render_logs_text`, `render_logs_browser_text`, `render_services_text`, `render_service_logs_text`, and `validate_service_id` are also `pub(crate)`, read from [`chat/mcp_bridge.rs`](../apps/rocm/src/chat/mcp_bridge.rs)'s read-only command runner; `read_optional_tail_lines` and `render_service_action_result` are also `pub(crate)`, read from [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s managed-spawn/attach path. Every other item in this list has no caller outside this module and stays private. The cluster's regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` the same way other mechanically relocated modules do; `assert_no_retry_line` stays here as a `pub(crate)` test helper because it checks the unconfirmed-stop rendering and `service_launch.rs`'s tests read it too. Two tests that call this module's `load_managed_service` only incidentally — their primary subject is `runtime_services::pin_service_record_to_runtime`/`restore_service_record_pin` and `runtime_services::restart_service_onto_runtime_with`'s outcome classification respectively — stay in `main.rs`'s shared `mod tests` instead. diff --git a/docs/testing.md b/docs/testing.md index b3369520b6..de66101bdb 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2619,7 +2619,7 @@ the per-PR mock lane. The host predicate itself is the shared than restated in either, so the test cannot drift from the production rule. The bypass decision's other arguments are in the same position — the unit tests -in `apps/rocm/src/service_launch.rs` pin `managed_service_reuse_refusal` itself, +in [`launch_transaction.rs`](../apps/rocm/src/service_launch/launch_transaction.rs) pin `managed_service_reuse_refusal` itself, not which values `serve()` hands it — so each one has a scenario instead: serve-30 for `require_api_key`, serve-32 for the requested recipe. The device policy needs neither: the predicate takes it by value rather than as an diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index ffbb2fd692..61111575ac 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -307,7 +307,7 @@ Feature: Model serving # `--require-api-key` reuse refusal (#604). A running server's key is read once # at launch; reuse cannot retroactively satisfy a demand for auth the server # never got, so the idempotency guard refuses instead of upgrading the record - # in place (see `spawn_managed_engine_child` in `service_launch.rs`). This is the + # in place (see `spawn_managed_engine_child` in `service_launch/spawn.rs`). This is the # local-side proof that the printed stop hint is not just present but correct # and directly runnable: the scenario stops the service with it and checks the # record is left `stopped`, the same proof `@id:service-stop-yes-stops-a- @@ -318,10 +318,10 @@ Feature: Model serving # this guard without a live GPU, same as every other `@requires-gpu` scenario # on this lane. Linux-only: `--require-api-key` forces auth even on a loopback # bind, and the serve path's `ensure_public_bind_engine_supported` guard - # (defined in `service_launch.rs`, called from `serve_cmd.rs`) refuses that combination - # for Lemonade on Windows (its server can't receive the key through the - # Windows detached-spawn primitive), so this scenario cannot reach the - # refusal it exercises there. + # (defined in `service_launch/endpoint_auth.rs`, called from `serve_cmd.rs`) + # refuses that combination for Lemonade on Windows (its server can't receive the + # key through the Windows detached-spawn primitive), so this scenario cannot + # reach the refusal it exercises there. @id:serve-require-api-key-refuses-unauthenticated-reuse @requires-gpu @requires-engine:lemonade @requires-os:linux Scenario: serve-24 - Reusing a running server without a key refuses a --require-api-key request Given a managed runtime is active @@ -725,7 +725,7 @@ Feature: Model serving # so it cannot be combined with the attached modes, which already stay on the # terminal. The waiting itself (a live server that becomes ready, one that # exits first) is covered by `wait_past_readiness_budget_*` in - # `apps/rocm/src/service_launch.rs`, since a scenario would need a slow engine. + # `apps/rocm/src/service_launch/readiness.rs`, since a scenario would need a slow engine. @id:serve-wait-conflicts-with-attached-modes Scenario: serve-43 - Waiting for readiness cannot be combined with attached mode When the user serves a model with both --wait and --verbose @@ -750,7 +750,7 @@ Feature: Model serving # "stop the service holding that port" advice is checked here by its wording # only; the advice is actually proven by # `stopping_the_reserving_record_frees_its_explicit_port_for_reuse` in - # apps/rocm/src/service_launch.rs, per AGENTS.md section 3. + # apps/rocm/src/service_launch/launch_transaction.rs, per AGENTS.md section 3. # Runs in the merge queue, like its automatic-port sibling serve-auto-port-skips-occupied. @id:serve-explicit-port-reserved @requires-gpu @merge-queue Scenario: serve-45 - Serving on a port another managed service already holds is refused diff --git a/tests/e2e-cucumber/tests/e2e/automations_steps.rs b/tests/e2e-cucumber/tests/e2e/automations_steps.rs index 95972ef56e..82176a92d7 100644 --- a/tests/e2e-cucumber/tests/e2e/automations_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/automations_steps.rs @@ -101,7 +101,7 @@ fn write_harness_autostart_claim(path: &std::path::Path) { /// Re-stamp a claim `plant_inflight_autostart_claim` planted, right before the /// CLI runs. The CLI honours a claim for only 30 seconds -/// (`AUTOSTART_CLAIM_TTL_MS` in `apps/rocm/src/service_launch.rs`). The mock +/// (`AUTOSTART_CLAIM_TTL_MS` in `apps/rocm/src/service_launch/helper_autostart.rs`). The mock /// lane runs up to 64 scenarios at once, and every step blocks a runtime /// worker while its `rocm` child runs, so this scenario's When can start more /// than 30 seconds after its Given. The CLI then rightly reads the claim as diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 5e126d1f01..7987e2c631 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -2350,7 +2350,7 @@ async fn plant_oom_launch(world: &mut E2eWorld) { // in the `When`, which left this step a no-op whose deletion changed nothing. // // The consumer of this variable is `e2e_simulate_oom_launch` in - // `apps/rocm/src/service_launch.rs`. Unlike `OOM_FAULT_INJECTION_ENV` (shared + // `apps/rocm/src/service_launch/readiness.rs`. Unlike `OOM_FAULT_INJECTION_ENV` (shared // through `e2e-report`, whose producer and consumer are both test-side), the // consumer here is the shipped binary's crate, which must not depend on the // e2e harness — so the name is spelled out on both sides. A typo fails loudly