diff --git a/README.md b/README.md index 340d3c8a52..ed685b71a8 100644 --- a/README.md +++ b/README.md @@ -842,7 +842,7 @@ rocm serve [--engine lemonade|vllm|aim] [--gpu auto|] [--runtime-id KEY | --env-id ID] [--host HOST] [--port PORT] - [--verbose] [--foreground | --managed] + [--verbose] [--foreground | --managed] [--wait] [--no-smoke-test] [--allow-public-bind] [--require-api-key] [--api-key KEY] [--temperature TEMP] [--top-p PROB] [--max-tokens N] @@ -912,6 +912,17 @@ can: directly and exits 0; if that signal fails too, it says so on stderr. `--managed` is the explicit form of the default background behavior. +`--wait` keeps `rocm serve` waiting until the server answers requests or its +engine gives up (vLLM and AIM after 5 minutes by default; raise it with +`ROCM_CLI_VLLM_READY_TIMEOUT_SECS` / `ROCM_CLI_AIM_READY_TIMEOUT_SECS`), instead +of returning `starting` after 45 seconds; if the server exits first (crashes, is killed, or is stopped with +`rocm services stop`), it fails with an error that points at the server's logs +instead of waiting forever. Ctrl-C ends the wait and leaves the server starting +in the background; a generated API key (public bind) is printed when the wait +begins, so interrupting the wait does not lose it. It cannot be combined with `--foreground` or `--verbose`, and +when the model is already being served and still starting, it waits on that +server instead (an already-serving one is reported at once). Watch the download with +`rocm logs --service --follow` from another terminal. `--no-smoke-test` skips the post-startup inference probe. Which model form to pass depends on the engine your GPU selects. The Lemonade @@ -1082,7 +1093,7 @@ Manage background servers started with `--managed`: ``` rocm services list [--all] [--json] -rocm services logs +rocm services logs [--follow] rocm services stop --yes rocm services restart --yes rocm services remove --yes @@ -1383,13 +1394,23 @@ provider settings are left untouched. ### Logs and cleanup ``` -rocm logs [--service ] [--search TERM ...] [QUERY ...] +rocm logs [--service [--follow]] [--search TERM ...] [QUERY ...] +rocm services logs [--follow] rocm uninstall [--yes] [--dry-run] [--keep-binaries] [--keep-config] [--keep-data] [--keep-cache] [--force-dev-binaries] ``` +`--follow` (`-f`) keeps printing a server's new log output, such as a model +download's progress, until the server stops (a server that ends `failed` makes the command exit +non-zero), and prints `Service is ready.` when it finishes starting. Ctrl-C +ends the follow and leaves the server running. On a terminal, a `Ctrl-C stops following, the server keeps running. Following ` +line stays on the row under the newest output. Piped or redirected output keeps +stdout to the log text alone: the hint (once), `is ready.` and `stopped following.` +lines go to stderr. A server that is already `failed` when the follow starts exits +non-zero too, after printing its log. + `rocm uninstall` removes four categories of files that ROCm CLI manages on this computer: its binaries, configuration, data, and caches. It lists what it will do and asks `[y/N]` (default: no) before doing it. Use `--dry-run` to only list, diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index 50f3caa662..4cb1401cc0 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -453,6 +453,15 @@ rocm serve qwen --verbose --device gpu_required")] /// Keep the server managed by ROCm CLI. #[arg(long)] managed: bool, + /// Keep waiting until the server answers requests or its engine gives up + /// (vLLM and AIM after 5 minutes by default, raised with + /// `ROCM_CLI_VLLM_READY_TIMEOUT_SECS` / `ROCM_CLI_AIM_READY_TIMEOUT_SECS`), + /// instead of returning `starting` after 45 seconds. + /// Also waits on a matching server that is already starting. Fails with a + /// pointer to the logs if the server exits first. Cannot be + /// combined with --foreground or --verbose, which already stay attached. + #[arg(long, conflicts_with_all = ["foreground", "verbose"])] + wait: bool, /// Stream every engine log line in this terminal instead of a deployment summary. #[arg(long, conflicts_with = "managed")] verbose: bool, @@ -552,11 +561,16 @@ rocm serve qwen --verbose --device gpu_required")] #[command(after_help = "EXAMPLES:\n \ rocm logs\n \ rocm logs --service \n \ +rocm logs --service --follow\n \ rocm logs --search error timeout")] Logs { /// Show logs for a managed service id. #[arg(long)] service: Option, + /// Keep printing new output for --service until the server stops (Ctrl-C ends the + /// follow and leaves the server running). Use it to watch a model download. + #[arg(long, short = 'f', requires = "service")] + follow: bool, /// Search recent logs for one or more words. #[arg(long, num_args = 1.., value_name = "QUERY")] search: Vec, @@ -1040,6 +1054,10 @@ pub(crate) enum ServicesCommand { Logs { /// Service id from `rocm services list`. service_id: String, + /// Keep printing new output until the server stops (Ctrl-C ends the follow and + /// leaves the server running). Use it to watch a model download. + #[arg(long, short = 'f')] + follow: bool, }, /// Stop a local model server. /// @@ -1909,6 +1927,22 @@ mod tests { Cli::try_parse_from(argv) } + #[test] + fn serve_wait_parses_and_conflicts_with_attached_modes() { + let cli = parse_serve(&["--wait"]).expect("--wait parses"); + assert!(matches!( + cli.command, + Some(Command::Serve { wait: true, .. }) + )); + let cli = parse_serve(&[]).expect("default parses"); + assert!(matches!( + cli.command, + Some(Command::Serve { wait: false, .. }) + )); + assert!(parse_serve(&["--wait", "--verbose"]).is_err()); + assert!(parse_serve(&["--wait", "--foreground"]).is_err()); + } + #[test] fn serve_parses_verbose_and_no_smoke_test_flags() { let cli = parse_serve(&["--verbose", "--no-smoke-test"]).expect("flags parse"); diff --git a/apps/rocm/src/cli_progress.rs b/apps/rocm/src/cli_progress.rs index 757476299b..65c5faa2a2 100644 --- a/apps/rocm/src/cli_progress.rs +++ b/apps/rocm/src/cli_progress.rs @@ -198,7 +198,7 @@ impl Spinner { /// CJK glyph occupies two terminal columns) to fit within `max_width` /// columns, appending an ellipsis when it doesn't already fit, so a repaint /// can never wrap to a second terminal row. -fn truncate_to_width(line: &str, max_width: usize) -> String { +pub(crate) fn truncate_to_width(line: &str, max_width: usize) -> String { if max_width == 0 { return String::new(); } diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 47326bc90d..ce37f8b6cc 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -169,7 +169,7 @@ use crate::service_control::run_internal_sandbox_tool; use crate::services_cmd::run_approved_service_action; use crate::services_cmd::services; use crate::services_text::{ - render_internal_status_text, render_logs_browser_text, render_logs_text, + follow_service_logs, render_internal_status_text, render_logs_browser_text, render_logs_text, render_service_logs_text, }; use crate::setup_cmd::{render_setup_status_text, setup}; @@ -822,6 +822,7 @@ fn dispatch(cli: Cli) -> Result<()> { port, foreground, managed, + wait, verbose, no_smoke_test, allow_public_bind, @@ -844,6 +845,7 @@ fn dispatch(cli: Cli) -> Result<()> { port, foreground, managed, + wait, verbose, no_smoke_test, allow_public_bind, @@ -863,6 +865,7 @@ fn dispatch(cli: Cli) -> Result<()> { Some(Command::Config { command }) => config(command), Some(Command::Logs { service, + follow, search, query, }) => { @@ -878,7 +881,11 @@ fn dispatch(cli: Cli) -> Result<()> { ); } if let Some(service_id) = service { - print!("{}", render_service_logs_text(&paths, &service_id)?); + if follow { + follow_service_logs(&paths, &service_id)?; + } else { + print!("{}", render_service_logs_text(&paths, &service_id)?); + } } else { let query = if search.is_empty() { (!query.is_empty()).then(|| query.join(" ")) diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 97f4d1d210..7854c2096b 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -78,6 +78,7 @@ use crate::service_launch::{ ensure_background_helper_running_quiet, ensure_public_bind_engine_supported, managed_service_reuse_refusal, no_usable_gpu_refusal, print_managed_launch_plain, resolve_endpoint_auth, run_attached_service, start_managed_service, validate_bind_host, + wait_begin_client_config, }; use crate::{ GatedEngineAction, cli_progress, device_policy_name, ensure_engine_allowed_on_os, @@ -482,6 +483,7 @@ pub(crate) struct ServeArgs { pub(crate) port: u16, pub(crate) foreground: bool, pub(crate) managed: bool, + pub(crate) wait: bool, pub(crate) verbose: bool, pub(crate) no_smoke_test: bool, pub(crate) allow_public_bind: bool, @@ -507,6 +509,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { port, foreground, managed, + wait, verbose, no_smoke_test, allow_public_bind, @@ -1113,10 +1116,15 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { } if background { - let mut spinner = - cli_progress::Spinner::new(format!("Starting {model} on {selected_engine}…")); - spinner.tick(); - let report = start_managed_service( + let spinner = std::cell::RefCell::new(cli_progress::Spinner::new(format!( + "Starting {model} on {selected_engine}…" + ))); + spinner.borrow_mut().tick(); + // The key is shown when the wait begins, since Ctrl-C or a failed wait + // would otherwise leave a running authenticated server whose key was never + // printed. It is then not repeated in the summary below. + let key_shown_before_wait = std::cell::Cell::new(false); + let launched = start_managed_service( &selected_engine, &service_id, &model, @@ -1132,8 +1140,41 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { launch_lock, require_api_key, gpu_refusal_waived_for_reuse, - &mut |_elapsed| spinner.tick(), - )?; + wait, + // Named with the id the launch resolved to: an already-starting + // server keeps its own, not the one generated above. The spinner + // has no id on it, and a non-TTY stderr gets no spinner at all. + &mut |waiting_on| { + spinner.borrow_mut().clear(); + eprintln!( + "Waiting for {waiting_on}; Ctrl-C stops waiting, the server keeps running. \ + Follow: rocm logs --service {waiting_on} --follow" + ); + if let Some(config) = wait_begin_client_config( + waiting_on, + &service_id, + &host, + port, + endpoint_auth.as_deref(), + ) { + key_shown_before_wait.set(true); + // Same intentional one-time key display as + // `print_managed_launch_plain`; see its rationale. + // codeql[rust/cleartext-logging] + print!("{config}"); + } + }, + &mut |_elapsed| spinner.borrow_mut().tick(), + ); + // `Spinner` has no `Drop`: clear its row so `Error:` is not glued to it. + let report = match launched { + Ok(report) => report, + Err(error) => { + spinner.borrow_mut().clear(); + return Err(error); + } + }; + let mut spinner = spinner.into_inner(); // Not when the launch was fabricated. `simulate_oom_managed_launch` // writes a `starting` record for a process that never existed, so the // helper would be left watching a pid that will never appear, with @@ -1168,7 +1209,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { } // Safe to move `endpoint_auth` here: this branch always returns, so the // fall-through (attached) path below never observes it moved. - let launched_key = if report.already_running { + let launched_key = if report.already_running || key_shown_before_wait.get() { None } else { endpoint_auth diff --git a/apps/rocm/src/serve_summary.rs b/apps/rocm/src/serve_summary.rs index 2351644d2f..73db2a4b0d 100644 --- a/apps/rocm/src/serve_summary.rs +++ b/apps/rocm/src/serve_summary.rs @@ -176,7 +176,7 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { }; let _ = writeln!( out, - " note: {explanation} - check `rocm logs --service {}` or `rocm services list`", + " note: {explanation} - follow `rocm logs --service {} --follow` or check `rocm services list`", summary.service_id ); } diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index c0dd281119..afce5b1c5f 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -39,7 +39,7 @@ use std::process::{Command as ProcessCommand, Stdio}; use std::thread; use std::time::Duration; -use anyhow::{Context, Result, bail}; +use anyhow::{Context, Result, anyhow, bail}; use rocm_core::{AppPaths, AutomationRuntimeState, EndpointReadiness, ManagedServiceRecord}; use rocm_core::{format_http_base_url, process_is_running}; use rocm_engine_protocol::{DevicePolicy, EngineRecipeHint, ResolveModelResponse}; @@ -702,6 +702,148 @@ fn spawn_managed_engine_child( }) } +/// How long one readiness wait polls before `rocm serve` reports `starting`. +const READINESS_WAIT: Duration = Duration::from_secs(45); + +/// `rocm serve --wait`: keep waiting past [`READINESS_WAIT`] (a first start +/// downloads the model, which can take far longer) until the server answers or +/// stops being live. A server that exited ends the wait with the last reading +/// instead of blocking forever. +fn wait_past_readiness_budget( + mut probe: impl FnMut() -> EndpointReadiness, + mut still_live: impl FnMut() -> bool, +) -> EndpointReadiness { + loop { + let readiness = probe(); + if readiness == EndpointReadiness::Serving || !still_live() { + return readiness; + } + } +} + +/// Whether `pid` is a running process that is not a zombie. The serve +/// supervisor is an unreaped child of this process, so after it dies +/// `process_is_running` alone keeps reporting it alive. +fn service_pid_is_alive(pid: u32) -> bool { + rocm_core::process_is_running(pid) && !process_is_zombie(pid) +} + +#[cfg(target_os = "linux")] +fn process_is_zombie(pid: u32) -> bool { + let Ok(stat) = fs::read_to_string(format!("/proc/{pid}/stat")) else { + return false; + }; + // `comm` may contain spaces and parentheses; the state follows the last `)`. + stat.rfind(')') + .and_then(|end| stat.get(end + 1..)) + .and_then(|rest| rest.split_whitespace().next()) + == Some("Z") +} + +#[cfg(not(target_os = "linux"))] +const fn process_is_zombie(_pid: u32) -> bool { + false +} + +/// Whether the record still describes a server that can finish starting: its +/// status is a live one *and* a tracked process is alive. The status alone is +/// not enough, since a killed engine never writes a terminal state. +fn record_still_live(record: &ManagedServiceRecord, pid_alive: impl Fn(u32) -> bool) -> bool { + if !crate::managed_service_status::managed_service_is_live(record) { + return false; + } + let pids = crate::managed_service_status::recorded_service_pids(record); + pids.is_empty() || pids.into_iter().any(pid_alive) +} + +/// [`record_still_live`] over the manifest on disk, so a `rocm services stop` +/// from another terminal (which writes only the manifest) ends the wait too. +fn service_still_live(paths: &AppPaths, service_id: &str, pid_alive: impl Fn(u32) -> bool) -> bool { + let Ok(mut latest) = ManagedServiceRecord::read_manifest(paths, service_id) else { + return false; + }; + let _ = latest.refresh_from_engine_state(); + record_still_live(&latest, pid_alive) +} + +/// Record a server that died mid-start as stopped, leaving a manifest that a +/// `services stop` already settled untouched. A `failed` state the engine +/// reported is persisted rather than overwritten, so the reason survives. +fn settle_exited_service(paths: &AppPaths, service_id: &str) { + let Ok(mut latest) = ManagedServiceRecord::read_manifest(paths, service_id) else { + return; + }; + let on_disk = latest.status.clone(); + let _ = latest.refresh_from_engine_state(); + if crate::managed_service_status::managed_service_is_live(&latest) { + latest.status = "stopped".to_owned(); + } else if latest.status == on_disk { + return; + } + let _ = latest.write(); +} + +/// The failure a `--wait` ends with when the server died before it became +/// ready, after recording the death in the manifest. `None` while the wait did +/// not apply, the endpoint is serving, or the server is still alive. +fn died_during_wait( + paths: &AppPaths, + service_id: &str, + wait_until_ready: bool, + readiness: EndpointReadiness, + pid_alive: impl Fn(u32) -> bool, +) -> Option { + if !wait_until_ready + || readiness == EndpointReadiness::Serving + || service_still_live(paths, service_id, pid_alive) + { + return None; + } + settle_exited_service(paths, service_id); + let mut message = format!( + "the server exited before it became ready (vLLM and AIM give up after 5 minutes by default; `ROCM_CLI_VLLM_READY_TIMEOUT_SECS` / `ROCM_CLI_AIM_READY_TIMEOUT_SECS` allow longer); see `rocm logs --service {service_id}` for its output" + ); + // A plain `serve` adds the memory guidance for a vLLM launch that ran out of + // VRAM; the wait's failure is the same launch, so it gets the same note. + // "starting" is the status the note keys on for "never became ready", which is + // what a `--wait` death is, whatever the settled record now says. + if let Ok(record) = ManagedServiceRecord::read_manifest(paths, service_id) { + for note in crate::serve_summary::append_oom_serve_note( + Vec::new(), + &record.engine, + false, + "starting", + Some(&record.log_path), + ) { + let _ = write!(message, "\n note: {note}"); + } + } + Some(anyhow!(message)) +} + +/// Record the outcome of a finished wait on the manifest as it is now, changing +/// only what the wait learned. The copy read at spawn time can be long stale: +/// `rocmd` may have recorded a restart or a new supervisor pid, and `rocm logs` +/// refreshes the record, so writing it back whole would revert those. +fn record_wait_outcome( + paths: &AppPaths, + held: &mut ManagedServiceRecord, + status: &str, + verified: bool, +) -> Result<()> { + let mut latest = ManagedServiceRecord::read_manifest(paths, &held.service_id) + .unwrap_or_else(|_| held.clone()); + status.clone_into(&mut latest.status); + if verified { + // Latch the verification the wait just performed, so the readiness checks + // behind `services list` and chat read it instead of re-probing. + latest.inference_verified_at_unix_ms = Some(rocm_core::unix_time_millis() as u64); + } + latest.write()?; + *held = latest; + Ok(()) +} + /// Test-only fault-injection switch, off (`const false`) in shipped binaries: /// the `e2e-oom-fault-injection` feature is enabled only by the `xtask e2e` /// mock-lane build, never by the release build. When enabled and @@ -804,6 +946,82 @@ fn simulate_oom_managed_launch( }) } +/// Probe `record`'s endpoint until it answers: once within the readiness +/// budget, or (with `wait_until_ready`) for as long as the server stays alive. +fn wait_for_record_ready( + paths: &AppPaths, + record: &ManagedServiceRecord, + wait_until_ready: bool, + endpoint_api_key: Option<&str>, + on_wait_tick: &mut dyn FnMut(Duration), +) -> EndpointReadiness { + let probe = |on_tick: &mut dyn FnMut(Duration)| { + wait_for_service_http_ready_with_progress( + &record.engine, + &record.host, + record.port, + &|| { + let mut latest = record.clone(); + let _ = latest.refresh_from_engine_state(); + latest.served_model_name().to_owned() + }, + endpoint_api_key, + READINESS_WAIT, + on_tick, + ) + }; + if wait_until_ready { + wait_past_readiness_budget( + || probe(&mut *on_wait_tick), + || service_still_live(paths, &record.service_id, service_pid_is_alive), + ) + } else { + probe(on_wait_tick) + } +} + +/// An equivalent service is already live. Without `--wait`, or once it is +/// serving, report it as is; otherwise wait for it like a fresh launch would. +fn wait_on_existing_service( + paths: &AppPaths, + mut report: ManagedLaunchReport, + wait_until_ready: bool, + on_wait_begin: &mut dyn FnMut(&str), + on_wait_tick: &mut dyn FnMut(Duration), +) -> Result { + if !wait_until_ready || report.status == "ready" { + return Ok(report); + } + let Ok(mut record) = ManagedServiceRecord::read_manifest(paths, &report.service_id) else { + return Ok(report); + }; + // The id of the server being waited on, which is not the caller's freshly + // generated one when it attached to an existing server. + on_wait_begin(&report.service_id); + // The key the caller generated belongs to a launch that did not happen; the + // existing service answers to its own. + let stored_key = endpoint_keys::endpoint_api_key(paths, &report.service_id); + let readiness = + wait_for_record_ready(paths, &record, true, stored_key.as_deref(), on_wait_tick); + if let Some(error) = died_during_wait( + paths, + &report.service_id, + true, + readiness, + service_pid_is_alive, + ) { + return Err(error); + } + report.status = status_for_readiness(readiness).to_owned(); + record_wait_outcome( + paths, + &mut record, + &report.status, + readiness == EndpointReadiness::Serving, + )?; + Ok(report) +} + #[allow(clippy::too_many_arguments)] pub(crate) fn start_managed_service( engine: &str, @@ -821,6 +1039,8 @@ pub(crate) fn start_managed_service( launch_lock: rocm_core::FileLock, require_api_key: bool, gpu_refusal_waived_for_reuse: bool, + wait_until_ready: bool, + on_wait_begin: &mut dyn FnMut(&str), on_wait_tick: &mut dyn FnMut(Duration), ) -> Result { let paths = AppPaths::discover()?; @@ -856,7 +1076,18 @@ pub(crate) fn start_managed_service( require_api_key, gpu_refusal_waived_for_reuse, )? { - ManagedSpawn::AlreadyRunning(report) => return Ok(report), + ManagedSpawn::AlreadyRunning(report) => { + // `--wait` applies to a service that is still starting too: the + // natural retry after `rocm serve m` printed `starting`. + drop(launch_lock); + return wait_on_existing_service( + &paths, + report, + wait_until_ready, + on_wait_begin, + on_wait_tick, + ); + } ManagedSpawn::Spawned { record, child_pid } => (*record, child_pid), }; // The claiming service record is now persisted, so the selected GPU is @@ -868,27 +1099,32 @@ pub(crate) fn start_managed_service( #[cfg(windows)] thread::sleep(Duration::from_millis(200)); - let readiness = wait_for_service_http_ready_with_progress( - engine, - host, - port, - &|| { - let mut latest = record.clone(); - let _ = latest.refresh_from_engine_state(); - latest.served_model_name().to_owned() - }, + if wait_until_ready { + on_wait_begin(service_id); + } + let readiness = wait_for_record_ready( + &paths, + &record, + wait_until_ready, endpoint_api_key, - Duration::from_secs(45), on_wait_tick, ); - let launch_status = status_for_readiness(readiness); - record.status = launch_status.to_owned(); - if readiness == EndpointReadiness::Serving { - // Latch the verification the wait just performed, so the readiness checks - // behind `services list` and chat read it instead of re-probing. - record.inference_verified_at_unix_ms = Some(rocm_core::unix_time_millis() as u64); + if let Some(error) = died_during_wait( + &paths, + service_id, + wait_until_ready, + readiness, + service_pid_is_alive, + ) { + return Err(error); } - record.write()?; + let launch_status = status_for_readiness(readiness); + record_wait_outcome( + &paths, + &mut record, + launch_status, + readiness == EndpointReadiness::Serving, + )?; let endpoint_url = format!("{}/v1", format_http_base_url(host, port)); record_cli_audit_event( &paths, @@ -956,6 +1192,23 @@ pub(crate) fn print_managed_launch_plain( println!(" readiness: {}", report.status); } +/// The client configuration to show when a `--wait` begins, so a key generated for +/// a public bind reaches the user even if the wait is interrupted or fails. +/// `None` unless the wait is on the launch that key belongs to: an attach to an +/// already-starting server (`waiting_on` differs from `service_id`) answers to +/// its own key, which is not shown here. +pub(crate) fn wait_begin_client_config( + waiting_on: &str, + service_id: &str, + host: &str, + port: u16, + api_key: Option<&str>, +) -> Option { + let key = api_key.filter(|_| waiting_on == service_id)?; + let endpoint_url = format!("{}/v1", format_http_base_url(host, port)); + Some(render_endpoint_client_config(&endpoint_url, key)) +} + /// Render the one-time secure client configuration for a public, authenticated /// endpoint. This is the *intended* channel for delivering the key to the user /// (unlike logs/status, which must never contain it) — it prints the key once at @@ -1679,6 +1932,301 @@ mod tests { use super::*; use crate::tests::test_paths; + #[test] + fn wait_past_readiness_budget_keeps_probing_a_live_starting_server() { + let mut readings = vec![ + EndpointReadiness::Unreachable, + EndpointReadiness::Unreachable, + EndpointReadiness::Serving, + ] + .into_iter(); + let mut probes = 0; + let result = wait_past_readiness_budget( + || { + probes += 1; + readings.next().expect("no probe after Serving") + }, + || true, + ); + assert_eq!(result, EndpointReadiness::Serving); + assert_eq!(probes, 3, "must outlast the first (45s) budget"); + } + + #[test] + fn wait_past_readiness_budget_stops_when_the_server_exits() { + let mut probes = 0; + let result = wait_past_readiness_budget( + || { + probes += 1; + EndpointReadiness::Unreachable + }, + || false, + ); + assert_eq!(result, EndpointReadiness::Unreachable); + assert_eq!(probes, 1); + } + + #[test] + fn wait_past_readiness_budget_returns_at_once_when_already_serving() { + let result = wait_past_readiness_budget( + || EndpointReadiness::Serving, + || panic!("liveness is irrelevant once serving"), + ); + assert_eq!(result, EndpointReadiness::Serving); + } + + fn record_tracking_pid(paths: &AppPaths, pid: u32, status: &str) -> ManagedServiceRecord { + let mut record = reusable_record(paths); + record.supervisor_pid = pid; + record.status = status.to_owned(); + record + } + + #[cfg(unix)] + fn exited_child_pid() -> u32 { + let mut child = std::process::Command::new("true").spawn().expect("spawn"); + let pid = child.id(); + child.wait().expect("reap"); + pid + } + + #[test] + #[cfg(unix)] + fn record_still_live_ends_the_wait_for_a_dead_pid_with_a_live_status() { + // The engine was killed before writing any terminal state: the status + // still says `starting`, but nothing tracked is running. + let (root, paths) = test_paths("wait-dead-pid"); + paths.ensure().expect("paths"); + let dead = record_tracking_pid(&paths, exited_child_pid(), "starting"); + assert!(!record_still_live(&dead, service_pid_is_alive)); + + let alive = record_tracking_pid(&paths, std::process::id(), "starting"); + assert!(record_still_live(&alive, service_pid_is_alive)); + + // A settled status ends the wait even while the pid is still around. + let stopped = record_tracking_pid(&paths, std::process::id(), "stopped"); + assert!(!record_still_live(&stopped, service_pid_is_alive)); + let _ = fs::remove_dir_all(root); + } + + #[test] + #[cfg(target_os = "linux")] + fn service_pid_is_alive_treats_an_unreaped_child_as_dead() { + // The serve supervisor is our own unreaped child; once it dies it is a + // zombie that `kill(pid, 0)` still reports as running. + let mut child = std::process::Command::new("true").spawn().expect("spawn"); + let pid = child.id(); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while !process_is_zombie(pid) { + assert!(std::time::Instant::now() < deadline, "child never exited"); + thread::sleep(Duration::from_millis(10)); + } + assert!( + process_is_running(pid), + "a zombie still answers kill(pid, 0)" + ); + assert!(!service_pid_is_alive(pid)); + child.wait().expect("reap"); + } + + #[test] + fn service_still_live_follows_the_manifest_on_disk() { + // `rocm services stop` from another terminal writes only the manifest. + let (root, paths) = test_paths("wait-manifest"); + paths.ensure().expect("paths"); + let mut record = record_tracking_pid(&paths, std::process::id(), "starting"); + record.write().expect("write"); + assert!(service_still_live(&paths, &record.service_id, |_| true)); + + record.status = "stopped".to_owned(); + record.write().expect("write"); + assert!(!service_still_live(&paths, &record.service_id, |_| true)); + assert!(!service_still_live(&paths, "no-such-service", |_| true)); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn settle_exited_service_demotes_a_live_manifest_and_spares_a_settled_one() { + let (root, paths) = test_paths("wait-settle"); + paths.ensure().expect("paths"); + let mut record = record_tracking_pid(&paths, 0, "starting"); + record.write().expect("write"); + settle_exited_service(&paths, &record.service_id); + let after = ManagedServiceRecord::read_manifest(&paths, &record.service_id).unwrap(); + assert_eq!(after.status, "stopped"); + + // A record that already failed keeps its reason: the guard, not the + // overwrite, is what spares it (a `stopped` -> `stopped` rewrite would + // be invisible). + record.status = "failed".to_owned(); + record.write().expect("write"); + settle_exited_service(&paths, &record.service_id); + let after = ManagedServiceRecord::read_manifest(&paths, &record.service_id).unwrap(); + assert_eq!(after.status, "failed"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn wait_on_existing_service_only_waits_for_one_that_is_not_serving_yet() { + // The probing path itself blocks for the readiness budget, so only the + // cases that must return at once are driven here; a manifest planted + // as `starting` would hang a wrong guard for 45 seconds and fail it. + let (root, paths) = test_paths("wait-existing"); + paths.ensure().expect("paths"); + let mut record = record_tracking_pid(&paths, 0, "starting"); + record.write().expect("write"); + let report = |status: &str| ManagedLaunchReport { + service_id: record.service_id.clone(), + endpoint_url: record.endpoint_url.clone(), + status: status.to_owned(), + already_running: true, + child_pid: None, + log_path: None, + manifest_path: None, + }; + let mut no_tick = |_: Duration| panic!("must not probe"); + + let untouched = wait_on_existing_service( + &paths, + report("starting"), + false, + &mut |_| panic!("no --wait, no banner"), + &mut no_tick, + ) + .expect("no --wait"); + assert_eq!(untouched.status, "starting"); + let ready = wait_on_existing_service( + &paths, + report("ready"), + true, + &mut |_| panic!("already ready, no banner"), + &mut no_tick, + ) + .expect("already ready"); + assert_eq!(ready.status, "ready"); + record.status = "starting".to_owned(); + assert_eq!( + ManagedServiceRecord::read_manifest(&paths, &record.service_id) + .unwrap() + .status, + "starting" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn died_during_wait_fails_with_a_logs_pointer_and_records_the_death() { + let (root, paths) = test_paths("wait-died"); + paths.ensure().expect("paths"); + let record = record_tracking_pid(&paths, std::process::id(), "starting"); + record.write().expect("write"); + let id = record.service_id; + let status = || { + ManagedServiceRecord::read_manifest(&paths, &id) + .unwrap() + .status + }; + + // Not waiting, already serving, or still alive: no failure, no change. + let pending = EndpointReadiness::Unreachable; + assert!(died_during_wait(&paths, &id, false, pending, |_| false).is_none()); + assert!( + died_during_wait(&paths, &id, true, EndpointReadiness::Serving, |_| false).is_none() + ); + assert!(died_during_wait(&paths, &id, true, pending, |_| true).is_none()); + assert_eq!(status(), "starting"); + + // Dead server: the error names the logs command and the record is settled. + let error = died_during_wait(&paths, &id, true, pending, |_| false).expect("died"); + let message = error.to_string(); + assert!( + message.contains("exited before it became ready"), + "{message}" + ); + assert!( + message.contains(&format!("rocm logs --service {id}")), + "{message}" + ); + assert_eq!(status(), "stopped"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn died_during_wait_adds_the_oom_guidance_for_a_vllm_launch_only() { + let (root, paths) = test_paths("wait-died-oom"); + paths.ensure().expect("paths"); + let mut record = record_tracking_pid(&paths, std::process::id(), "starting"); + record.engine = "vllm".to_owned(); + record.write().expect("write"); + fs::write( + &record.log_path, + "torch.OutOfMemoryError: HIP out of memory. Tried to allocate 7.21 GiB.\n", + ) + .expect("log"); + let id = record.service_id; + let pending = EndpointReadiness::Unreachable; + let message = died_during_wait(&paths, &id, true, pending, |_| false) + .expect("died") + .to_string(); + assert!(message.contains("--gpu-memory-utilization"), "{message}"); + + // Same log, another engine: that engine has no such flag to suggest. + let mut record = ManagedServiceRecord::read_manifest(&paths, &id).unwrap(); + record.engine = "lemonade".to_owned(); + "starting".clone_into(&mut record.status); + record.write().expect("write"); + let message = died_during_wait(&paths, &id, true, pending, |_| false) + .expect("died") + .to_string(); + assert!(!message.contains("--gpu-memory-utilization"), "{message}"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_wait_outcome_keeps_what_changed_on_disk_during_the_wait() { + let (root, paths) = test_paths("wait-outcome"); + paths.ensure().expect("paths"); + let mut held = record_tracking_pid(&paths, 111, "starting"); + held.write().expect("write"); + // While the wait ran, `rocmd` recorded a restart and a new supervisor. + let mut elsewhere = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); + elsewhere.restart_count = 3; + elsewhere.supervisor_pid = 222; + elsewhere.write().expect("write"); + + record_wait_outcome(&paths, &mut held, "ready", true).expect("outcome"); + let after = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); + assert_eq!(after.status, "ready"); + assert!(after.inference_verified_at_unix_ms.is_some()); + assert_eq!(after.restart_count, 3, "the restart count was reverted"); + assert_eq!(after.supervisor_pid, 222, "the supervisor pid was reverted"); + + record_wait_outcome(&paths, &mut held, "starting", false).expect("outcome"); + let after = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); + assert_eq!(after.status, "starting"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn wait_begin_shows_the_key_only_for_the_launch_it_belongs_to() { + let own = wait_begin_client_config("svc-new", "svc-new", "0.0.0.0", 8000, Some("k-secret")) + .expect("own launch shows its key"); + // No `{own}` in the failure messages: the text carries the dummy key, + // and formatting it into an assert is what CodeQL flags as logging it. + assert!(own.contains("api key: k-secret"), "key line missing"); + assert!( + own.contains("http://0.0.0.0:8000/v1/models"), + "models url missing" + ); + // Attached to an already-starting server: its key is not this one. + assert!( + wait_begin_client_config("svc-old", "svc-new", "0.0.0.0", 8000, Some("k-secret")) + .is_none() + ); + // Loopback binds have no key to show. + assert!(wait_begin_client_config("svc-new", "svc-new", "127.0.0.1", 8000, None).is_none()); + } + #[test] fn serve_bind_validation_requires_public_ack() { validate_bind_host("127.0.0.1", false).unwrap(); diff --git a/apps/rocm/src/services_cmd.rs b/apps/rocm/src/services_cmd.rs index c4e9204fa9..b559503110 100644 --- a/apps/rocm/src/services_cmd.rs +++ b/apps/rocm/src/services_cmd.rs @@ -41,9 +41,9 @@ use crate::service_records::{ prune_managed_service_records, remove_managed_service_record, service_prune_min_age_hours, }; use crate::services_text::{ - render_service_action_result, render_service_action_result_with_helper_note, - render_service_logs_text, render_services_json, render_services_text, service_stop_unconfirmed, - validate_service_id, + follow_service_logs, render_service_action_result, + render_service_action_result_with_helper_note, render_service_logs_text, render_services_json, + render_services_text, service_stop_unconfirmed, validate_service_id, }; pub(crate) fn services(command: Option) -> Result<()> { @@ -60,9 +60,13 @@ pub(crate) fn services(command: Option) -> Result<()> { } Ok(()) } - ServicesCommand::Logs { service_id } => { - print!("{}", render_service_logs_text(&paths, &service_id)?); - Ok(()) + ServicesCommand::Logs { service_id, follow } => { + if follow { + follow_service_logs(&paths, &service_id) + } else { + print!("{}", render_service_logs_text(&paths, &service_id)?); + Ok(()) + } } ServicesCommand::Stop { service_id, yes } => { run_approved_service_action(&paths, "stop_server", &service_id, yes) diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index a44816c90f..6bae507e7f 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -9,7 +9,8 @@ //! Mechanically relocated from `main.rs` with no behavior change, and later //! narrowed: the process-control cluster (stop, restart and the internal //! sandbox-tool runner) lives in `service_control.rs` (see -//! `docs/architecture.md`). +//! `docs/architecture.md`). The `--follow` tail (`LogTail`, `FollowFooter`, +//! `follow_service_logs`) is new code written here, not relocated. //! //! This module owns: the CLI/TUI logs browser (`render_logs_text`, //! `render_logs_browser_text`, `render_logs_browser_page_text`, @@ -20,8 +21,11 @@ //! status surfaces (`render_services_text`, `render_services_json`, //! `render_service_logs_text`, `render_service_logs_text_with_options`, //! `render_service_action_result`, `read_managed_service_manifest`, -//! `load_managed_service`), the internal `rocmd`-compatible status surface -//! (`render_internal_status_text`), the `stop_server` answer shape +//! `load_managed_service`), the `--follow` tail behind `rocm logs --service +//! --follow` and `rocm services logs --follow` +//! (`follow_service_logs`, `follow_service_logs_with`, the `LogTail` and +//! `LogSegment` types, `write_log_segments`), the internal `rocmd`-compatible +//! status surface (`render_internal_status_text`), the `stop_server` answer shape //! (`internal_stop_server_value`, `service_stop_unconfirmed`) and its renderer //! (`render_unconfirmed_service_stop`, `process_check_command`), plus //! `validate_service_id`, `list_log_files`, `read_optional_tail_lines`, and the @@ -36,8 +40,9 @@ //! `render_internal_status_text`, `render_service_logs_text`, //! `render_logs_browser_text`, and `render_logs_text` — re-imported via //! `use crate::services_text::{...};`. `services_cmd.rs`'s `services()` calls -//! `render_services_text`, `render_services_json` and `render_service_logs_text` -//! through its own imports, and its `run_approved_service_action` and +//! `render_services_text`, `render_services_json`, `render_service_logs_text` +//! and `follow_service_logs` through its own imports, `dispatch()` calls +//! `follow_service_logs` for `rocm logs --service --follow`, and its `run_approved_service_action` and //! `finish_service_action` also read `render_service_action_result`, //! `service_stop_unconfirmed` and `validate_service_id`. `service_records.rs` //! reads `validate_service_id` (in `service_record_artifacts` and @@ -75,8 +80,10 @@ use std::collections::VecDeque; use std::fmt::Write as _; use std::fs; -use std::io::{self, BufRead}; +use std::io::{self, BufRead, Read as _, Seek as _, SeekFrom}; use std::path::{Path, PathBuf}; +use std::thread; +use std::time::Duration; use anyhow::{Context, Result, bail}; use rocm_core::{AppPaths, ManagedServiceRecord}; @@ -526,6 +533,442 @@ pub(crate) fn render_service_action_result_with_helper_note( text } +/// How often `--follow` looks for new log output. +const LOG_FOLLOW_POLL: Duration = Duration::from_millis(500); +/// How often `--follow` re-reads the service record to see whether it stopped. +const LOG_FOLLOW_LIVENESS_EVERY: Duration = Duration::from_secs(2); + +/// Reads whatever a growing log file gained since the last [`Self::poll`]. +/// +/// Download bars (tqdm, huggingface_hub) redraw in place with `\r`, so a +/// finished segment ends at either `\r` or `\n`; a trailing partial segment is +/// held back until its terminator arrives. +struct LogTail { + path: PathBuf, + offset: u64, + pending: Vec, +} + +/// One completed piece of log output and whether it ended in a carriage return. +#[derive(Debug, PartialEq, Eq)] +struct LogSegment { + text: String, + redraw: bool, +} + +impl LogTail { + /// Start following at the file's current end. + fn at_end(path: &Path) -> Self { + let offset = fs::metadata(path).map_or(0, |meta| meta.len()); + Self { + path: path.to_path_buf(), + offset, + pending: Vec::new(), + } + } + + fn poll(&mut self) -> Vec { + let Ok(mut file) = fs::File::open(&self.path) else { + return Vec::new(); + }; + let len = file.metadata().map_or(0, |meta| meta.len()); + if len < self.offset { + // Truncated or replaced (a restart reuses the path): start over. + self.offset = 0; + self.pending.clear(); + } + if file.seek(SeekFrom::Start(self.offset)).is_err() { + return Vec::new(); + } + let mut fresh = Vec::new(); + if file.read_to_end(&mut fresh).is_err() { + return Vec::new(); + } + self.offset += fresh.len() as u64; + self.pending.extend_from_slice(&fresh); + + let mut segments = Vec::new(); + let mut start = 0; + let mut index = 0; + while index < self.pending.len() { + let byte = self.pending[index]; + // A `\r` at the very end of the data may be the first half of a `\r\n` + // that the next poll completes; hold it back rather than emit an + // empty segment (which would erase a final redraw frame on a tty). + if byte == b'\r' && index + 1 == self.pending.len() { + break; + } + if byte == b'\n' || byte == b'\r' { + let mut redraw = byte == b'\r'; + // The footer's row bookkeeping assumes the log text moves no cursor. + let text = rocm_core::terminal::strip_terminal_control_sequences( + &String::from_utf8_lossy(&self.pending[start..index]), + ); + // `\r\n` is one line break, not a redraw followed by an empty line. + if redraw && self.pending.get(index + 1) == Some(&b'\n') { + redraw = false; + index += 1; + } + segments.push(LogSegment { text, redraw }); + start = index + 1; + } + index += 1; + } + self.pending.drain(..start); + segments + } + + /// The unterminated output still held back, for when following ends: a + /// last write cut short by a crash is otherwise never shown. + fn finish(&mut self) -> Option { + let mut bytes = std::mem::take(&mut self.pending); + if bytes.last() == Some(&b'\r') { + bytes.pop(); + } + // Same stripping as `poll`: a cut-off write can end mid-escape. + (!bytes.is_empty()).then(|| LogSegment { + text: rocm_core::terminal::strip_terminal_control_sequences(&String::from_utf8_lossy( + &bytes, + )), + redraw: false, + }) + } +} + +/// Writes `segments`; returns whether the last one was a redraw, which leaves the +/// cursor on that row (the next segment erases it first) instead of a fresh one. +fn write_log_segments( + out: &mut impl io::Write, + segments: &[LogSegment], + tty: bool, +) -> Result { + for segment in segments { + match (tty, segment.redraw) { + // Redraw the same terminal row, as the engine's own bar does. + (true, true) => write!(out, "\r\x1b[2K{}", segment.text)?, + (true, false) => writeln!(out, "\r\x1b[2K{}", segment.text)?, + (false, _) => writeln!(out, "{}", segment.text)?, + } + } + out.flush()?; + Ok(tty && segments.last().is_some_and(|segment| segment.redraw)) +} + +/// The "how to stop" hint kept on the row just below the newest log output of a +/// terminal follow, so it stays in view however much the log scrolls. +/// +/// While drawn, the cursor sits on the footer row. [`Self::erase`] puts it back +/// where the log output left off: on a fresh row, or, after a redraw segment +/// (a download bar), at the end of that still-open row, one row up. +struct FollowFooter { + text: String, + drawn: bool, + /// The row above the footer is an unfinished redraw row. + open_row: bool, +} + +/// Led by the part that says the server keeps running: service ids run 40+ +/// characters, so on a narrow terminal the end of the hint is what gets cut. +fn follow_hint(service_id: &str) -> String { + format!("Ctrl-C stops following, the server keeps running. Following {service_id}") +} + +impl FollowFooter { + fn new(service_id: &str) -> Self { + Self { + text: follow_hint(service_id), + drawn: false, + open_row: false, + } + } + + fn erase(&mut self, out: &mut impl io::Write) -> Result<()> { + if self.drawn { + write!(out, "\r\x1b[2K")?; + if self.open_row { + write!(out, "\x1b[1A")?; + } + self.drawn = false; + } + Ok(()) + } + + /// Clear the footer and end an unfinished redraw row, leaving the cursor on + /// a fresh row. + fn finish(&mut self, out: &mut impl io::Write) -> Result<()> { + self.erase(out)?; + if self.open_row { + writeln!(out)?; + self.open_row = false; + } + out.flush()?; + Ok(()) + } + + /// One line of the follow's own, written below the log output with the + /// footer redrawn under it. An unfinished download bar row is ended first, + /// not overwritten. + fn write_notice( + &mut self, + out: &mut impl io::Write, + text: &str, + cols: Option, + ) -> Result<()> { + self.finish(out)?; + writeln!(out, "{text}")?; + self.draw(out, false, cols) + } + + /// `cols` truncates the text so it never wraps onto a second row, which the + /// one-row erase could not clear. + fn draw( + &mut self, + out: &mut impl io::Write, + open_row: bool, + cols: Option, + ) -> Result<()> { + if open_row { + writeln!(out)?; + } + let shown: String = match cols { + Some(cols) => { + crate::cli_progress::truncate_to_width(&self.text, cols.saturating_sub(1)) + } + None => self.text.clone(), + }; + write!(out, "\r\x1b[2K{shown}")?; + out.flush()?; + self.drawn = true; + self.open_row = open_row; + Ok(()) + } + + /// Write a batch of new output with the footer kept below it. A batch with + /// nothing in it leaves the screen alone (no flicker on an idle log). + fn write_batch( + &mut self, + out: &mut impl io::Write, + segments: &[LogSegment], + cols: Option, + ) -> Result<()> { + if segments.is_empty() { + return Ok(()); + } + self.erase(out)?; + let open_row = write_log_segments(out, segments, true)?; + self.draw(out, open_row, cols) + } +} + +fn terminal_cols() -> Option { + crossterm::terminal::size() + .ok() + .map(|(cols, _)| usize::from(cols)) +} + +/// `rocm logs --service --follow`: print the recent output, then keep +/// printing new output until the service is no longer starting or running. +/// Ctrl-C ends the follow and leaves the service running. +pub(crate) fn follow_service_logs(paths: &AppPaths, service_id: &str) -> Result<()> { + use std::io::IsTerminal as _; + let mut out = io::stdout(); + let tty = out.is_terminal(); + follow_service_logs_with( + paths, + service_id, + &mut out, + &mut io::stderr(), + tty, + LOG_FOLLOW_POLL, + LOG_FOLLOW_LIVENESS_EVERY, + &terminal_cols, + ) +} + +/// True once, the first time `status` reads `ready`. +fn newly_ready(announced: &mut bool, status: &str) -> bool { + let ready = status == "ready" && !*announced; + *announced |= ready; + ready +} + +/// A line about the follow itself (the hint, "is ready.", "stopped following."). +/// On a terminal it shares the log's stream and row layout; piped, it goes to +/// `err` so `rocm logs -f | grep error` sees log text only. +fn write_note( + out: &mut impl io::Write, + err: &mut impl io::Write, + tty: bool, + line: &str, +) -> Result<()> { + if tty { + writeln!(out, "{line}")?; + } else { + writeln!(err, "{line}")?; + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +fn follow_service_logs_with( + paths: &AppPaths, + service_id: &str, + out: &mut impl io::Write, + err: &mut impl io::Write, + tty: bool, + poll: Duration, + liveness_every: Duration, + cols: &dyn Fn() -> Option, +) -> Result<()> { + follow_service_logs_via( + paths, + service_id, + &|| load_managed_service(paths, service_id), + out, + err, + tty, + poll, + liveness_every, + cols, + ) +} + +/// [`follow_service_logs_with`] with the record read injected: a `ready` status +/// only sticks while the endpoint answers, so a test cannot plant one on disk. +#[allow(clippy::too_many_arguments)] +fn follow_service_logs_via( + paths: &AppPaths, + service_id: &str, + load: &dyn Fn() -> Result, + out: &mut impl io::Write, + err: &mut impl io::Write, + tty: bool, + poll: Duration, + liveness_every: Duration, + cols: &dyn Fn() -> Option, +) -> Result<()> { + let record = load()?; + // Open the tail before the snapshot: output written in between is then shown + // twice (in the snapshot and by the first poll) rather than lost, and the + // tail can start mid-line. + let mut tail = LogTail::at_end(&record.log_path); + write!(out, "{}", render_service_logs_text(paths, service_id)?)?; + out.flush()?; + if !managed_service_is_live(&record) { + write_note( + out, + err, + tty, + &format!( + "\nService {service_id} is {}; not following.", + record.status + ), + )?; + // Same outcome as a follow that sees the server fail: not a success. + if record.status == "failed" { + bail!("service {service_id} failed; see the log output above"); + } + return Ok(()); + } + // On a terminal the hint is a footer that stays under the newest output; a + // pipe or log file gets it once, since it has no row to keep it on. + let mut footer = tty.then(|| FollowFooter::new(service_id)); + match footer.as_mut() { + Some(footer) => footer.draw(out, false, cols())?, + None => write_note(out, err, tty, &format!("\n{}", follow_hint(service_id)))?, + } + out.flush()?; + + let result = follow_loop( + load, + &record, + &mut tail, + &mut footer, + out, + err, + tty, + poll, + liveness_every, + cols, + ); + if result.is_err() + && let Some(footer) = footer.as_mut() + { + // Leave the terminal on a clean row so the caller's `Error:` is not glued + // to the footer or to an unfinished download bar. + let _ = footer.finish(out); + } + result +} + +#[allow(clippy::too_many_arguments)] +fn follow_loop( + load: &dyn Fn() -> Result, + record: &ManagedServiceRecord, + tail: &mut LogTail, + footer: &mut Option, + out: &mut impl io::Write, + err: &mut impl io::Write, + tty: bool, + poll: Duration, + liveness_every: Duration, + cols: &dyn Fn() -> Option, +) -> Result<()> { + let service_id = record.service_id.as_str(); + let mut announced_ready = record.status == "ready"; + let mut since_check = Duration::ZERO; + loop { + thread::sleep(poll); + match footer.as_mut() { + Some(footer) => footer.write_batch(out, &tail.poll(), cols())?, + None => { + write_log_segments(out, &tail.poll(), tty)?; + } + } + since_check += poll; + if since_check < liveness_every { + continue; + } + since_check = Duration::ZERO; + let record = load()?; + if newly_ready(&mut announced_ready, &record.status) { + let line = format!("Service {service_id} is ready."); + match footer.as_mut() { + Some(footer) => footer.write_notice(out, &line, cols())?, + None => write_note(out, err, tty, &line)?, + } + } + if !managed_service_is_live(&record) { + // Output written between the last poll and the exit. + let mut last = tail.poll(); + last.extend(tail.finish()); + let mut open_row = false; + if let Some(footer) = footer.as_mut() { + footer.erase(out)?; + open_row = footer.open_row; + } + open_row = write_log_segments(out, &last, tty)? || (open_row && last.is_empty()); + if open_row { + writeln!(out)?; + } + write_note( + out, + err, + tty, + &format!( + "Service {service_id} is {}; stopped following.", + record.status + ), + )?; + // Same outcome as `serve --wait`: a server that failed is not a success. + if record.status == "failed" { + bail!("service {service_id} failed; see the log output above"); + } + return Ok(()); + } + } +} + pub(crate) fn render_service_action_result(tool: &str, value: &serde_json::Value) -> String { let output = value.get("output").unwrap_or(value); if tool == "stop_server" && service_stop_unconfirmed(value) { @@ -1113,6 +1556,775 @@ pub(crate) mod tests { Ok(()) } + #[test] + fn log_tail_yields_only_new_complete_segments() -> Result<()> { + let (root, paths) = test_paths("log-tail"); + paths.ensure()?; + let log = root.join("tail.log"); + fs::write(&log, "old line\n")?; + let mut tail = LogTail::at_end(&log); + assert!(tail.poll().is_empty()); + + // tqdm-style bar redraws with `\r`; the last piece has no terminator yet. + fs::write(&log, "old line\n 28%|##\r 33%|###\rdone\r\nhalf")?; + let segments = tail.poll(); + assert_eq!( + segments, + vec![ + LogSegment { + text: " 28%|##".into(), + redraw: true + }, + LogSegment { + text: " 33%|###".into(), + redraw: true + }, + LogSegment { + text: "done".into(), + redraw: false + }, + ] + ); + // The held-back partial segment completes on the next poll. + let mut file = fs::OpenOptions::new().append(true).open(&log)?; + std::io::Write::write_all(&mut file, b" line\n")?; + assert_eq!( + tail.poll(), + vec![LogSegment { + text: "half line".into(), + redraw: false + }] + ); + + // A truncated log (restart reusing the path) is read from the start. + fs::write(&log, "new\n")?; + assert_eq!( + tail.poll(), + vec![LogSegment { + text: "new".into(), + redraw: false + }] + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn log_tail_holds_back_a_trailing_cr_and_flushes_the_rest_on_finish() -> Result<()> { + let (root, paths) = test_paths("log-tail-edges"); + paths.ensure()?; + let log = root.join("edges.log"); + fs::write(&log, "")?; + let mut tail = LogTail::at_end(&log); + + // A `\r` ending the data may be half of a `\r\n`: it is not a redraw yet. + fs::write(&log, " 99%|#\r")?; + assert!(tail.poll().is_empty()); + let mut file = fs::OpenOptions::new().append(true).open(&log)?; + std::io::Write::write_all(&mut file, b"\n")?; + assert_eq!( + tail.poll(), + vec![LogSegment { + text: " 99%|#".into(), + redraw: false + }], + "the held `\\r` joins the next `\\n` instead of yielding an empty segment" + ); + + // Output cut short before any terminator is shown when following ends. + std::io::Write::write_all(&mut file, b"crashed mid-wri")?; + assert!(tail.poll().is_empty()); + assert_eq!( + tail.finish(), + Some(LogSegment { + text: "crashed mid-wri".into(), + redraw: false + }) + ); + assert_eq!(tail.finish(), None); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn write_log_segments_redraws_in_place_only_on_a_tty() -> Result<()> { + let segments = [ + LogSegment { + text: "28%".into(), + redraw: true, + }, + LogSegment { + text: "done".into(), + redraw: false, + }, + ]; + let mut tty = Vec::new(); + write_log_segments(&mut tty, &segments, true)?; + // A redraw segment leaves the cursor on its row; the next segment (or + // line) erases the row first, and a plain line ends with a newline. + assert_eq!(String::from_utf8(tty)?, "\r\x1b[2K28%\r\x1b[2Kdone\n"); + + let mut piped = Vec::new(); + write_log_segments(&mut piped, &segments, false)?; + assert_eq!(String::from_utf8(piped)?, "28%\ndone\n"); + Ok(()) + } + + fn seg(text: &str, redraw: bool) -> LogSegment { + LogSegment { + text: text.into(), + redraw, + } + } + + const ERASE: &str = "\r\x1b[2K"; + + #[test] + fn follow_footer_is_redrawn_under_each_batch_of_new_lines() -> Result<()> { + let mut footer = FollowFooter::new("svc_a"); + let hint = footer.text.clone(); + let mut out = Vec::new(); + footer.draw(&mut out, false, None)?; + assert_eq!(String::from_utf8(out.clone())?, format!("{ERASE}{hint}")); + + out.clear(); + footer.write_batch(&mut out, &[seg("one", false), seg("two", false)], None)?; + // Footer row cleared, lines written over it, footer redrawn on the next row. + assert_eq!( + String::from_utf8(out)?, + format!("{ERASE}{ERASE}one\n{ERASE}two\n{ERASE}{hint}") + ); + Ok(()) + } + + #[test] + fn follow_footer_leaves_an_idle_screen_alone() -> Result<()> { + let mut footer = FollowFooter::new("svc_a"); + let mut out = Vec::new(); + footer.draw(&mut out, false, None)?; + out.clear(); + footer.write_batch(&mut out, &[], None)?; + assert!(out.is_empty(), "{out:?}"); + Ok(()) + } + + #[test] + fn follow_footer_keeps_a_download_bar_on_its_own_row_above_it() -> Result<()> { + let mut footer = FollowFooter::new("svc_a"); + let hint = footer.text.clone(); + let mut out = Vec::new(); + footer.draw(&mut out, false, None)?; + out.clear(); + + footer.write_batch(&mut out, &[seg("28%", true)], None)?; + // The bar row stays open; the footer goes on the row below it. + assert_eq!( + String::from_utf8(out.clone())?, + format!("{ERASE}{ERASE}28%\n{ERASE}{hint}") + ); + + out.clear(); + footer.write_batch(&mut out, &[seg("43%", true)], None)?; + // Erase the footer, step back up onto the bar row, redraw it in place. + assert_eq!( + String::from_utf8(out.clone())?, + format!("{ERASE}\x1b[1A{ERASE}43%\n{ERASE}{hint}") + ); + + out.clear(); + footer.write_batch(&mut out, &[seg("done", false)], None)?; + // A finished line replaces the bar, and the footer is back on a fresh row. + assert_eq!( + String::from_utf8(out)?, + format!("{ERASE}\x1b[1A{ERASE}done\n{ERASE}{hint}") + ); + Ok(()) + } + + #[test] + fn follow_footer_is_cut_to_one_terminal_row() -> Result<()> { + let mut footer = FollowFooter::new("svc_a"); + let mut out = Vec::new(); + footer.draw(&mut out, false, Some(30))?; + // One column short of the width, so it never wraps, and what is kept is + // the head of the hint: the part saying the server keeps running. + let drawn = String::from_utf8(out)?; + let shown = drawn.strip_prefix(ERASE).expect("erase prefix"); + assert_eq!(unicode_width::UnicodeWidthStr::width(shown), 29, "{shown}"); + assert!(shown.starts_with("Ctrl-C stops following, the"), "{shown}"); + Ok(()) + } + + #[test] + fn log_tail_strips_cursor_escapes_from_engine_output() -> Result<()> { + let (root, _paths) = test_paths("tail-escapes"); + fs::create_dir_all(&root)?; + let log = root.join("engine.log"); + fs::write(&log, "")?; + let mut tail = LogTail::at_end(&log); + fs::write(&log, "before\x1b[1Aafter\n")?; + let segments = tail.poll(); + assert_eq!(segments.len(), 1); + assert_eq!(segments[0].text, "beforeafter"); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn log_tail_finish_strips_cursor_escapes_from_a_cut_off_write() -> Result<()> { + let (root, _paths) = test_paths("tail-finish-escapes"); + fs::create_dir_all(&root)?; + let log = root.join("engine.log"); + fs::write(&log, "")?; + let mut tail = LogTail::at_end(&log); + fs::write(&log, "last\x1b[1Aword")?; + assert!(tail.poll().is_empty()); + let segment = tail.finish().expect("unterminated output is returned"); + assert_eq!(segment.text, "lastword"); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn follow_service_logs_keeps_the_hint_under_new_output_on_a_tty() -> Result<()> { + let (root, paths) = test_paths("follow-tty"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "svc_follow_tty", + "vllm", + "qwen3.5", + "Qwen/Qwen3.5", + "127.0.0.1", + 11438, + "managed", + std::process::id(), + None, + None, + None, + ); + record.status = "starting".to_owned(); + record.write()?; + fs::write(&record.log_path, "before\n")?; + + let log_path = record.log_path; + let manifest_paths = paths.clone(); + let writer = thread::spawn(move || -> Result<()> { + let appended = (|| -> Result<()> { + thread::sleep(Duration::from_millis(150)); + let mut file = fs::OpenOptions::new().append(true).open(&log_path)?; + std::io::Write::write_all(&mut file, b"Downloading 43%\n")?; + thread::sleep(Duration::from_millis(150)); + Ok(()) + })(); + let mut record = load_managed_service(&manifest_paths, "svc_follow_tty")?; + record.status = "stopped".to_owned(); + record.write()?; + appended + }); + let mut out = Vec::new(); + let mut err = Vec::new(); + follow_service_logs_with( + &paths, + "svc_follow_tty", + &mut out, + &mut err, + true, + Duration::from_millis(20), + Duration::from_millis(40), + &|| None, + )?; + writer.join().expect("writer thread")?; + let out = String::from_utf8(out)?; + let hint = FollowFooter::new("svc_follow_tty").text; + // The hint is drawn again right under the line that scrolled it away, + // and only the stop message follows it, after the footer row is cleared. + assert!( + out.contains(&format!("Downloading 43%\n{ERASE}{hint}")), + "{out:?}" + ); + assert!( + out.ends_with(&format!( + "{hint}{ERASE}Service svc_follow_tty is stopped; stopped following.\n" + )), + "{out:?}" + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn follow_service_logs_ends_an_open_download_bar_row_before_the_stop_message() -> Result<()> { + let (root, paths) = test_paths("follow-tty-bar"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "svc_follow_bar", + "vllm", + "qwen3.5", + "Qwen/Qwen3.5", + "127.0.0.1", + 11440, + "managed", + std::process::id(), + None, + None, + None, + ); + record.status = "starting".to_owned(); + record.write()?; + fs::write(&record.log_path, "")?; + + let log_path = record.log_path; + let manifest_paths = paths.clone(); + let writer = thread::spawn(move || -> Result<()> { + let appended = (|| -> Result<()> { + thread::sleep(Duration::from_millis(150)); + let mut file = fs::OpenOptions::new().append(true).open(&log_path)?; + std::io::Write::write_all(&mut file, b"28%\r43%\r\r")?; + thread::sleep(Duration::from_millis(150)); + Ok(()) + })(); + let mut record = load_managed_service(&manifest_paths, "svc_follow_bar")?; + record.status = "stopped".to_owned(); + record.write()?; + appended + }); + let mut out = Vec::new(); + let mut err = Vec::new(); + follow_service_logs_with( + &paths, + "svc_follow_bar", + &mut out, + &mut err, + true, + Duration::from_millis(20), + Duration::from_millis(40), + &|| None, + )?; + writer.join().expect("writer thread")?; + let out = String::from_utf8(out)?; + // The footer is erased and the cursor goes back up onto the bar's row; that row + // is then ended with a newline so the stop message starts on its own row. + assert!( + out.ends_with(&format!( + "{ERASE}\x1b[1A\nService svc_follow_bar is stopped; stopped following.\n" + )), + "{out:?}" + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn follow_service_logs_draws_no_footer_for_a_stopped_service_on_a_tty() -> Result<()> { + let (root, paths) = test_paths("follow-tty-stopped"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "svc_follow_tty_stopped", + "vllm", + "qwen3.5", + "Qwen/Qwen3.5", + "127.0.0.1", + 11439, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write()?; + let mut out = Vec::new(); + let mut err = Vec::new(); + follow_service_logs_with( + &paths, + "svc_follow_tty_stopped", + &mut out, + &mut err, + true, + Duration::from_millis(1), + Duration::from_millis(1), + &|| None, + )?; + let out = String::from_utf8(out)?; + assert!(!out.contains("Following"), "{out:?}"); + assert!(out.contains("not following."), "{out:?}"); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn follow_service_logs_prints_snapshot_and_returns_for_a_stopped_service() -> Result<()> { + let (root, paths) = test_paths("follow-stopped"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "svc_follow_stopped", + "vllm", + "qwen3.5", + "Qwen/Qwen3.5", + "127.0.0.1", + 11436, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write()?; + fs::write(&record.log_path, "Downloading 86%\n")?; + + let mut out = Vec::new(); + let mut err = Vec::new(); + follow_service_logs_with( + &paths, + "svc_follow_stopped", + &mut out, + &mut err, + false, + Duration::from_millis(1), + Duration::from_millis(1), + &|| None, + )?; + let out = String::from_utf8(out)?; + let err = String::from_utf8(err)?; + assert!(out.contains("Downloading 86%"), "{out}"); + // The notice is not log text, so a pipe keeps it off stdout. + assert!(!out.contains("not following"), "{out}"); + assert!( + err.contains("Service svc_follow_stopped is stopped; not following."), + "{err}" + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn newly_ready_fires_once_on_the_first_ready_status() { + let mut announced = false; + assert!(!newly_ready(&mut announced, "starting")); + assert!(newly_ready(&mut announced, "ready")); + assert!(!newly_ready(&mut announced, "ready")); + // A follow that began on an already-ready server has announced nothing new. + let mut announced = true; + assert!(!newly_ready(&mut announced, "ready")); + } + + #[test] + fn follow_service_logs_fails_when_the_service_failed() -> Result<()> { + let (root, paths) = test_paths("follow-ready-failed"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "svc_follow_fail", + "vllm", + "qwen3.5", + "Qwen/Qwen3.5", + "127.0.0.1", + 11438, + "managed", + std::process::id(), + None, + None, + None, + ); + record.status = "starting".to_owned(); + record.write()?; + fs::write(&record.log_path, "before\n")?; + + let manifest_paths = paths.clone(); + let writer = thread::spawn(move || -> Result<()> { + thread::sleep(Duration::from_millis(200)); + let mut record = load_managed_service(&manifest_paths, "svc_follow_fail")?; + record.status = "failed".to_owned(); + record.write() + }); + let mut out = Vec::new(); + let mut err = Vec::new(); + let result = follow_service_logs_with( + &paths, + "svc_follow_fail", + &mut out, + &mut err, + false, + Duration::from_millis(20), + Duration::from_millis(40), + &|| None, + ); + writer.join().expect("writer thread")?; + let err = String::from_utf8(err)?; + assert!(err.contains("is failed; stopped following"), "{err}"); + let error = result.expect_err("a failed service must not exit 0"); + assert!( + error.to_string().contains("svc_follow_fail failed"), + "{error}" + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn follow_service_logs_streams_new_output_until_the_service_stops() -> Result<()> { + let (root, paths) = test_paths("follow-live"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "svc_follow_live", + "vllm", + "qwen3.5", + "Qwen/Qwen3.5", + "127.0.0.1", + 11437, + "managed", + std::process::id(), + None, + None, + None, + ); + record.status = "starting".to_owned(); + record.write()?; + fs::write(&record.log_path, "before\n")?; + + let log_path = record.log_path; + let manifest_paths = paths.clone(); + let writer = thread::spawn(move || -> Result<()> { + let appended = (|| -> Result<()> { + thread::sleep(Duration::from_millis(150)); + let mut file = fs::OpenOptions::new().append(true).open(&log_path)?; + std::io::Write::write_all(&mut file, b"Downloading 43%\n")?; + thread::sleep(Duration::from_millis(150)); + Ok(()) + })(); + // Stop the service even when the append failed, so the follow loop + // under test always ends instead of hanging the run. + let mut record = load_managed_service(&manifest_paths, "svc_follow_live")?; + record.status = "stopped".to_owned(); + record.write()?; + appended + }); + let mut out = Vec::new(); + let mut err = Vec::new(); + follow_service_logs_with( + &paths, + "svc_follow_live", + &mut out, + &mut err, + false, + Duration::from_millis(20), + Duration::from_millis(40), + &|| None, + )?; + writer.join().expect("writer thread")?; + let out = String::from_utf8(out)?; + let err = String::from_utf8(err)?; + // Piped: stdout is the log text alone, with no terminal escapes, and the + // follow's own lines go to stderr, the hint exactly once. + assert!(out.contains("Downloading 43%"), "{out}"); + assert!(!out.contains('\x1b'), "{out:?}"); + assert!(!out.contains("Following"), "{out}"); + assert!(!out.contains("stopped following"), "{out}"); + assert_eq!(err.matches("Following svc_follow_live").count(), 1, "{err}"); + assert!(err.contains("stopped following"), "{err}"); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + fn starting_record(paths: &AppPaths, id: &str, port: u16) -> Result { + let mut record = ManagedServiceRecord::new( + paths, + id, + "vllm", + "qwen3.5", + "Qwen/Qwen3.5", + "127.0.0.1", + port, + "managed", + std::process::id(), + None, + None, + None, + ); + record.status = "starting".to_owned(); + record.write()?; + Ok(record) + } + + /// A record read that walks through `statuses` (the last one repeats). + fn scripted_statuses( + base: ManagedServiceRecord, + statuses: &'static [&'static str], + ) -> impl Fn() -> Result { + let calls = std::cell::Cell::new(0_usize); + move || { + let call = calls.get(); + calls.set(call + 1); + let mut record = base.clone(); + record.status = statuses[call.min(statuses.len() - 1)].to_owned(); + Ok(record) + } + } + + #[test] + fn follow_service_logs_announces_ready_once_when_the_server_comes_up() -> Result<()> { + let (root, paths) = test_paths("follow-ready-line"); + paths.ensure()?; + let record = starting_record(&paths, "svc_follow_ready", 11441)?; + fs::write(&record.log_path, "before\n")?; + // Initial read, then one read per liveness check. + let load = scripted_statuses( + record, + &["starting", "starting", "ready", "ready", "stopped"], + ); + let mut out = Vec::new(); + let mut err = Vec::new(); + follow_service_logs_via( + &paths, + "svc_follow_ready", + &load, + &mut out, + &mut err, + false, + Duration::from_millis(5), + Duration::from_millis(5), + &|| None, + )?; + let err = String::from_utf8(err)?; + assert_eq!( + err.matches("Service svc_follow_ready is ready.").count(), + 1, + "{err}" + ); + assert!(!String::from_utf8(out)?.contains("is ready.")); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn follow_service_logs_ends_a_download_bar_row_before_the_ready_line_on_a_tty() -> Result<()> { + let (root, paths) = test_paths("follow-ready-bar"); + paths.ensure()?; + let record = starting_record(&paths, "svc_follow_rbar", 11442)?; + fs::write(&record.log_path, "")?; + let log_path = record.log_path.clone(); + let appender = thread::spawn(move || -> Result<()> { + thread::sleep(Duration::from_millis(50)); + let mut file = fs::OpenOptions::new().append(true).open(&log_path)?; + std::io::Write::write_all(&mut file, b"28%\r43%\r")?; + Ok(()) + }); + // The bar is on screen when the status turns ready (liveness check 3 on). + let calls = std::cell::Cell::new(0_usize); + let base = record; + let load = move || -> Result { + let call = calls.get(); + calls.set(call + 1); + if call >= 3 { + thread::sleep(Duration::from_millis(30)); + } + let mut record = base.clone(); + record.status = match call { + 0..=7 => "starting", + 8..=9 => "ready", + _ => "stopped", + } + .to_owned(); + Ok(record) + }; + let mut out = Vec::new(); + let mut err = Vec::new(); + follow_service_logs_via( + &paths, + "svc_follow_rbar", + &load, + &mut out, + &mut err, + true, + Duration::from_millis(10), + Duration::from_millis(10), + &|| None, + )?; + appender.join().expect("appender thread")?; + let out = String::from_utf8(out)?; + // The bar's row is ended with a newline, then the ready line gets its own + // row; it never replaces the bar. + assert!( + out.contains(&format!( + "{ERASE}\x1b[1A\nService svc_follow_rbar is ready.\n" + )), + "{out:?}" + ); + assert!(err.is_empty(), "a terminal follow writes nothing to stderr"); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn follow_service_logs_clears_the_footer_when_the_follow_errors() -> Result<()> { + let (root, paths) = test_paths("follow-error"); + paths.ensure()?; + let record = starting_record(&paths, "svc_follow_gone", 11443)?; + fs::write(&record.log_path, "")?; + let manifest = record.manifest_path; + // The service record disappears mid-follow, so the next liveness read fails. + let remover = thread::spawn(move || -> Result<()> { + thread::sleep(Duration::from_millis(100)); + fs::remove_file(manifest)?; + Ok(()) + }); + let mut out = Vec::new(); + let mut err = Vec::new(); + let result = follow_service_logs_with( + &paths, + "svc_follow_gone", + &mut out, + &mut err, + true, + Duration::from_millis(20), + Duration::from_millis(40), + &|| None, + ); + remover.join().expect("remover thread")?; + assert!(result.is_err(), "a vanished record must end the follow"); + let out = String::from_utf8(out)?; + // The footer row is erased, so `Error:` starts on a clean row. + assert!(out.ends_with(ERASE), "{out:?}"); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn follow_service_logs_fails_for_a_service_that_is_already_failed() -> Result<()> { + let (root, paths) = test_paths("follow-already-failed"); + paths.ensure()?; + let mut record = starting_record(&paths, "svc_follow_dead", 11444)?; + record.status = "failed".to_owned(); + record.supervisor_pid = 0; + record.write()?; + let mut out = Vec::new(); + let mut err = Vec::new(); + let result = follow_service_logs_with( + &paths, + "svc_follow_dead", + &mut out, + &mut err, + false, + Duration::from_millis(1), + Duration::from_millis(1), + &|| None, + ); + let error = result.expect_err("a failed service must not exit 0"); + assert!( + error.to_string().contains("svc_follow_dead failed"), + "{error}" + ); + assert!(String::from_utf8(err)?.contains("is failed; not following.")); + let _ = fs::remove_dir_all(root); + Ok(()) + } + #[test] fn render_service_logs_text_tails_manifest_log() -> Result<()> { let (root, paths) = test_paths("service-logs"); diff --git a/docs/architecture.md b/docs/architecture.md index 4524094e24..e3036f1d07 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -85,7 +85,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`service_launch.rs`](../apps/rocm/src/service_launch.rs) — mechanical relocation, owns types (Phase 6b, ROCMAI-91): spawning/attaching to a managed service process and streaming its log, plus the endpoint-key/auth checks that gate a public bind — `AttachOutcome`/`RawModeGuard`/`ManagedSpawn`/`AutostartClaim` are its private types; `ManagedLaunchReport` is `pub(crate)`, since [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) reads its fields directly for the interactive summary. Most of this module's functions are `pub(crate)` for the same reason — other modules, including [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`automations.rs`](../apps/rocm/src/automations.rs), [`service_control.rs`](../apps/rocm/src/service_control.rs), and [`services_text.rs`](../apps/rocm/src/services_text.rs), call into it; `main.rs` does not. `ServicesCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs) rather than moving alongside, and `services()` lives in [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs); the record removal and pruning it dispatches to now lives in [`service_records.rs`](../apps/rocm/src/service_records.rs), and only stop/restart are also reachable from the chat/LLM sandbox tool-calling path, out of scope for Phase 6b. - [`service_records.rs`](../apps/rocm/src/service_records.rs) — mechanical relocation, owns types: removing one managed-service record and every file it owns (`remove_managed_service_record`, `service_record_artifacts`) and the `rocm services prune` plan/render/apply steps (`build_service_prune_plan`, `render_service_prune_plan`, `apply_service_prune_plan`, `prune_managed_service_records`) with the age gate behind them (`DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS`, `service_prune_min_age_hours`). It reads record state through [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) and [`services_text.rs`](../apps/rocm/src/services_text.rs) and builds its advice text with [`service_hints.rs`](../apps/rocm/src/service_hints.rs); none of the three depends on it. `remove_managed_service_record`, `prune_managed_service_records`, `service_prune_min_age_hours` and `DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS` are `pub(crate)` (`services()` in [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs) calls the first three; the `ServicesCommand::Prune` clap definition in [`cli.rs`](../apps/rocm/src/cli.rs) uses the constant), and `ServicePruneOutcome`, the return type of `prune_managed_service_records`, is `pub(crate)` with `pub(crate)` `text`, `removed_records`, `removed_files` and `failures` fields that `services()` reads. Every other item is private. The cluster's regression tests live in this module's own `mod tests`, reaching the shared `test_paths`, `plant_service_record` and `plant_service_record_without_its_key` fixtures through `crate::tests::` (the last two widened to `pub(crate)` for that). - [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm services`' command handler (`services()`, which dispatches `list`, `logs`, `stop`, `restart`, `remove` and `prune`) and `run_approved_service_action`, the approval-gated stop/restart path. `ServicesCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs). Both functions are `pub(crate)`: `dispatch()` calls `services()` through a re-import, and [`advised_commands.rs`](../apps/rocm/src/advised_commands.rs)'s test reaches `run_approved_service_action` through a test-only re-import in `main.rs`. The module reads from [`services_text.rs`](../apps/rocm/src/services_text.rs), [`service_control.rs`](../apps/rocm/src/service_control.rs), [`service_records.rs`](../apps/rocm/src/service_records.rs), [`service_hints.rs`](../apps/rocm/src/service_hints.rs) and [`cli_audit.rs`](../apps/rocm/src/cli_audit.rs); none of them calls back into it. The cluster's one regression test lives in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::`. -- [`services_text.rs`](../apps/rocm/src/services_text.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-209): the CLI/TUI logs browser (`render_logs_text`, `render_logs_browser_text`, `render_logs_browser_page_text`, `render_logs_browser_page_text_with_options`, `logs_browser_total_pages`, the `LogBrowserEntry` type, `collect_log_browser_entries`, `filter_log_browser_entries`, `log_browser_source_label`, `write_past_attempts_hint`), the `rocm services`/`rocm services logs` status surfaces (`render_services_text`, `render_services_json`, `render_service_logs_text`, `render_service_logs_text_with_options`, `render_service_action_result`, `read_managed_service_manifest`, `load_managed_service`), the internal `rocmd`-compatible status surface (`render_internal_status_text`), the `stop_server` answer shape and its renderer (`internal_stop_server_value`, `service_stop_unconfirmed`, `render_unconfirmed_service_stop`, `process_check_command`), plus `validate_service_id`, `list_log_files`, `read_optional_tail_lines`, and the CLI-lifecycle-log tail formatting helpers (`format_cli_lifecycle_tail_line`, `lifecycle_event_label`, `humanize_log_label`, `lifecycle_field`, `read_tail_lines`). The managed-service stop and restart implementation that used to live here is in [`service_control.rs`](../apps/rocm/src/service_control.rs). The `LogBrowserEntry` struct moved here from `main.rs` with the logs browser and stays private. `dispatch()` calls directly into this module at several points — `render_internal_status_text`, `render_service_logs_text`, `render_logs_browser_text`, and `render_logs_text` — all re-imported via `use crate::services_text::{...};`; [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs)'s `services()` calls `render_services_text`, `render_services_json` and `render_service_logs_text` through its own imports. `render_service_action_result`, `service_stop_unconfirmed` and `validate_service_id` are also `pub(crate)` and read from `services_cmd.rs`'s `run_approved_service_action`; `validate_service_id` is also read from [`service_records.rs`](../apps/rocm/src/service_records.rs)'s `remove_managed_service_record`/`service_record_artifacts`, and `load_managed_service` from its `remove_managed_service_record`/`apply_service_prune_plan`. `load_managed_service` is `pub(crate)`, read from [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) (reconciling live servers against a runtime activation), from [`service_control.rs`](../apps/rocm/src/service_control.rs) and from [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs)'s own test module; `read_managed_service_manifest` and `internal_stop_server_value` are `pub(crate)` for `service_control.rs`, and `process_check_command` for its tests; `read_tail_lines`, `render_logs_text`, `render_logs_browser_text`, `render_services_text`, `render_service_logs_text`, and `validate_service_id` are also `pub(crate)`, read from [`chat.rs`](../apps/rocm/src/chat.rs)'s deterministic tool-output summaries; `read_optional_tail_lines` and `render_service_action_result` are also `pub(crate)`, read from [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s managed-spawn/attach path. Every other item in this list has no caller outside this module and stays private. The cluster's regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` the same way other mechanically relocated modules do; `assert_no_retry_line` stays here as a `pub(crate)` test helper because it checks the unconfirmed-stop rendering and `service_launch.rs`'s tests read it too. Two tests that call this module's `load_managed_service` only incidentally — their primary subject is `runtime_services::pin_service_record_to_runtime`/`restore_service_record_pin` and `runtime_services::restart_service_onto_runtime_with`'s outcome classification respectively — stay in `main.rs`'s shared `mod tests` instead. +- [`services_text.rs`](../apps/rocm/src/services_text.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-209): the CLI/TUI logs browser (`render_logs_text`, `render_logs_browser_text`, `render_logs_browser_page_text`, `render_logs_browser_page_text_with_options`, `logs_browser_total_pages`, the `LogBrowserEntry` type, `collect_log_browser_entries`, `filter_log_browser_entries`, `log_browser_source_label`, `write_past_attempts_hint`), the `rocm services`/`rocm services logs` status surfaces (`render_services_text`, `render_services_json`, `render_service_logs_text`, `render_service_logs_text_with_options`, `render_service_action_result`, `read_managed_service_manifest`, `load_managed_service`), the `--follow` tail behind `rocm logs --service --follow` and `rocm services logs --follow` (`follow_service_logs`, `follow_service_logs_with`, the `LogTail`, `LogSegment` and `FollowFooter` types, `write_log_segments`, `newly_ready`; new code written in this module, not relocated), the internal `rocmd`-compatible status surface (`render_internal_status_text`), the `stop_server` answer shape and its renderer (`internal_stop_server_value`, `service_stop_unconfirmed`, `render_unconfirmed_service_stop`, `process_check_command`), plus `validate_service_id`, `list_log_files`, `read_optional_tail_lines`, and the CLI-lifecycle-log tail formatting helpers (`format_cli_lifecycle_tail_line`, `lifecycle_event_label`, `humanize_log_label`, `lifecycle_field`, `read_tail_lines`). The managed-service stop and restart implementation that used to live here is in [`service_control.rs`](../apps/rocm/src/service_control.rs). The `LogBrowserEntry` struct moved here from `main.rs` with the logs browser and stays private. `dispatch()` calls directly into this module at several points — `render_internal_status_text`, `render_service_logs_text`, `render_logs_browser_text`, and `render_logs_text` — all re-imported via `use crate::services_text::{...};`; [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs)'s `services()` calls `render_services_text`, `render_services_json` and `render_service_logs_text` through its own imports. `render_service_action_result`, `service_stop_unconfirmed` and `validate_service_id` are also `pub(crate)` and read from `services_cmd.rs`'s `run_approved_service_action`; `validate_service_id` is also read from [`service_records.rs`](../apps/rocm/src/service_records.rs)'s `remove_managed_service_record`/`service_record_artifacts`, and `load_managed_service` from its `remove_managed_service_record`/`apply_service_prune_plan`. `load_managed_service` is `pub(crate)`, read from [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) (reconciling live servers against a runtime activation), from [`service_control.rs`](../apps/rocm/src/service_control.rs) and from [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs)'s own test module; `read_managed_service_manifest` and `internal_stop_server_value` are `pub(crate)` for `service_control.rs`, and `process_check_command` for its tests; `read_tail_lines`, `render_logs_text`, `render_logs_browser_text`, `render_services_text`, `render_service_logs_text`, and `validate_service_id` are also `pub(crate)`, read from [`chat.rs`](../apps/rocm/src/chat.rs)'s deterministic tool-output summaries; `read_optional_tail_lines` and `render_service_action_result` are also `pub(crate)`, read from [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s managed-spawn/attach path. Every other item in this list has no caller outside this module and stays private. The cluster's regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` the same way other mechanically relocated modules do; `assert_no_retry_line` stays here as a `pub(crate)` test helper because it checks the unconfirmed-stop rendering and `service_launch.rs`'s tests read it too. Two tests that call this module's `load_managed_service` only incidentally — their primary subject is `runtime_services::pin_service_record_to_runtime`/`restore_service_record_pin` and `runtime_services::restart_service_onto_runtime_with`'s outcome classification respectively — stay in `main.rs`'s shared `mod tests` instead. - [`setup_cmd.rs`](../apps/rocm/src/setup_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm setup`'s command handler (`setup()`, which runs `setup status` by default and `setup reset`), the text `rocm setup status` prints (`render_setup_status_text`) and the reset that clears the saved first-time prompt state (`reset_setup_prompt_state`). `SetupCommand` lives in [`cli.rs`](../apps/rocm/src/cli.rs). `setup()` and `render_setup_status_text` are `pub(crate)` and re-imported in `main.rs`: `dispatch()` calls the first, and [`chat.rs`](../apps/rocm/src/chat.rs)'s read-only handling of `rocm setup` and `rocm setup status` reaches the second as `crate::render_setup_status_text`; `reset_setup_prompt_state` is private. The three regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` and `write_test_pip_runtime` through `crate::test_support::`. - [`shell_quote.rs`](../apps/rocm/src/shell_quote.rs) — shared CLI-output component (`shell_quote`): the CLI's name for `rocm-core`'s POSIX quoter, [`posix_shell_quote`](../crates/rocm-core/src/quoting.rs). `rocm-dash-tui` keeps a copy as `ui::exec::quote_display_arg` in [`exec.rs`](../crates/rocm-dash-tui/src/ui/exec.rs), since the crate-edge check forbids it `rocm-core`; a per-character walk and a property test here hold the two to the same output. It covers the remote `ssh` command line, the chat approval card and the plan for a free-form request, the install dry-run `command:` line, the [`service_hints.rs`](../apps/rocm/src/service_hints.rs) hints, the runtime-activation report, the `rocm services logs ` lines `rocm services` lists and the `rocm comfyui install --runtime-id` advice a failed ComfyUI reinstall prints — not every `rocm …` hint the CLI prints; older ones such as `rocm logs ` still interpolate their value bare. Also holds `has_terminal_control_character`, the predicate the chat tool-call guards refuse on. - [`storage.rs`](../apps/rocm/src/storage.rs) — full domain extraction. diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 6b18e694a1..9a5a68ccd5 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -402,6 +402,8 @@ After a managed or foreground serve attempt, inspect local server records: rocm services rocm services list --all rocm services logs +rocm services logs --follow +rocm serve --wait # waits past 45s and returns once the server answers or its engine gives up (also for a matching server still starting; an already-serving one returns at once) ``` Expected result: @@ -419,6 +421,21 @@ Expected result: block at all. - `rocm services list --all` shows saved history, including the failed or stopped attempts the default view hides. +- `rocm services logs --follow` prints the recent output, then new + output as it arrives (a download bar redraws on one terminal row), and returns + by itself with `Service is ; stopped following.` once the server + stops (exit non-zero if it ended `failed`), with a `Service is ready.` + line when it finishes starting; Ctrl-C ends the follow and the server keeps running. + On a terminal, a `Ctrl-C stops following, the server keeps running. Following ` line stays on the + row directly under the newest output (and under the download bar) as the log + grows; piped, stdout carries only the log text and the hint (once), ready and + stopped-following lines go to stderr. A server already `failed` when the follow + starts also exits non-zero. +- `rocm serve --wait` keeps waiting past 45 seconds and returns once the + server answers or its engine gives up (vLLM and AIM: 5 minutes by default, override + with `ROCM_CLI_VLLM_READY_TIMEOUT_SECS` / `ROCM_CLI_AIM_READY_TIMEOUT_SECS`). If the server is killed or stopped meanwhile, it exits non-zero + with `the server exited before it became ready` and a `rocm logs --service` + pointer, and the record shows `stopped`. - The logs command shows the exact service failure or startup output. - `rocm storage report` lists these records under `local server records` with the folder that holds them; no `rocm storage` command deletes them, and the diff --git a/docs/testing.md b/docs/testing.md index c6339bd937..a2c8dd2fe2 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1064,8 +1064,16 @@ Log browser smoke: rocm logs rocm logs --search server rocm logs --service +rocm logs --service --follow # Ctrl-C to stop following ``` +While a first-time model download is running, `rocm serve --wait` keeps +waiting past the usual 45 seconds until the server answers or its engine gives up +(vLLM and AIM stop after 5 minutes by default, see `ROCM_CLI_VLLM_READY_TIMEOUT_SECS` +and `ROCM_CLI_AIM_READY_TIMEOUT_SECS`); either way an exit fails the command. `--wait` with `--verbose` is refused (`serve-43`); `--foreground` is refused +by the argument parser, covered by a unit test. Run the log follow above from a +second terminal to watch the download. + The no-log first-run path should report `no CLI logs found yet` without creating the configured `data/logs` directory. diff --git a/tests/e2e-cucumber/features/logs.feature b/tests/e2e-cucumber/features/logs.feature index ddd97f0248..c58430162b 100644 --- a/tests/e2e-cucumber/features/logs.feature +++ b/tests/e2e-cucumber/features/logs.feature @@ -24,3 +24,39 @@ Feature: Log inspection Given recorded command logs containing several lines about a topic When the user asks for one service's logs and a search term together Then the CLI refuses and explains only one may be used + + # `--follow` keeps printing new output while a server starts (a model download + # can take many minutes). It ends by itself once the server is no longer + # starting or running, so the scenario plants a stopped record: the snapshot + # is printed, the CLI says it is not following, and the command returns + # instead of waiting. Streaming new output from a live service is covered by + # `follow_service_logs_streams_new_output_until_the_service_stops` in + # `apps/rocm/src/services_text.rs`. + @id:logs-follow-stopped-service-returns + Scenario: logs-04 - Following a stopped server's logs prints them and returns + Given a stopped local server whose log shows a download at 43 percent + When the user follows that server's logs + Then the CLI prints the download line, says it is not following, and exits + + @id:logs-follow-requires-service + Scenario: logs-05 - Following without naming a server is refused + When the user asks to follow logs without naming a service + Then the CLI refuses because --follow needs --service + + # The same follow behavior through the `rocm services logs` entry point, using + # the `-f` short form (the `rocm logs` scenarios above use `--follow`). + @id:logs-services-follow-stopped-service-returns + Scenario: logs-06 - Following a stopped server's logs through the services command returns + Given a stopped local server whose log shows a download at 43 percent + When the user follows that server's logs with rocm services logs -f + Then the CLI prints the download line, says it is not following, and exits + + # A server that ended `failed` is not a success, whether `--follow` finds it + # already failed (this scenario) or sees it fail while following (covered by + # `follow_service_logs_fails_when_the_service_failed` in + # `apps/rocm/src/services_text.rs`). + @id:logs-follow-failed-service-exits-nonzero + Scenario: logs-07 - Following a failed server's logs prints them and exits with an error + Given a failed local server whose log shows a download at 43 percent + When the user follows that server's logs + Then the CLI prints the download line, says it is not following, and exits with an error diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 97b2d91677..2bdccc5a7b 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -662,6 +662,7 @@ Feature: Model serving Then serving is refused before any engine starts And the CLI says the image needs eight GPUs and multi-GPU serving is not supported And Docker was never called + # ROCMAI-827: `rocm serve` started the background automation helper # (`rocm daemon`) on every launch, even with every automation watcher off — # when it has nothing to do — and that helper outlived the service. With no @@ -673,7 +674,6 @@ Feature: Model serving Given a managed runtime is active When the user serves a GGUF model on lemonade with no automation enabled Then serving started no background helper - # The positive case for the OOM note (EAI-8059 review asked for it # explicitly, alongside the withholding the unit tests pin): when # THIS invocation launches a managed vLLM serve that OOMs and never becomes @@ -720,3 +720,13 @@ Feature: Model serving And the user is told no AMD GPU was detected And the refusal points the user at the WSL driver status instead of /dev/kfd And inspecting that machine reports the driver status as missing, which is what the refusal sends the user to check + + # `--wait` makes a managed serve wait for readiness past the usual 45 seconds, + # so it cannot be combined with the attached modes, which already stay on the + # terminal. The waiting itself (a live server that becomes ready, one that + # exits first) is covered by `wait_past_readiness_budget_*` in + # `apps/rocm/src/service_launch.rs`, since a scenario would need a slow engine. + @id:serve-wait-conflicts-with-attached-modes + Scenario: serve-43 - Waiting for readiness cannot be combined with attached mode + When the user serves a model with both --wait and --verbose + Then the CLI refuses and names the conflicting flags diff --git a/tests/e2e-cucumber/tests/e2e/logs_steps.rs b/tests/e2e-cucumber/tests/e2e/logs_steps.rs index ab0986d0de..e6e31444c2 100644 --- a/tests/e2e-cucumber/tests/e2e/logs_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/logs_steps.rs @@ -123,3 +123,123 @@ fn ok_output(world: &E2eWorld) -> String { assert_eq!(rc, 0, "expected success, got rc={rc}:\n{}", combined(world)); world.cli_output.clone().unwrap_or_default() } + +// ── --follow ─────────────────────────────────────────────────────── + +const FOLLOW_SERVICE_ID: &str = "vllm-e2e-follow"; + +#[given("a stopped local server whose log shows a download at 43 percent")] +async fn plant_stopped_server_with_log(world: &mut E2eWorld) { + plant_server_with_log(world, "stopped"); +} + +#[given("a failed local server whose log shows a download at 43 percent")] +async fn plant_failed_server_with_log(world: &mut E2eWorld) { + plant_server_with_log(world, "failed"); +} + +fn plant_server_with_log(world: &mut E2eWorld, status: &str) { + let root = world.isolated_root.as_ref().expect("no isolated root"); + let data = root.path().join("data"); + let services = data.join("services"); + let states = data.join("engines").join("vllm").join("state"); + std::fs::create_dir_all(&services).expect("failed to create services dir"); + std::fs::create_dir_all(&states).expect("failed to create engine state dir"); + let log_path = services.join(format!("{FOLLOW_SERVICE_ID}.log")); + std::fs::write(&log_path, "Downloading model.safetensors: 43%\n").expect("failed to write log"); + let record = serde_json::json!({ + "service_id": FOLLOW_SERVICE_ID, + "engine": "vllm", + "model_ref": "qwen", + "canonical_model_id": "Qwen/Qwen3.5", + "host": "127.0.0.1", + "port": 9, + "endpoint_url": "http://127.0.0.1:9/v1", + "mode": "managed", + "status": status, + "supervisor_pid": 0, + "engine_pid": null, + "manifest_path": services.join(format!("{FOLLOW_SERVICE_ID}.json")), + "log_path": log_path, + "engine_state_path": states.join(format!("{FOLLOW_SERVICE_ID}.json")), + "created_at_unix_ms": 1_700_000_000_000_u64, + }); + std::fs::write( + services.join(format!("{FOLLOW_SERVICE_ID}.json")), + serde_json::to_vec_pretty(&record).expect("failed to serialize record"), + ) + .expect("failed to write service record"); +} + +#[when("the user follows that server's logs")] +async fn follow_stopped_server(world: &mut E2eWorld) { + let (stdout, stderr, rc) = + crate::run_rocm(world, &["logs", "--service", FOLLOW_SERVICE_ID, "--follow"]); + record(world, stdout, stderr, rc); +} + +#[when("the user follows that server's logs with rocm services logs -f")] +async fn follow_stopped_server_through_services(world: &mut E2eWorld) { + let (stdout, stderr, rc) = + crate::run_rocm(world, &["services", "logs", FOLLOW_SERVICE_ID, "-f"]); + record(world, stdout, stderr, rc); +} + +#[when("the user asks to follow logs without naming a service")] +async fn follow_without_service(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["logs", "--follow"]); + record(world, stdout, stderr, rc); +} + +#[then("the CLI prints the download line, says it is not following, and exits")] +async fn prints_and_exits(world: &mut E2eWorld) { + let out = ok_output(world); + let err = world.cli_stderr.clone().unwrap_or_default(); + assert!( + out.contains("Downloading model.safetensors: 43%"), + "expected the planted log line, got:\n{out}" + ); + // Piped, stdout is log text only; the notice about the follow goes to stderr. + let note = format!("Service {FOLLOW_SERVICE_ID} is stopped; not following."); + assert!(!out.contains(¬e), "notice leaked onto stdout:\n{out}"); + assert!( + err.contains(¬e), + "expected the not-following line, got:\n{err}" + ); +} + +#[then("the CLI prints the download line, says it is not following, and exits with an error")] +async fn prints_and_fails(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no command rc recorded"); + let out = world.cli_output.clone().unwrap_or_default(); + let err = world.cli_stderr.clone().unwrap_or_default(); + assert_ne!( + rc, 0, + "a failed server must not follow as success:\n{out}\n{err}" + ); + assert!( + out.contains("Downloading model.safetensors: 43%"), + "expected the planted log line, got:\n{out}" + ); + assert!( + err.contains(&format!( + "Service {FOLLOW_SERVICE_ID} is failed; not following." + )), + "expected the not-following line, got:\n{err}" + ); + assert!( + err.contains(&format!("service {FOLLOW_SERVICE_ID} failed")), + "expected the failure error, got:\n{err}" + ); +} + +#[then("the CLI refuses because --follow needs --service")] +async fn refuses_follow(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no command rc recorded"); + assert!(rc != 0, "expected refusal, got rc=0:\n{}", combined(world)); + assert!( + combined(world).contains("--service"), + "expected the error to name --service, got:\n{}", + combined(world) + ); +} diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 455a5ae46d..be384a80a9 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -2732,6 +2732,29 @@ async fn assert_stop_hint_clears_the_service(world: &mut E2eWorld) { ); } +#[when("the user serves a model with both --wait and --verbose")] +async fn serve_wait_and_verbose(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["serve", "qwen", "--wait", "--verbose"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the CLI refuses and names the conflicting flags")] +async fn serve_wait_refused(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no command rc recorded"); + let text = format!( + "{}\n{}", + world.cli_output.as_deref().unwrap_or(""), + world.cli_stderr.as_deref().unwrap_or("") + ); + assert!(rc != 0, "expected refusal, got rc=0:\n{text}"); + assert!( + text.contains("--wait") && text.contains("--verbose"), + "expected both flags named, got:\n{text}" + ); +} + /// The discriminating assertion for the fabricated-launch daemon suppression. /// /// `simulate_oom_managed_launch` writes a `starting` record for a process that