From a789a8a9d4a79dc7110f1fec679b49cd8c9aa914 Mon Sep 17 00:00:00 2001 From: juhovainio Date: Tue, 3 Nov 2026 16:39:55 +0200 Subject: [PATCH 01/12] ROCMAI-791: add rocm logs --follow to watch a starting server's output `rocm logs --service --follow` (and `rocm services logs --follow`) prints the recent output, then keeps streaming new output, including in-place `\r` progress bars, until the server stops. Ctrl-C ends the follow and leaves the server running. Signed-off-by: juhovainio --- README.md | 7 +- apps/rocm/src/main.rs | 18 +- apps/rocm/src/services_cmd.rs | 14 +- apps/rocm/src/services_text.rs | 307 ++++++++++++++++++++- docs/manual-testing.md | 1 + docs/testing.md | 1 + tests/e2e-cucumber/features/logs.feature | 18 ++ tests/e2e-cucumber/tests/e2e/logs_steps.rs | 77 ++++++ 8 files changed, 434 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 1147f3ed41..51742a0163 100644 --- a/README.md +++ b/README.md @@ -1288,13 +1288,18 @@ provider settings are left untouched. ### Logs and cleanup ``` -rocm logs [--service ] [--search TERM ...] [QUERY ...] +rocm logs [--service [--follow]] [--search TERM ...] [QUERY ...] +rocm services logs [--follow] rocm uninstall [--yes] [--dry-run] [--keep-binaries] [--keep-config] [--keep-data] [--keep-cache] [--force-dev-binaries] ``` +`--follow` (`-f`) keeps printing a server's new log output, such as a model +download's progress, until the server stops. Ctrl-C ends the follow and leaves +the server running. + `rocm uninstall` removes four categories of files that ROCm CLI manages on this computer: its binaries, configuration, data, and caches. It lists what it will do and asks `[y/N]` (default: no) before doing it. Use `--dry-run` to only list, diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 14418f0935..df8e39b74a 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -148,7 +148,7 @@ use crate::service_records::DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS; use crate::services_cmd::run_approved_service_action; use crate::services_cmd::services; use crate::services_text::{ - render_internal_status_text, render_logs_browser_text, render_logs_text, + follow_service_logs, render_internal_status_text, render_logs_browser_text, render_logs_text, render_service_logs_text, run_internal_sandbox_tool, }; use crate::setup_cmd::{render_setup_status_text, setup}; @@ -682,11 +682,16 @@ rocm serve qwen --verbose --device gpu_required")] #[command(after_help = "EXAMPLES:\n \ rocm logs\n \ rocm logs --service \n \ +rocm logs --service --follow\n \ rocm logs --search error timeout")] Logs { /// Show logs for a managed service id. #[arg(long)] service: Option, + /// Keep printing new output for --service until the server stops (Ctrl-C ends the + /// follow and leaves the server running). Use it to watch a model download. + #[arg(long, short = 'f', requires = "service")] + follow: bool, /// Search recent logs for one or more words. #[arg(long, num_args = 1.., value_name = "QUERY")] search: Vec, @@ -1170,6 +1175,10 @@ enum ServicesCommand { Logs { /// Service id from `rocm services list`. service_id: String, + /// Keep printing new output until the server stops (Ctrl-C ends the follow and + /// leaves the server running). Use it to watch a model download. + #[arg(long, short = 'f')] + follow: bool, }, /// Stop a local model server. Stop { @@ -2162,6 +2171,7 @@ fn dispatch(cli: Cli) -> Result<()> { Some(Command::Config { command }) => config(command), Some(Command::Logs { service, + follow, search, query, }) => { @@ -2177,7 +2187,11 @@ fn dispatch(cli: Cli) -> Result<()> { ); } if let Some(service_id) = service { - print!("{}", render_service_logs_text(&paths, &service_id)?); + if follow { + follow_service_logs(&paths, &service_id)?; + } else { + print!("{}", render_service_logs_text(&paths, &service_id)?); + } } else { let query = if search.is_empty() { (!query.is_empty()).then(|| query.join(" ")) diff --git a/apps/rocm/src/services_cmd.rs b/apps/rocm/src/services_cmd.rs index 1481cd6c2a..5979edd6ae 100644 --- a/apps/rocm/src/services_cmd.rs +++ b/apps/rocm/src/services_cmd.rs @@ -36,8 +36,8 @@ use crate::service_records::{ prune_managed_service_records, remove_managed_service_record, service_prune_min_age_hours, }; use crate::services_text::{ - render_service_action_result_with_helper_note, render_service_logs_text, render_services_json, - render_services_text, run_internal_sandbox_tool, validate_service_id, + follow_service_logs, render_service_action_result_with_helper_note, render_service_logs_text, + render_services_json, render_services_text, run_internal_sandbox_tool, validate_service_id, }; pub(crate) fn services(command: Option) -> Result<()> { @@ -54,9 +54,13 @@ pub(crate) fn services(command: Option) -> Result<()> { } Ok(()) } - ServicesCommand::Logs { service_id } => { - print!("{}", render_service_logs_text(&paths, &service_id)?); - Ok(()) + ServicesCommand::Logs { service_id, follow } => { + if follow { + follow_service_logs(&paths, &service_id) + } else { + print!("{}", render_service_logs_text(&paths, &service_id)?); + Ok(()) + } } ServicesCommand::Stop { service_id, yes } => { run_approved_service_action(&paths, "stop_server", &service_id, yes) diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index 21284cb006..4fe435bb99 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -88,7 +88,7 @@ use std::collections::VecDeque; use std::fmt::Write as _; use std::fs; -use std::io::{self, BufRead}; +use std::io::{self, BufRead, Read as _, Seek as _, SeekFrom}; use std::path::{Path, PathBuf}; #[cfg(not(windows))] use std::process::Stdio; @@ -550,6 +550,165 @@ pub(crate) fn render_service_action_result_with_helper_note( text } +/// How often `--follow` looks for new log output. +const LOG_FOLLOW_POLL: Duration = Duration::from_millis(500); +/// How often `--follow` re-reads the service record to see whether it stopped. +const LOG_FOLLOW_LIVENESS_EVERY: Duration = Duration::from_secs(2); + +/// Reads whatever a growing log file gained since the last [`Self::poll`]. +/// +/// Download bars (tqdm, huggingface_hub) redraw in place with `\r`, so a +/// finished segment ends at either `\r` or `\n`; a trailing partial segment is +/// held back until its terminator arrives. +struct LogTail { + path: PathBuf, + offset: u64, + pending: Vec, +} + +/// One completed piece of log output and whether it ended in a carriage return. +#[derive(Debug, PartialEq, Eq)] +struct LogSegment { + text: String, + redraw: bool, +} + +impl LogTail { + /// Start following at the file's current end. + fn at_end(path: &Path) -> Self { + let offset = fs::metadata(path).map_or(0, |meta| meta.len()); + Self { + path: path.to_path_buf(), + offset, + pending: Vec::new(), + } + } + + fn poll(&mut self) -> Vec { + let Ok(mut file) = fs::File::open(&self.path) else { + return Vec::new(); + }; + let len = file.metadata().map_or(0, |meta| meta.len()); + if len < self.offset { + // Truncated or replaced (a restart reuses the path): start over. + self.offset = 0; + self.pending.clear(); + } + if file.seek(SeekFrom::Start(self.offset)).is_err() { + return Vec::new(); + } + let mut fresh = Vec::new(); + if file.read_to_end(&mut fresh).is_err() { + return Vec::new(); + } + self.offset += fresh.len() as u64; + self.pending.extend_from_slice(&fresh); + + let mut segments = Vec::new(); + let mut start = 0; + let mut index = 0; + while index < self.pending.len() { + let byte = self.pending[index]; + if byte == b'\n' || byte == b'\r' { + let mut redraw = byte == b'\r'; + let text = String::from_utf8_lossy(&self.pending[start..index]).into_owned(); + // `\r\n` is one line break, not a redraw followed by an empty line. + if redraw && self.pending.get(index + 1) == Some(&b'\n') { + redraw = false; + index += 1; + } + segments.push(LogSegment { text, redraw }); + start = index + 1; + } + index += 1; + } + self.pending.drain(..start); + segments + } +} + +fn write_log_segments(out: &mut impl io::Write, segments: &[LogSegment], tty: bool) -> Result<()> { + for segment in segments { + match (tty, segment.redraw) { + // Redraw the same terminal row, as the engine's own bar does. + (true, true) => write!(out, "\r\x1b[2K{}", segment.text)?, + (true, false) => writeln!(out, "\r\x1b[2K{}", segment.text)?, + (false, _) => writeln!(out, "{}", segment.text)?, + } + } + out.flush()?; + Ok(()) +} + +/// `rocm logs --service --follow`: print the recent output, then keep +/// printing new output until the service is no longer starting or running. +/// Ctrl-C ends the follow and leaves the service running. +pub(crate) fn follow_service_logs(paths: &AppPaths, service_id: &str) -> Result<()> { + use std::io::IsTerminal as _; + let mut out = io::stdout(); + let tty = out.is_terminal(); + follow_service_logs_with( + paths, + service_id, + &mut out, + tty, + LOG_FOLLOW_POLL, + LOG_FOLLOW_LIVENESS_EVERY, + ) +} + +fn follow_service_logs_with( + paths: &AppPaths, + service_id: &str, + out: &mut impl io::Write, + tty: bool, + poll: Duration, + liveness_every: Duration, +) -> Result<()> { + let record = load_managed_service(paths, service_id)?; + // Open the tail before the snapshot so output written in between is not lost. + let mut tail = LogTail::at_end(&record.log_path); + write!(out, "{}", render_service_logs_text(paths, service_id)?)?; + if !managed_service_is_live(&record) { + writeln!( + out, + "\nService {service_id} is {}; not following.", + record.status + )?; + return Ok(()); + } + writeln!( + out, + "\nFollowing {service_id} - Ctrl-C stops following and leaves the server running." + )?; + out.flush()?; + + let mut since_check = Duration::ZERO; + loop { + thread::sleep(poll); + write_log_segments(out, &tail.poll(), tty)?; + since_check += poll; + if since_check < liveness_every { + continue; + } + since_check = Duration::ZERO; + let record = load_managed_service(paths, service_id)?; + if !managed_service_is_live(&record) { + // Output written between the last poll and the exit. + write_log_segments(out, &tail.poll(), tty)?; + if tty { + writeln!(out)?; + } + writeln!( + out, + "Service {service_id} is {}; stopped following.", + record.status + )?; + return Ok(()); + } + } +} + pub(crate) fn render_service_action_result(tool: &str, value: &serde_json::Value) -> String { let output = value.get("output").unwrap_or(value); let action = service_action_past_tense(tool); @@ -1659,6 +1818,152 @@ mod tests { Ok(()) } + #[test] + fn log_tail_yields_only_new_complete_segments() -> Result<()> { + let (root, paths) = test_paths("log-tail"); + paths.ensure()?; + let log = root.join("tail.log"); + fs::write(&log, "old line\n")?; + let mut tail = LogTail::at_end(&log); + assert!(tail.poll().is_empty()); + + // tqdm-style bar redraws with `\r`; the last piece has no terminator yet. + fs::write(&log, "old line\n 28%|##\r 33%|###\rdone\r\nhalf")?; + let segments = tail.poll(); + assert_eq!( + segments, + vec![ + LogSegment { + text: " 28%|##".into(), + redraw: true + }, + LogSegment { + text: " 33%|###".into(), + redraw: true + }, + LogSegment { + text: "done".into(), + redraw: false + }, + ] + ); + // The held-back partial segment completes on the next poll. + let mut file = fs::OpenOptions::new().append(true).open(&log)?; + std::io::Write::write_all(&mut file, b" line\n")?; + assert_eq!( + tail.poll(), + vec![LogSegment { + text: "half line".into(), + redraw: false + }] + ); + + // A truncated log (restart reusing the path) is read from the start. + fs::write(&log, "new\n")?; + assert_eq!( + tail.poll(), + vec![LogSegment { + text: "new".into(), + redraw: false + }] + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn follow_service_logs_prints_snapshot_and_returns_for_a_stopped_service() -> Result<()> { + let (root, paths) = test_paths("follow-stopped"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "svc_follow_stopped", + "vllm", + "qwen3.5", + "Qwen/Qwen3.5", + "127.0.0.1", + 11436, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write()?; + fs::write(&record.log_path, "Downloading 86%\n")?; + + let mut out = Vec::new(); + follow_service_logs_with( + &paths, + "svc_follow_stopped", + &mut out, + false, + Duration::from_millis(1), + Duration::from_millis(1), + )?; + let out = String::from_utf8(out)?; + assert!(out.contains("Downloading 86%"), "{out}"); + assert!( + out.contains("Service svc_follow_stopped is stopped; not following."), + "{out}" + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn follow_service_logs_streams_new_output_until_the_service_stops() -> Result<()> { + let (root, paths) = test_paths("follow-live"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "svc_follow_live", + "vllm", + "qwen3.5", + "Qwen/Qwen3.5", + "127.0.0.1", + 11437, + "managed", + std::process::id(), + None, + None, + None, + ); + record.status = "starting".to_owned(); + record.write()?; + fs::write(&record.log_path, "before\n")?; + + let log_path = record.log_path; + let manifest_paths = paths.clone(); + let writer = thread::spawn(move || -> Result<()> { + thread::sleep(Duration::from_millis(150)); + let mut file = fs::OpenOptions::new().append(true).open(&log_path)?; + std::io::Write::write_all(&mut file, b"Downloading 43%\n")?; + thread::sleep(Duration::from_millis(150)); + let mut record = load_managed_service(&manifest_paths, "svc_follow_live")?; + record.status = "stopped".to_owned(); + record.write()?; + Ok(()) + }); + let mut out = Vec::new(); + follow_service_logs_with( + &paths, + "svc_follow_live", + &mut out, + false, + Duration::from_millis(20), + Duration::from_millis(40), + )?; + writer.join().expect("writer thread")?; + let out = String::from_utf8(out)?; + assert!(out.contains("Following svc_follow_live"), "{out}"); + assert!(out.contains("Downloading 43%"), "{out}"); + assert!(out.contains("stopped following"), "{out}"); + let _ = fs::remove_dir_all(root); + Ok(()) + } + #[test] fn render_service_logs_text_tails_manifest_log() -> Result<()> { let (root, paths) = test_paths("service-logs"); diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 1185621dbe..1fde09a5a1 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -364,6 +364,7 @@ After a managed or foreground serve attempt, inspect local server records: rocm services rocm services list --all rocm services logs +rocm services logs --follow ``` Expected result: diff --git a/docs/testing.md b/docs/testing.md index 6f7ac7ae80..629e89b0bc 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1006,6 +1006,7 @@ Log browser smoke: rocm logs rocm logs --search server rocm logs --service +rocm logs --service --follow # Ctrl-C to stop following ``` The no-log first-run path should report `no CLI logs found yet` without diff --git a/tests/e2e-cucumber/features/logs.feature b/tests/e2e-cucumber/features/logs.feature index ddd97f0248..51c5314ccd 100644 --- a/tests/e2e-cucumber/features/logs.feature +++ b/tests/e2e-cucumber/features/logs.feature @@ -24,3 +24,21 @@ Feature: Log inspection Given recorded command logs containing several lines about a topic When the user asks for one service's logs and a search term together Then the CLI refuses and explains only one may be used + + # `--follow` keeps printing new output while a server starts (a model download + # can take many minutes). It ends by itself once the server is no longer + # starting or running, so the scenario plants a stopped record: the snapshot + # is printed, the CLI says it is not following, and the command returns + # instead of waiting. Streaming new output from a live service is covered by + # `follow_service_logs_streams_new_output_until_the_service_stops` in + # `apps/rocm/src/services_text.rs`. + @id:logs-follow-stopped-service-returns + Scenario: logs-04 - Following a stopped server's logs prints them and returns + Given a stopped local server whose log shows a download at 43 percent + When the user follows that server's logs + Then the CLI prints the download line, says it is not following, and exits + + @id:logs-follow-requires-service + Scenario: logs-05 - Following without naming a server is refused + When the user asks to follow logs without naming a service + Then the CLI refuses because --follow needs --service diff --git a/tests/e2e-cucumber/tests/e2e/logs_steps.rs b/tests/e2e-cucumber/tests/e2e/logs_steps.rs index ab0986d0de..6ab899f315 100644 --- a/tests/e2e-cucumber/tests/e2e/logs_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/logs_steps.rs @@ -123,3 +123,80 @@ fn ok_output(world: &E2eWorld) -> String { assert_eq!(rc, 0, "expected success, got rc={rc}:\n{}", combined(world)); world.cli_output.clone().unwrap_or_default() } + +// ── --follow ─────────────────────────────────────────────────────── + +const FOLLOW_SERVICE_ID: &str = "vllm-e2e-follow"; + +#[given("a stopped local server whose log shows a download at 43 percent")] +async fn plant_stopped_server_with_log(world: &mut E2eWorld) { + let root = world.isolated_root.as_ref().expect("no isolated root"); + let data = root.path().join("data"); + let services = data.join("services"); + let states = data.join("engines").join("vllm").join("state"); + std::fs::create_dir_all(&services).expect("failed to create services dir"); + std::fs::create_dir_all(&states).expect("failed to create engine state dir"); + let log_path = services.join(format!("{FOLLOW_SERVICE_ID}.log")); + std::fs::write(&log_path, "Downloading model.safetensors: 43%\n").expect("failed to write log"); + let record = serde_json::json!({ + "service_id": FOLLOW_SERVICE_ID, + "engine": "vllm", + "model_ref": "qwen", + "canonical_model_id": "Qwen/Qwen3.5", + "host": "127.0.0.1", + "port": 9, + "endpoint_url": "http://127.0.0.1:9/v1", + "mode": "managed", + "status": "stopped", + "supervisor_pid": 0, + "engine_pid": null, + "manifest_path": services.join(format!("{FOLLOW_SERVICE_ID}.json")), + "log_path": log_path, + "engine_state_path": states.join(format!("{FOLLOW_SERVICE_ID}.json")), + "created_at_unix_ms": 1_700_000_000_000_u64, + }); + std::fs::write( + services.join(format!("{FOLLOW_SERVICE_ID}.json")), + serde_json::to_vec_pretty(&record).expect("failed to serialize record"), + ) + .expect("failed to write service record"); +} + +#[when("the user follows that server's logs")] +async fn follow_stopped_server(world: &mut E2eWorld) { + let (stdout, stderr, rc) = + crate::run_rocm(world, &["logs", "--service", FOLLOW_SERVICE_ID, "--follow"]); + record(world, stdout, stderr, rc); +} + +#[when("the user asks to follow logs without naming a service")] +async fn follow_without_service(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["logs", "--follow"]); + record(world, stdout, stderr, rc); +} + +#[then("the CLI prints the download line, says it is not following, and exits")] +async fn prints_and_exits(world: &mut E2eWorld) { + let out = ok_output(world); + assert!( + out.contains("Downloading model.safetensors: 43%"), + "expected the planted log line, got:\n{out}" + ); + assert!( + out.contains(&format!( + "Service {FOLLOW_SERVICE_ID} is stopped; not following." + )), + "expected the not-following line, got:\n{out}" + ); +} + +#[then("the CLI refuses because --follow needs --service")] +async fn refuses_follow(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no command rc recorded"); + assert!(rc != 0, "expected refusal, got rc=0:\n{}", combined(world)); + assert!( + combined(world).contains("--service"), + "expected the error to name --service, got:\n{}", + combined(world) + ); +} From 82d482ae91d846676cc13427f78243402917c761 Mon Sep 17 00:00:00 2001 From: juhovainio Date: Tue, 3 Nov 2026 17:17:44 +0200 Subject: [PATCH 02/12] ROCMAI-791: add rocm serve --wait to wait out a first-time model download --wait keeps a managed serve waiting past the 45 second readiness budget until the server answers or exits. It conflicts with --verbose and --foreground. Co-Authored-By: Claude Sonnet 5.5 Signed-off-by: juhovainio --- README.md | 7 +- apps/rocm/src/main.rs | 24 ++++ apps/rocm/src/serve_cmd.rs | 3 + apps/rocm/src/service_launch.rs | 103 +++++++++++++++--- docs/manual-testing.md | 1 + docs/testing.md | 7 +- .../features/model_serving.feature | 11 +- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 23 ++++ 8 files changed, 163 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 51742a0163..b00e519e32 100644 --- a/README.md +++ b/README.md @@ -842,7 +842,7 @@ rocm serve [--engine lemonade|vllm|aim] [--gpu auto|] [--runtime-id KEY | --env-id ID] [--host HOST] [--port PORT] - [--verbose] [--foreground | --managed] + [--verbose] [--foreground | --managed | --wait] [--no-smoke-test] [--allow-public-bind] [--require-api-key] [--api-key KEY] [--temperature TEMP] [--top-p PROB] [--max-tokens N] @@ -908,6 +908,11 @@ can: - Press **Ctrl-C** to stop the server instead. `--managed` is the explicit form of the default background behavior. +`--wait` keeps `rocm serve` waiting until the server answers requests, however +long a first-time model download takes, instead of returning `starting` after 45 +seconds; it returns early, reporting the last status, if the server exits. It +cannot be combined with `--foreground` or `--verbose`. Watch the download with +`rocm logs --service --follow` from another terminal. `--no-smoke-test` skips the post-startup inference probe. Which model form to pass depends on the engine your GPU selects. The Lemonade diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index df8e39b74a..a20377ea27 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -583,6 +583,12 @@ rocm serve qwen --verbose --device gpu_required")] /// Keep the server managed by ROCm CLI. #[arg(long)] managed: bool, + /// Keep waiting until the server answers requests, however long a first-time + /// model download takes, instead of returning `starting` after 45 seconds. + /// Returns early if the server exits. Cannot be combined with --foreground + /// or --verbose, which already stay attached. + #[arg(long, conflicts_with_all = ["foreground", "verbose"])] + wait: bool, /// Stream every engine log line in this terminal instead of a deployment summary. #[arg(long, conflicts_with = "managed")] verbose: bool, @@ -2130,6 +2136,7 @@ fn dispatch(cli: Cli) -> Result<()> { port, foreground, managed, + wait, verbose, no_smoke_test, allow_public_bind, @@ -2152,6 +2159,7 @@ fn dispatch(cli: Cli) -> Result<()> { port, foreground, managed, + wait, verbose, no_smoke_test, allow_public_bind, @@ -3530,6 +3538,22 @@ pub(crate) mod tests { Cli::try_parse_from(argv) } + #[test] + fn serve_wait_parses_and_conflicts_with_attached_modes() { + let cli = parse_serve(&["--wait"]).expect("--wait parses"); + assert!(matches!( + cli.command, + Some(Command::Serve { wait: true, .. }) + )); + let cli = parse_serve(&[]).expect("default parses"); + assert!(matches!( + cli.command, + Some(Command::Serve { wait: false, .. }) + )); + assert!(parse_serve(&["--wait", "--verbose"]).is_err()); + assert!(parse_serve(&["--wait", "--foreground"]).is_err()); + } + #[test] fn serve_parses_verbose_and_no_smoke_test_flags() { let cli = parse_serve(&["--verbose", "--no-smoke-test"]).expect("flags parse"); diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index e163dd6dd4..b63418a9df 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -880,6 +880,7 @@ pub(crate) struct ServeArgs { pub(crate) port: u16, pub(crate) foreground: bool, pub(crate) managed: bool, + pub(crate) wait: bool, pub(crate) verbose: bool, pub(crate) no_smoke_test: bool, pub(crate) allow_public_bind: bool, @@ -905,6 +906,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { port, foreground, managed, + wait, verbose, no_smoke_test, allow_public_bind, @@ -1500,6 +1502,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { launch_lock, require_api_key, gpu_refusal_waived_for_reuse, + wait, &mut |_elapsed| spinner.tick(), )?; ensure_background_helper_running_quiet(summary_mode)?; diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 57d405e5d2..e4ed58b218 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -672,6 +672,25 @@ fn spawn_managed_engine_child( }) } +/// How long one readiness wait polls before `rocm serve` reports `starting`. +const READINESS_WAIT: Duration = Duration::from_secs(45); + +/// `rocm serve --wait`: keep waiting past [`READINESS_WAIT`] (a first start +/// downloads the model, which can take far longer) until the server answers or +/// stops being live. A server that exited ends the wait with the last reading +/// instead of blocking forever. +fn wait_past_readiness_budget( + mut probe: impl FnMut() -> EndpointReadiness, + mut still_live: impl FnMut() -> bool, +) -> EndpointReadiness { + loop { + let readiness = probe(); + if readiness == EndpointReadiness::Serving || !still_live() { + return readiness; + } + } +} + #[allow(clippy::too_many_arguments)] pub(crate) fn start_managed_service( engine: &str, @@ -689,6 +708,7 @@ pub(crate) fn start_managed_service( launch_lock: rocm_core::FileLock, require_api_key: bool, gpu_refusal_waived_for_reuse: bool, + wait_until_ready: bool, on_wait_tick: &mut dyn FnMut(Duration), ) -> Result { let paths = AppPaths::discover()?; @@ -720,19 +740,33 @@ pub(crate) fn start_managed_service( #[cfg(windows)] thread::sleep(Duration::from_millis(200)); - let readiness = wait_for_service_http_ready_with_progress( - engine, - host, - port, - &|| { - let mut latest = record.clone(); - let _ = latest.refresh_from_engine_state(); - latest.served_model_name().to_owned() - }, - endpoint_api_key, - Duration::from_secs(45), - on_wait_tick, - ); + let probe = |on_tick: &mut dyn FnMut(Duration)| { + wait_for_service_http_ready_with_progress( + engine, + host, + port, + &|| { + let mut latest = record.clone(); + let _ = latest.refresh_from_engine_state(); + latest.served_model_name().to_owned() + }, + endpoint_api_key, + READINESS_WAIT, + on_tick, + ) + }; + let readiness = if wait_until_ready { + wait_past_readiness_budget( + || probe(&mut *on_wait_tick), + || { + let mut latest = record.clone(); + let _ = latest.refresh_from_engine_state(); + crate::managed_service_status::managed_service_is_live(&latest) + }, + ) + } else { + probe(on_wait_tick) + }; let launch_status = status_for_readiness(readiness); record.status = launch_status.to_owned(); if readiness == EndpointReadiness::Serving { @@ -1418,6 +1452,49 @@ mod tests { use super::*; use crate::tests::test_paths; + #[test] + fn wait_past_readiness_budget_keeps_probing_a_live_starting_server() { + let mut readings = vec![ + EndpointReadiness::Unreachable, + EndpointReadiness::Unreachable, + EndpointReadiness::Serving, + ] + .into_iter(); + let mut probes = 0; + let result = wait_past_readiness_budget( + || { + probes += 1; + readings.next().expect("no probe after Serving") + }, + || true, + ); + assert_eq!(result, EndpointReadiness::Serving); + assert_eq!(probes, 3, "must outlast the first (45s) budget"); + } + + #[test] + fn wait_past_readiness_budget_stops_when_the_server_exits() { + let mut probes = 0; + let result = wait_past_readiness_budget( + || { + probes += 1; + EndpointReadiness::Unreachable + }, + || false, + ); + assert_eq!(result, EndpointReadiness::Unreachable); + assert_eq!(probes, 1); + } + + #[test] + fn wait_past_readiness_budget_returns_at_once_when_already_serving() { + let result = wait_past_readiness_budget( + || EndpointReadiness::Serving, + || panic!("liveness is irrelevant once serving"), + ); + assert_eq!(result, EndpointReadiness::Serving); + } + #[test] fn serve_bind_validation_requires_public_ack() { validate_bind_host("127.0.0.1", false).unwrap(); diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 1fde09a5a1..f66af87d4f 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -365,6 +365,7 @@ rocm services rocm services list --all rocm services logs rocm services logs --follow +rocm serve --wait # from another terminal: returns once the server answers ``` Expected result: diff --git a/docs/testing.md b/docs/testing.md index 629e89b0bc..62f46cb9ca 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1009,6 +1009,11 @@ rocm logs --service rocm logs --service --follow # Ctrl-C to stop following ``` +While a first-time model download is running, `rocm serve --wait` keeps +waiting past the usual 45 seconds until the server answers (or exits), and +`--wait` with `--verbose` or `--foreground` is refused (`serve-41`). Run the +log follow above from a second terminal to watch the download. + The no-log first-run path should report `no CLI logs found yet` without creating the configured `data/logs` directory. @@ -1606,7 +1611,7 @@ all four combinations, `stop_output_carries_the_helper_note_...` and bad config from failing a `serve` after its launch. The user-visible side is `automations-11` in [automations.feature](../tests/e2e-cucumber/features/automations.feature) (mock lane: enable starts a live helper, disabling the last watcher makes it -exit) and `serve-40` in +exit) and `serve-41` in [model_serving.feature](../tests/e2e-cucumber/features/model_serving.feature) (GPU lane: serving with no watcher enabled starts no helper). diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index d32cf52156..b8a6001274 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -628,7 +628,6 @@ Feature: Model serving And the CLI says the image needs eight GPUs and multi-GPU serving is not supported And Docker was never called - # ROCMAI-827: `rocm serve` started the background automation helper # (`rocm daemon`) on every launch, even with every automation watcher off — # when it has nothing to do — and that helper outlived the service. With no @@ -640,3 +639,13 @@ Feature: Model serving Given a managed runtime is active When the user serves a GGUF model on lemonade with no automation enabled Then serving started no background helper + + # `--wait` makes a managed serve wait for readiness past the usual 45 seconds, + # so it cannot be combined with the attached modes, which already stay on the + # terminal. The waiting itself (a live server that becomes ready, one that + # exits first) is covered by `wait_past_readiness_budget_*` in + # `apps/rocm/src/service_launch.rs`, since a scenario would need a slow engine. + @id:serve-wait-conflicts-with-attached-modes + Scenario: serve-41 - Waiting for readiness cannot be combined with attached mode + When the user serves a model with both --wait and --verbose + Then the CLI refuses and names the conflicting flags diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index d5d40e0c04..9c278e4a14 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -2599,3 +2599,26 @@ async fn assert_stop_hint_clears_the_service(world: &mut E2eWorld) { "the on-disk record must persist the stop, got:\n{record}" ); } + +#[when("the user serves a model with both --wait and --verbose")] +async fn serve_wait_and_verbose(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm(world, &["serve", "qwen", "--wait", "--verbose"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("the CLI refuses and names the conflicting flags")] +async fn serve_wait_refused(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no command rc recorded"); + let text = format!( + "{}\n{}", + world.cli_output.as_deref().unwrap_or(""), + world.cli_stderr.as_deref().unwrap_or("") + ); + assert!(rc != 0, "expected refusal, got rc=0:\n{text}"); + assert!( + text.contains("--wait") && text.contains("--verbose"), + "expected both flags named, got:\n{text}" + ); +} From 2f6d0b505a4086392ad4cce996e27f3edc8aa923 Mon Sep 17 00:00:00 2001 From: juhovainio Date: Wed, 4 Nov 2026 13:20:00 +0200 Subject: [PATCH 03/12] ROCMAI-791: end serve --wait when the server dies; harden --follow output serve --wait now checks that a tracked process is alive (not a zombie) and re-reads the manifest, so a killed engine or a services stop ends the wait with an error pointing at the logs instead of waiting forever. logs --follow holds back a trailing CR, flushes unterminated output on exit, and gains tests for the tty redraw arms and services logs -f (logs-06). Docs list the new items; README synopsis matches the clap conflicts. Signed-off-by: juhovainio --- README.md | 8 +- apps/rocm/src/main.rs | 4 +- apps/rocm/src/service_launch.rs | 161 ++++++++++++++++++++- apps/rocm/src/services_text.rs | 109 ++++++++++++-- docs/architecture.md | 2 +- docs/manual-testing.md | 8 + docs/testing.md | 3 +- tests/e2e-cucumber/features/logs.feature | 8 + tests/e2e-cucumber/tests/e2e/logs_steps.rs | 7 + 9 files changed, 289 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index b00e519e32..bad213a0b8 100644 --- a/README.md +++ b/README.md @@ -842,7 +842,7 @@ rocm serve [--engine lemonade|vllm|aim] [--gpu auto|] [--runtime-id KEY | --env-id ID] [--host HOST] [--port PORT] - [--verbose] [--foreground | --managed | --wait] + [--verbose] [--foreground | --managed] [--wait] [--no-smoke-test] [--allow-public-bind] [--require-api-key] [--api-key KEY] [--temperature TEMP] [--top-p PROB] [--max-tokens N] @@ -910,8 +910,10 @@ can: `--managed` is the explicit form of the default background behavior. `--wait` keeps `rocm serve` waiting until the server answers requests, however long a first-time model download takes, instead of returning `starting` after 45 -seconds; it returns early, reporting the last status, if the server exits. It -cannot be combined with `--foreground` or `--verbose`. Watch the download with +seconds; if the server exits first (crashes, is killed, or is stopped with +`rocm services stop`), it fails with an error that points at the server's logs +instead of waiting forever. Ctrl-C ends the wait and leaves the server starting +in the background. It cannot be combined with `--foreground` or `--verbose`. Watch the download with `rocm logs --service --follow` from another terminal. `--no-smoke-test` skips the post-startup inference probe. diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index a20377ea27..7da27292fe 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -585,8 +585,8 @@ rocm serve qwen --verbose --device gpu_required")] managed: bool, /// Keep waiting until the server answers requests, however long a first-time /// model download takes, instead of returning `starting` after 45 seconds. - /// Returns early if the server exits. Cannot be combined with --foreground - /// or --verbose, which already stay attached. + /// Fails with a pointer to the logs if the server exits first. Cannot be + /// combined with --foreground or --verbose, which already stay attached. #[arg(long, conflicts_with_all = ["foreground", "verbose"])] wait: bool, /// Stream every engine log line in this terminal instead of a deployment summary. diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index e4ed58b218..e4329810d8 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -691,6 +691,62 @@ fn wait_past_readiness_budget( } } +/// Whether `pid` is a running process that is not a zombie. The serve +/// supervisor is an unreaped child of this process, so after it dies +/// `process_is_running` alone keeps reporting it alive. +fn service_pid_is_alive(pid: u32) -> bool { + rocm_core::process_is_running(pid) && !process_is_zombie(pid) +} + +#[cfg(target_os = "linux")] +fn process_is_zombie(pid: u32) -> bool { + let Ok(stat) = fs::read_to_string(format!("/proc/{pid}/stat")) else { + return false; + }; + // `comm` may contain spaces and parentheses; the state follows the last `)`. + stat.rfind(')') + .and_then(|end| stat.get(end + 1..)) + .and_then(|rest| rest.split_whitespace().next()) + == Some("Z") +} + +#[cfg(not(target_os = "linux"))] +fn process_is_zombie(_pid: u32) -> bool { + false +} + +/// Whether the record still describes a server that can finish starting: its +/// status is a live one *and* a tracked process is alive. The status alone is +/// not enough, since a killed engine never writes a terminal state. +fn record_still_live(record: &ManagedServiceRecord, pid_alive: impl Fn(u32) -> bool) -> bool { + if !crate::managed_service_status::managed_service_is_live(record) { + return false; + } + let pids = crate::managed_service_status::recorded_service_pids(record); + pids.is_empty() || pids.into_iter().any(pid_alive) +} + +/// [`record_still_live`] over the manifest on disk, so a `rocm services stop` +/// from another terminal (which writes only the manifest) ends the wait too. +fn service_still_live(paths: &AppPaths, service_id: &str, pid_alive: impl Fn(u32) -> bool) -> bool { + let Ok(mut latest) = ManagedServiceRecord::read_manifest(paths, service_id) else { + return false; + }; + let _ = latest.refresh_from_engine_state(); + record_still_live(&latest, pid_alive) +} + +/// Record a server that died mid-start as stopped, leaving a manifest that a +/// `services stop` already settled untouched. +fn settle_exited_service(paths: &AppPaths, service_id: &str) { + if let Ok(mut latest) = ManagedServiceRecord::read_manifest(paths, service_id) + && crate::managed_service_status::managed_service_is_live(&latest) + { + latest.status = "stopped".to_owned(); + let _ = latest.write(); + } +} + #[allow(clippy::too_many_arguments)] pub(crate) fn start_managed_service( engine: &str, @@ -758,15 +814,20 @@ pub(crate) fn start_managed_service( let readiness = if wait_until_ready { wait_past_readiness_budget( || probe(&mut *on_wait_tick), - || { - let mut latest = record.clone(); - let _ = latest.refresh_from_engine_state(); - crate::managed_service_status::managed_service_is_live(&latest) - }, + || service_still_live(&paths, service_id, service_pid_is_alive), ) } else { probe(on_wait_tick) }; + if wait_until_ready + && readiness != EndpointReadiness::Serving + && !service_still_live(&paths, service_id, service_pid_is_alive) + { + settle_exited_service(&paths, service_id); + bail!( + "the server exited before it became ready; see `rocm logs --service {service_id}` for its output" + ); + } let launch_status = status_for_readiness(readiness); record.status = launch_status.to_owned(); if readiness == EndpointReadiness::Serving { @@ -1495,6 +1556,96 @@ mod tests { assert_eq!(result, EndpointReadiness::Serving); } + fn record_tracking_pid(paths: &AppPaths, pid: u32, status: &str) -> ManagedServiceRecord { + let mut record = reusable_record(paths); + record.supervisor_pid = pid; + record.status = status.to_owned(); + record + } + + #[cfg(unix)] + fn exited_child_pid() -> u32 { + let mut child = std::process::Command::new("true").spawn().expect("spawn"); + let pid = child.id(); + child.wait().expect("reap"); + pid + } + + #[test] + #[cfg(unix)] + fn record_still_live_ends_the_wait_for_a_dead_pid_with_a_live_status() { + // The engine was killed before writing any terminal state: the status + // still says `starting`, but nothing tracked is running. + let (root, paths) = test_paths("wait-dead-pid"); + paths.ensure().expect("paths"); + let dead = record_tracking_pid(&paths, exited_child_pid(), "starting"); + assert!(!record_still_live(&dead, service_pid_is_alive)); + + let alive = record_tracking_pid(&paths, std::process::id(), "starting"); + assert!(record_still_live(&alive, service_pid_is_alive)); + + // A settled status ends the wait even while the pid is still around. + let stopped = record_tracking_pid(&paths, std::process::id(), "stopped"); + assert!(!record_still_live(&stopped, service_pid_is_alive)); + let _ = fs::remove_dir_all(root); + } + + #[test] + #[cfg(target_os = "linux")] + fn service_pid_is_alive_treats_an_unreaped_child_as_dead() { + // The serve supervisor is our own unreaped child; once it dies it is a + // zombie that `kill(pid, 0)` still reports as running. + let mut child = std::process::Command::new("true").spawn().expect("spawn"); + let pid = child.id(); + let deadline = std::time::Instant::now() + Duration::from_secs(10); + while !process_is_zombie(pid) { + assert!(std::time::Instant::now() < deadline, "child never exited"); + thread::sleep(Duration::from_millis(10)); + } + assert!( + process_is_running(pid), + "a zombie still answers kill(pid, 0)" + ); + assert!(!service_pid_is_alive(pid)); + child.wait().expect("reap"); + } + + #[test] + fn service_still_live_follows_the_manifest_on_disk() { + // `rocm services stop` from another terminal writes only the manifest. + let (root, paths) = test_paths("wait-manifest"); + paths.ensure().expect("paths"); + let mut record = record_tracking_pid(&paths, std::process::id(), "starting"); + record.write().expect("write"); + assert!(service_still_live(&paths, &record.service_id, |_| true)); + + record.status = "stopped".to_owned(); + record.write().expect("write"); + assert!(!service_still_live(&paths, &record.service_id, |_| true)); + assert!(!service_still_live(&paths, "no-such-service", |_| true)); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn settle_exited_service_demotes_a_live_manifest_and_spares_a_settled_one() { + let (root, paths) = test_paths("wait-settle"); + paths.ensure().expect("paths"); + let mut record = record_tracking_pid(&paths, 0, "starting"); + record.write().expect("write"); + settle_exited_service(&paths, &record.service_id); + let after = ManagedServiceRecord::read_manifest(&paths, &record.service_id).unwrap(); + assert_eq!(after.status, "stopped"); + + // A stop that already settled the record keeps its own fields. + record.status = "stopped".to_owned(); + record.stop_requested_unix_ms = Some(7); + record.write().expect("write"); + settle_exited_service(&paths, &record.service_id); + let after = ManagedServiceRecord::read_manifest(&paths, &record.service_id).unwrap(); + assert_eq!(after.stop_requested_unix_ms, Some(7)); + let _ = fs::remove_dir_all(root); + } + #[test] fn serve_bind_validation_requires_public_ack() { validate_bind_host("127.0.0.1", false).unwrap(); diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index 4fe435bb99..4d23538dc7 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -18,7 +18,10 @@ //! status surfaces (`render_services_text`, `render_services_json`, //! `render_service_logs_text`, `render_service_logs_text_with_options`, //! `render_service_action_result`, `read_managed_service_manifest`, -//! `load_managed_service`), the internal `rocmd`-compatible status/sandbox +//! `load_managed_service`), the `--follow` tail behind `rocm logs --service +//! --follow` and `rocm services logs --follow` +//! (`follow_service_logs`, `follow_service_logs_with`, the `LogTail` and +//! `LogSegment` types, `write_log_segments`), the internal `rocmd`-compatible status/sandbox //! surface (`render_internal_status_text`, `run_internal_sandbox_tool`), and //! the managed-service stop/restart implementation behind both the sandbox //! tool and `rocm services stop|restart` (`terminate_recorded_service_pids`, @@ -46,8 +49,9 @@ //! `render_service_logs_text`, `render_logs_browser_text`, and //! `render_logs_text` — re-imported via `use crate::services_text::{...};`. //! `services_cmd.rs`'s `services()` calls `render_services_text`, -//! `render_services_json` and `render_service_logs_text` through its own -//! imports, and its `run_approved_service_action` also reads +//! `render_services_json`, `render_service_logs_text` and +//! `follow_service_logs` through its own imports, and `dispatch()` calls +//! `follow_service_logs` for `rocm logs --service --follow`, and its `run_approved_service_action` also reads //! `render_service_action_result`, `run_internal_sandbox_tool`, and //! `validate_service_id`. `service_records.rs` reads `validate_service_id` //! (in `service_record_artifacts` and `remove_managed_service_record`) and @@ -609,6 +613,12 @@ impl LogTail { let mut index = 0; while index < self.pending.len() { let byte = self.pending[index]; + // A `\r` at the very end of the data may be the first half of a `\r\n` + // that the next poll completes; hold it back rather than emit an + // empty segment (which would erase a final redraw frame on a tty). + if byte == b'\r' && index + 1 == self.pending.len() { + break; + } if byte == b'\n' || byte == b'\r' { let mut redraw = byte == b'\r'; let text = String::from_utf8_lossy(&self.pending[start..index]).into_owned(); @@ -625,6 +635,19 @@ impl LogTail { self.pending.drain(..start); segments } + + /// The unterminated output still held back, for when following ends: a + /// last write cut short by a crash is otherwise never shown. + fn finish(&mut self) -> Option { + let mut bytes = std::mem::take(&mut self.pending); + if bytes.last() == Some(&b'\r') { + bytes.pop(); + } + (!bytes.is_empty()).then(|| LogSegment { + text: String::from_utf8_lossy(&bytes).into_owned(), + redraw: false, + }) + } } fn write_log_segments(out: &mut impl io::Write, segments: &[LogSegment], tty: bool) -> Result<()> { @@ -695,7 +718,9 @@ fn follow_service_logs_with( let record = load_managed_service(paths, service_id)?; if !managed_service_is_live(&record) { // Output written between the last poll and the exit. - write_log_segments(out, &tail.poll(), tty)?; + let mut last = tail.poll(); + last.extend(tail.finish()); + write_log_segments(out, &last, tty)?; if tty { writeln!(out)?; } @@ -1871,6 +1896,67 @@ mod tests { Ok(()) } + #[test] + fn log_tail_holds_back_a_trailing_cr_and_flushes_the_rest_on_finish() -> Result<()> { + let (root, paths) = test_paths("log-tail-edges"); + paths.ensure()?; + let log = root.join("edges.log"); + fs::write(&log, "")?; + let mut tail = LogTail::at_end(&log); + + // A `\r` ending the data may be half of a `\r\n`: it is not a redraw yet. + fs::write(&log, " 99%|#\r")?; + assert!(tail.poll().is_empty()); + let mut file = fs::OpenOptions::new().append(true).open(&log)?; + std::io::Write::write_all(&mut file, b"\n")?; + assert_eq!( + tail.poll(), + vec![LogSegment { + text: " 99%|#".into(), + redraw: false + }], + "the held `\\r` joins the next `\\n` instead of yielding an empty segment" + ); + + // Output cut short before any terminator is shown when following ends. + std::io::Write::write_all(&mut file, b"crashed mid-wri")?; + assert!(tail.poll().is_empty()); + assert_eq!( + tail.finish(), + Some(LogSegment { + text: "crashed mid-wri".into(), + redraw: false + }) + ); + assert_eq!(tail.finish(), None); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn write_log_segments_redraws_in_place_only_on_a_tty() -> Result<()> { + let segments = [ + LogSegment { + text: "28%".into(), + redraw: true, + }, + LogSegment { + text: "done".into(), + redraw: false, + }, + ]; + let mut tty = Vec::new(); + write_log_segments(&mut tty, &segments, true)?; + // A redraw segment leaves the cursor on its row; the next segment (or + // line) erases the row first, and a plain line ends with a newline. + assert_eq!(String::from_utf8(tty)?, "\r\x1b[2K28%\r\x1b[2Kdone\n"); + + let mut piped = Vec::new(); + write_log_segments(&mut piped, &segments, false)?; + assert_eq!(String::from_utf8(piped)?, "28%\ndone\n"); + Ok(()) + } + #[test] fn follow_service_logs_prints_snapshot_and_returns_for_a_stopped_service() -> Result<()> { let (root, paths) = test_paths("follow-stopped"); @@ -1937,14 +2023,19 @@ mod tests { let log_path = record.log_path; let manifest_paths = paths.clone(); let writer = thread::spawn(move || -> Result<()> { - thread::sleep(Duration::from_millis(150)); - let mut file = fs::OpenOptions::new().append(true).open(&log_path)?; - std::io::Write::write_all(&mut file, b"Downloading 43%\n")?; - thread::sleep(Duration::from_millis(150)); + let appended = (|| -> Result<()> { + thread::sleep(Duration::from_millis(150)); + let mut file = fs::OpenOptions::new().append(true).open(&log_path)?; + std::io::Write::write_all(&mut file, b"Downloading 43%\n")?; + thread::sleep(Duration::from_millis(150)); + Ok(()) + })(); + // Stop the service even when the append failed, so the follow loop + // under test always ends instead of hanging the run. let mut record = load_managed_service(&manifest_paths, "svc_follow_live")?; record.status = "stopped".to_owned(); record.write()?; - Ok(()) + appended }); let mut out = Vec::new(); follow_service_logs_with( diff --git a/docs/architecture.md b/docs/architecture.md index 8dc93b544b..c9d65388ce 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -75,7 +75,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`service_launch.rs`](../apps/rocm/src/service_launch.rs) — mechanical relocation, owns types (Phase 6b, ROCMAI-91): spawning/attaching to a managed service process and streaming its log, plus the endpoint-key/auth checks that gate a public bind — `AttachOutcome`/`RawModeGuard`/`ManagedSpawn`/`AutostartClaim` are its private types; `ManagedLaunchReport` is `pub(crate)`, since [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) reads its fields directly for the interactive summary. Most of this module's functions are `pub(crate)` for the same reason — other modules, including [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`automations.rs`](../apps/rocm/src/automations.rs), and [`services_text.rs`](../apps/rocm/src/services_text.rs), call into it; `main.rs` does not. `ServicesCommand` stayed in [`main.rs`](../apps/rocm/src/main.rs) rather than moving alongside, and `services()` lives in [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs); the record removal and pruning it dispatches to now lives in [`service_records.rs`](../apps/rocm/src/service_records.rs), and only stop/restart are also reachable from the chat/LLM sandbox tool-calling path, out of scope for Phase 6b. - [`service_records.rs`](../apps/rocm/src/service_records.rs) — mechanical relocation, owns types: removing one managed-service record and every file it owns (`remove_managed_service_record`, `service_record_artifacts`) and the `rocm services prune` plan/render/apply steps (`build_service_prune_plan`, `render_service_prune_plan`, `apply_service_prune_plan`, `prune_managed_service_records`) with the age gate behind them (`DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS`, `service_prune_min_age_hours`). It reads record state through [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) and [`services_text.rs`](../apps/rocm/src/services_text.rs) and builds its advice text with [`service_hints.rs`](../apps/rocm/src/service_hints.rs); none of the three depends on it. `remove_managed_service_record`, `prune_managed_service_records`, `service_prune_min_age_hours` and `DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS` are `pub(crate)` (`services()` in [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs) calls the first three; the `ServicesCommand::Prune` clap definition in [`main.rs`](../apps/rocm/src/main.rs) uses the constant, which `main.rs` re-imports), and `ServicePruneOutcome`, the return type of `prune_managed_service_records`, is `pub(crate)` with `pub(crate)` `text`, `removed_records`, `removed_files` and `failures` fields that `services()` reads. Every other item is private. The cluster's regression tests live in this module's own `mod tests`, reaching the shared `test_paths`, `plant_service_record` and `plant_service_record_without_its_key` fixtures through `crate::tests::` (the last two widened to `pub(crate)` for that). - [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm services`' command handler (`services()`, which dispatches `list`, `logs`, `stop`, `restart`, `remove` and `prune`) and `run_approved_service_action`, the approval-gated stop/restart path. `ServicesCommand` stays in [`main.rs`](../apps/rocm/src/main.rs). Both functions are `pub(crate)`: `dispatch()` calls `services()` through a re-import, and [`advised_commands.rs`](../apps/rocm/src/advised_commands.rs)'s test reaches `run_approved_service_action` through a test-only re-import in `main.rs`. The module reads from [`services_text.rs`](../apps/rocm/src/services_text.rs), [`service_records.rs`](../apps/rocm/src/service_records.rs), [`service_hints.rs`](../apps/rocm/src/service_hints.rs) and [`cli_audit.rs`](../apps/rocm/src/cli_audit.rs); none of them calls back into it. The cluster's one regression test lives in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::`. -- [`services_text.rs`](../apps/rocm/src/services_text.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-209): the CLI/TUI logs browser (`render_logs_text`, `render_logs_browser_text`, `render_logs_browser_page_text`, `render_logs_browser_page_text_with_options`, `logs_browser_total_pages`, the `LogBrowserEntry` type, `collect_log_browser_entries`, `filter_log_browser_entries`, `log_browser_source_label`, `write_past_attempts_hint`), the `rocm services`/`rocm services logs` status surfaces (`render_services_text`, `render_services_json`, `render_service_logs_text`, `render_service_logs_text_with_options`, `render_service_action_result`, `read_managed_service_manifest`, `load_managed_service`), the internal `rocmd`-compatible status/sandbox surface (`render_internal_status_text`, `run_internal_sandbox_tool`), and the managed-service stop/restart implementation behind both the sandbox tool and `rocm services stop|restart` (`terminate_recorded_service_pids`, the `MANAGED_STOP_GRACE` constant, `stop_internal_managed_service`, `stop_internal_managed_service_with`, `stop_internal_managed_service_confirmed_with`, `unload_lemonade_service_model`, `apply_restart_runtime_pin`, `load_service_for_restart`, `restart_child_argv`, `restart_internal_managed_service`, `restart_internal_managed_service_with`, `update_restarting_record`, `record_restarted_child`, `record_restart_readiness`), plus `validate_service_id`, `list_log_files`, `read_optional_tail_lines`, and the CLI-lifecycle-log tail formatting helpers (`format_cli_lifecycle_tail_line`, `lifecycle_event_label`, `humanize_log_label`, `lifecycle_field`, `read_tail_lines`). Two items physically interleaved in this span in `main.rs` moved with it rather than staying behind, since neither has a caller outside the cluster: the `LogBrowserEntry` struct and the `MANAGED_STOP_GRACE` constant; both stay private. `dispatch()` calls directly into this module at several points — `render_internal_status_text`, `run_internal_sandbox_tool`, `render_service_logs_text`, `render_logs_browser_text`, and `render_logs_text` — all re-imported via `use crate::services_text::{...};`; [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs)'s `services()` calls `render_services_text`, `render_services_json` and `render_service_logs_text` through its own imports. `render_service_action_result`, `run_internal_sandbox_tool`, and `validate_service_id` are also `pub(crate)` and read from `services_cmd.rs`'s `run_approved_service_action`; `validate_service_id` is also read from [`service_records.rs`](../apps/rocm/src/service_records.rs)'s `remove_managed_service_record`/`service_record_artifacts`, and `load_managed_service` from its `remove_managed_service_record`/`apply_service_prune_plan`. `load_managed_service` and `restart_internal_managed_service` are `pub(crate)`, read from [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) (reconciling live servers against a runtime activation) and from [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs)'s own test module; `read_tail_lines`, `render_logs_text`, `render_logs_browser_text`, `render_services_text`, `render_service_logs_text`, and `validate_service_id` are also `pub(crate)`, read from [`chat.rs`](../apps/rocm/src/chat.rs)'s deterministic tool-output summaries; `read_optional_tail_lines` and `render_service_action_result` are also `pub(crate)`, read from [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s managed-spawn/attach path. Every other item in this list has no caller outside this module and stays private, including `restart_internal_managed_service_with`, `restart_child_argv`, `load_service_for_restart`, `apply_restart_runtime_pin`, and `unload_lemonade_service_model` — see [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs)'s and [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s entries above for how those two modules' own doc comments now describe this module, rather than `main.rs`, as that caller. The cluster's regression tests live in this module's own `mod tests`, reaching shared fixtures (`test_paths`, `plant_service_record_on_runtime`, `OLD_RUNTIME_KEY`, `NEW_RUNTIME_KEY`) through `crate::tests::` the same way other mechanically relocated modules do, and `runtime_services::` through its own `use crate::runtime_services;` for the handful of tests that drive this cluster's restart/stop behavior through `runtime_services`'s public entry points rather than calling it directly. Two tests that call this cluster's `load_managed_service` only incidentally — their primary subject is `runtime_services::pin_service_record_to_runtime`/`restore_service_record_pin` and `runtime_services::restart_service_onto_runtime_with`'s outcome classification respectively — stay in `main.rs`'s shared `mod tests` instead. +- [`services_text.rs`](../apps/rocm/src/services_text.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-209): the CLI/TUI logs browser (`render_logs_text`, `render_logs_browser_text`, `render_logs_browser_page_text`, `render_logs_browser_page_text_with_options`, `logs_browser_total_pages`, the `LogBrowserEntry` type, `collect_log_browser_entries`, `filter_log_browser_entries`, `log_browser_source_label`, `write_past_attempts_hint`), the `rocm services`/`rocm services logs` status surfaces (`render_services_text`, `render_services_json`, `render_service_logs_text`, `render_service_logs_text_with_options`, `render_service_action_result`, `read_managed_service_manifest`, `load_managed_service`), the `--follow` tail behind `rocm logs --service --follow` and `rocm services logs --follow` (`follow_service_logs`, `follow_service_logs_with`, the `LogTail` and `LogSegment` types, `write_log_segments`), the internal `rocmd`-compatible status/sandbox surface (`render_internal_status_text`, `run_internal_sandbox_tool`), and the managed-service stop/restart implementation behind both the sandbox tool and `rocm services stop|restart` (`terminate_recorded_service_pids`, the `MANAGED_STOP_GRACE` constant, `stop_internal_managed_service`, `stop_internal_managed_service_with`, `stop_internal_managed_service_confirmed_with`, `unload_lemonade_service_model`, `apply_restart_runtime_pin`, `load_service_for_restart`, `restart_child_argv`, `restart_internal_managed_service`, `restart_internal_managed_service_with`, `update_restarting_record`, `record_restarted_child`, `record_restart_readiness`), plus `validate_service_id`, `list_log_files`, `read_optional_tail_lines`, and the CLI-lifecycle-log tail formatting helpers (`format_cli_lifecycle_tail_line`, `lifecycle_event_label`, `humanize_log_label`, `lifecycle_field`, `read_tail_lines`). Two items physically interleaved in this span in `main.rs` moved with it rather than staying behind, since neither has a caller outside the cluster: the `LogBrowserEntry` struct and the `MANAGED_STOP_GRACE` constant; both stay private. `dispatch()` calls directly into this module at several points — `render_internal_status_text`, `run_internal_sandbox_tool`, `render_service_logs_text`, `render_logs_browser_text`, and `render_logs_text` — all re-imported via `use crate::services_text::{...};`; [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs)'s `services()` calls `render_services_text`, `render_services_json`, `render_service_logs_text` and `follow_service_logs` through its own imports, and `dispatch()` calls `follow_service_logs` for `rocm logs --service --follow`. `render_service_action_result`, `run_internal_sandbox_tool`, and `validate_service_id` are also `pub(crate)` and read from `services_cmd.rs`'s `run_approved_service_action`; `validate_service_id` is also read from [`service_records.rs`](../apps/rocm/src/service_records.rs)'s `remove_managed_service_record`/`service_record_artifacts`, and `load_managed_service` from its `remove_managed_service_record`/`apply_service_prune_plan`. `load_managed_service` and `restart_internal_managed_service` are `pub(crate)`, read from [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) (reconciling live servers against a runtime activation) and from [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs)'s own test module; `read_tail_lines`, `render_logs_text`, `render_logs_browser_text`, `render_services_text`, `render_service_logs_text`, and `validate_service_id` are also `pub(crate)`, read from [`chat.rs`](../apps/rocm/src/chat.rs)'s deterministic tool-output summaries; `read_optional_tail_lines` and `render_service_action_result` are also `pub(crate)`, read from [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s managed-spawn/attach path. Every other item in this list has no caller outside this module and stays private, including `restart_internal_managed_service_with`, `restart_child_argv`, `load_service_for_restart`, `apply_restart_runtime_pin`, and `unload_lemonade_service_model` — see [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs)'s and [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s entries above for how those two modules' own doc comments now describe this module, rather than `main.rs`, as that caller. The cluster's regression tests live in this module's own `mod tests`, reaching shared fixtures (`test_paths`, `plant_service_record_on_runtime`, `OLD_RUNTIME_KEY`, `NEW_RUNTIME_KEY`) through `crate::tests::` the same way other mechanically relocated modules do, and `runtime_services::` through its own `use crate::runtime_services;` for the handful of tests that drive this cluster's restart/stop behavior through `runtime_services`'s public entry points rather than calling it directly. Two tests that call this cluster's `load_managed_service` only incidentally — their primary subject is `runtime_services::pin_service_record_to_runtime`/`restore_service_record_pin` and `runtime_services::restart_service_onto_runtime_with`'s outcome classification respectively — stay in `main.rs`'s shared `mod tests` instead. - [`setup_cmd.rs`](../apps/rocm/src/setup_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm setup`'s command handler (`setup()`, which runs `setup status` by default and `setup reset`), the text `rocm setup status` prints (`render_setup_status_text`) and the reset that clears the saved first-time prompt state (`reset_setup_prompt_state`). `SetupCommand` stays in [`main.rs`](../apps/rocm/src/main.rs). `setup()` and `render_setup_status_text` are `pub(crate)` and re-imported in `main.rs`: `dispatch()` calls the first, and [`chat.rs`](../apps/rocm/src/chat.rs)'s read-only handling of `rocm setup` and `rocm setup status` reaches the second as `crate::render_setup_status_text`; `reset_setup_prompt_state` is private. The three regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` and `write_test_pip_runtime` through `crate::test_support::`. - [`storage.rs`](../apps/rocm/src/storage.rs) — full domain extraction. - [`system_packages.rs`](../apps/rocm/src/system_packages.rs) — full domain extraction, a leaf of the crate (split out of [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)): the host system-package installs that `rocm install sdk` and `rocm engines install vllm` run through `sudo` apt, dnf or zypper. It owns the distro-aware OpenMPI, libatomic and libnuma flows (`ensure_openmpi_for_vllm`, `ensure_libatomic_for_torch`, `ensure_libnuma_for_torch`), the choice between printing the commands and running them (`SystemPackageInstallAction`, `system_package_install_action`), the guard that refuses an `apt-get install` which would remove ROCm packages, and `real_os_release`, the real host's os-release that every one of these plans is built from. Production code uses only `std`, `anyhow` and `rocm_core`; the three `ensure_*` functions and `real_os_release` are `pub(crate)` and the rest is private. `runtime_install_finalize.rs` re-exports the three `ensure_*` names for [`install_cmd.rs`](../apps/rocm/src/install_cmd.rs) and [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), and [`driver_install.rs`](../apps/rocm/src/driver_install.rs) imports `real_os_release` from here directly. `SdkInstallConsents` stays in `runtime_install_finalize.rs` (it carries a `therock::SdkInstallConsent`); the functions here take its plain `system_packages` bool. `torch_runtime_dep_checks_disabled` is the E2E hook that skips the torch dependency checks; it only does anything under `e2e-test-hooks`. Its tests cover the consent decision (reaching `consents_for_install_sdk_argv` in `runtime_install_finalize.rs`'s test module) and, under `e2e-test-hooks` on Linux, that `real_os_release` ignores a simulated host root. diff --git a/docs/manual-testing.md b/docs/manual-testing.md index f66af87d4f..5c80387e03 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -383,6 +383,14 @@ Expected result: block at all. - `rocm services list --all` shows saved history, including the failed or stopped attempts the default view hides. +- `rocm services logs --follow` prints the recent output, then new + output as it arrives (a download bar redraws on one terminal row), and returns + by itself with `Service is ; stopped following.` once the server + stops; Ctrl-C ends the follow and the server keeps running. +- `rocm serve --wait` keeps waiting past 45 seconds and returns once the + server answers. If the server is killed or stopped meanwhile, it exits non-zero + with `the server exited before it became ready` and a `rocm logs --service` + pointer, and the record shows `stopped`. - The logs command shows the exact service failure or startup output. - `rocm storage report` lists these records under `local server records` with the folder that holds them; no `rocm storage` command deletes them, and the diff --git a/docs/testing.md b/docs/testing.md index 62f46cb9ca..50bfdde339 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1010,7 +1010,8 @@ rocm logs --service --follow # Ctrl-C to stop following ``` While a first-time model download is running, `rocm serve --wait` keeps -waiting past the usual 45 seconds until the server answers (or exits), and +waiting past the usual 45 seconds until the server answers (or exits, which fails the +command), and `--wait` with `--verbose` or `--foreground` is refused (`serve-41`). Run the log follow above from a second terminal to watch the download. diff --git a/tests/e2e-cucumber/features/logs.feature b/tests/e2e-cucumber/features/logs.feature index 51c5314ccd..004818827b 100644 --- a/tests/e2e-cucumber/features/logs.feature +++ b/tests/e2e-cucumber/features/logs.feature @@ -42,3 +42,11 @@ Feature: Log inspection Scenario: logs-05 - Following without naming a server is refused When the user asks to follow logs without naming a service Then the CLI refuses because --follow needs --service + + # The same follow behavior through the `rocm services logs` entry point, using + # the `-f` short form (the `rocm logs` scenarios above use `--follow`). + @id:services-logs-follow-stopped-service-returns + Scenario: logs-06 - Following a stopped server's logs through the services command returns + Given a stopped local server whose log shows a download at 43 percent + When the user follows that server's logs with rocm services logs -f + Then the CLI prints the download line, says it is not following, and exits diff --git a/tests/e2e-cucumber/tests/e2e/logs_steps.rs b/tests/e2e-cucumber/tests/e2e/logs_steps.rs index 6ab899f315..0dc14673b2 100644 --- a/tests/e2e-cucumber/tests/e2e/logs_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/logs_steps.rs @@ -169,6 +169,13 @@ async fn follow_stopped_server(world: &mut E2eWorld) { record(world, stdout, stderr, rc); } +#[when("the user follows that server's logs with rocm services logs -f")] +async fn follow_stopped_server_through_services(world: &mut E2eWorld) { + let (stdout, stderr, rc) = + crate::run_rocm(world, &["services", "logs", FOLLOW_SERVICE_ID, "-f"]); + record(world, stdout, stderr, rc); +} + #[when("the user asks to follow logs without naming a service")] async fn follow_without_service(world: &mut E2eWorld) { let (stdout, stderr, rc) = crate::run_rocm(world, &["logs", "--follow"]); From b9deedfc93638c3ef40efed95e0482e939ab5e72 Mon Sep 17 00:00:00 2001 From: juhovainio Date: Wed, 4 Nov 2026 14:09:59 +0200 Subject: [PATCH 04/12] ROCMAI-791: test the serve --wait server-died path; keep a failed state Pull the post-wait decision into died_during_wait and assert the error, the resulting record state and the no-op cases together. settle_exited_service now persists an engine-reported failed state instead of overwriting it, with a failed-record test for its guard. README notes --wait is ignored for an already-running service; testing.md serve-41 wording matches the scenario. Signed-off-by: juhovainio --- README.md | 4 +- apps/rocm/src/service_launch.rs | 102 ++++++++++++++++++++++++++------ docs/testing.md | 3 +- 3 files changed, 89 insertions(+), 20 deletions(-) diff --git a/README.md b/README.md index bad213a0b8..37799e5a32 100644 --- a/README.md +++ b/README.md @@ -913,7 +913,9 @@ long a first-time model download takes, instead of returning `starting` after 45 seconds; if the server exits first (crashes, is killed, or is stopped with `rocm services stop`), it fails with an error that points at the server's logs instead of waiting forever. Ctrl-C ends the wait and leaves the server starting -in the background. It cannot be combined with `--foreground` or `--verbose`. Watch the download with +in the background. It cannot be combined with `--foreground` or `--verbose`, and +it applies only to a launch: when the model is already being served, `rocm serve` +reports its current status and returns without waiting. Watch the download with `rocm logs --service --follow` from another terminal. `--no-smoke-test` skips the post-startup inference probe. diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index e4329810d8..4ac218903b 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -39,7 +39,7 @@ use std::process::{Command as ProcessCommand, Stdio}; use std::thread; use std::time::Duration; -use anyhow::{Context, Result, bail}; +use anyhow::{Context, Result, anyhow, bail}; use rocm_core::{AppPaths, AutomationRuntimeState, EndpointReadiness, ManagedServiceRecord}; use rocm_core::{format_http_base_url, process_is_running}; use rocm_engine_protocol::{DevicePolicy, EngineRecipeHint, ResolveModelResponse}; @@ -737,14 +737,42 @@ fn service_still_live(paths: &AppPaths, service_id: &str, pid_alive: impl Fn(u32 } /// Record a server that died mid-start as stopped, leaving a manifest that a -/// `services stop` already settled untouched. +/// `services stop` already settled untouched. A `failed` state the engine +/// reported is persisted rather than overwritten, so the reason survives. fn settle_exited_service(paths: &AppPaths, service_id: &str) { - if let Ok(mut latest) = ManagedServiceRecord::read_manifest(paths, service_id) - && crate::managed_service_status::managed_service_is_live(&latest) - { + let Ok(mut latest) = ManagedServiceRecord::read_manifest(paths, service_id) else { + return; + }; + let on_disk = latest.status.clone(); + let _ = latest.refresh_from_engine_state(); + if crate::managed_service_status::managed_service_is_live(&latest) { latest.status = "stopped".to_owned(); - let _ = latest.write(); + } else if latest.status == on_disk { + return; + } + let _ = latest.write(); +} + +/// The failure a `--wait` ends with when the server died before it became +/// ready, after recording the death in the manifest. `None` while the wait did +/// not apply, the endpoint is serving, or the server is still alive. +fn died_during_wait( + paths: &AppPaths, + service_id: &str, + wait_until_ready: bool, + readiness: EndpointReadiness, + pid_alive: impl Fn(u32) -> bool, +) -> Option { + if !wait_until_ready + || readiness == EndpointReadiness::Serving + || service_still_live(paths, service_id, pid_alive) + { + return None; } + settle_exited_service(paths, service_id); + Some(anyhow!( + "the server exited before it became ready; see `rocm logs --service {service_id}` for its output" + )) } #[allow(clippy::too_many_arguments)] @@ -819,14 +847,14 @@ pub(crate) fn start_managed_service( } else { probe(on_wait_tick) }; - if wait_until_ready - && readiness != EndpointReadiness::Serving - && !service_still_live(&paths, service_id, service_pid_is_alive) - { - settle_exited_service(&paths, service_id); - bail!( - "the server exited before it became ready; see `rocm logs --service {service_id}` for its output" - ); + if let Some(error) = died_during_wait( + &paths, + service_id, + wait_until_ready, + readiness, + service_pid_is_alive, + ) { + return Err(error); } let launch_status = status_for_readiness(readiness); record.status = launch_status.to_owned(); @@ -1636,13 +1664,51 @@ mod tests { let after = ManagedServiceRecord::read_manifest(&paths, &record.service_id).unwrap(); assert_eq!(after.status, "stopped"); - // A stop that already settled the record keeps its own fields. - record.status = "stopped".to_owned(); - record.stop_requested_unix_ms = Some(7); + // A record that already failed keeps its reason: the guard, not the + // overwrite, is what spares it (a `stopped` -> `stopped` rewrite would + // be invisible). + record.status = "failed".to_owned(); record.write().expect("write"); settle_exited_service(&paths, &record.service_id); let after = ManagedServiceRecord::read_manifest(&paths, &record.service_id).unwrap(); - assert_eq!(after.stop_requested_unix_ms, Some(7)); + assert_eq!(after.status, "failed"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn died_during_wait_fails_with_a_logs_pointer_and_records_the_death() { + let (root, paths) = test_paths("wait-died"); + paths.ensure().expect("paths"); + let record = record_tracking_pid(&paths, std::process::id(), "starting"); + record.write().expect("write"); + let id = record.service_id; + let status = || { + ManagedServiceRecord::read_manifest(&paths, &id) + .unwrap() + .status + }; + + // Not waiting, already serving, or still alive: no failure, no change. + let pending = EndpointReadiness::Unreachable; + assert!(died_during_wait(&paths, &id, false, pending, |_| false).is_none()); + assert!( + died_during_wait(&paths, &id, true, EndpointReadiness::Serving, |_| false).is_none() + ); + assert!(died_during_wait(&paths, &id, true, pending, |_| true).is_none()); + assert_eq!(status(), "starting"); + + // Dead server: the error names the logs command and the record is settled. + let error = died_during_wait(&paths, &id, true, pending, |_| false).expect("died"); + let message = error.to_string(); + assert!( + message.contains("exited before it became ready"), + "{message}" + ); + assert!( + message.contains(&format!("rocm logs --service {id}")), + "{message}" + ); + assert_eq!(status(), "stopped"); let _ = fs::remove_dir_all(root); } diff --git a/docs/testing.md b/docs/testing.md index 50bfdde339..12714810d0 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1012,7 +1012,8 @@ rocm logs --service --follow # Ctrl-C to stop following While a first-time model download is running, `rocm serve --wait` keeps waiting past the usual 45 seconds until the server answers (or exits, which fails the command), and -`--wait` with `--verbose` or `--foreground` is refused (`serve-41`). Run the +`--wait` with `--verbose` is refused (`serve-41`; `--foreground` is refused by the +argument parser, covered by a unit test). Run the log follow above from a second terminal to watch the download. The no-log first-run path should report `no CLI logs found yet` without From 8fedcfaffc1262d3352407f89f4a739ccd91d511 Mon Sep 17 00:00:00 2001 From: juhovainio Date: Wed, 4 Nov 2026 15:59:27 +0200 Subject: [PATCH 05/12] ROCMAI-791: fix CI: const fn on non-Linux zombie probe, feature-qualified logs-06 id Signed-off-by: juhovainio --- apps/rocm/src/service_launch.rs | 2 +- tests/e2e-cucumber/features/logs.feature | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 4ac218903b..2ba7ab13f4 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -711,7 +711,7 @@ fn process_is_zombie(pid: u32) -> bool { } #[cfg(not(target_os = "linux"))] -fn process_is_zombie(_pid: u32) -> bool { +const fn process_is_zombie(_pid: u32) -> bool { false } diff --git a/tests/e2e-cucumber/features/logs.feature b/tests/e2e-cucumber/features/logs.feature index 004818827b..f3cd6cd5ae 100644 --- a/tests/e2e-cucumber/features/logs.feature +++ b/tests/e2e-cucumber/features/logs.feature @@ -45,7 +45,7 @@ Feature: Log inspection # The same follow behavior through the `rocm services logs` entry point, using # the `-f` short form (the `rocm logs` scenarios above use `--follow`). - @id:services-logs-follow-stopped-service-returns + @id:logs-services-follow-stopped-service-returns Scenario: logs-06 - Following a stopped server's logs through the services command returns Given a stopped local server whose log shows a download at 43 percent When the user follows that server's logs with rocm services logs -f From d0062bfb1e67d77a28fe36faa47716fc1dbea08c Mon Sep 17 00:00:00 2001 From: juhovainio Date: Wed, 4 Nov 2026 16:31:11 +0200 Subject: [PATCH 06/12] ROCMAI-791: keep the --follow hint under the newest log line on a terminal The one-time "Following ... Ctrl-C stops following" line scrolled out of view as soon as the log filled the screen. On a terminal it is now a footer on the row below the newest output, redrawn only when new output arrives and kept below an in-place download bar. It is cut to the terminal width and cleared before the "stopped following" message. Piped output keeps the one-time line. Signed-off-by: juhovainio --- README.md | 4 +- apps/rocm/src/services_text.rs | 369 ++++++++++++++++++++++++++++++++- docs/manual-testing.md | 3 + 3 files changed, 366 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 37799e5a32..999ecde36e 100644 --- a/README.md +++ b/README.md @@ -1307,7 +1307,9 @@ rocm uninstall [--yes] [--dry-run] `--follow` (`-f`) keeps printing a server's new log output, such as a model download's progress, until the server stops. Ctrl-C ends the follow and leaves -the server running. +the server running. On a terminal, a `Following - Ctrl-C stops following ...` +line stays on the row under the newest output; piped or redirected output prints +it once. `rocm uninstall` removes four categories of files that ROCm CLI manages on this computer: its binaries, configuration, data, and caches. It lists what it will diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index 4d23538dc7..05b6f6c16b 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -650,7 +650,13 @@ impl LogTail { } } -fn write_log_segments(out: &mut impl io::Write, segments: &[LogSegment], tty: bool) -> Result<()> { +/// Writes `segments`; returns whether the last one was a redraw, which leaves the +/// cursor on that row (the next segment erases it first) instead of a fresh one. +fn write_log_segments( + out: &mut impl io::Write, + segments: &[LogSegment], + tty: bool, +) -> Result { for segment in segments { match (tty, segment.redraw) { // Redraw the same terminal row, as the engine's own bar does. @@ -660,7 +666,87 @@ fn write_log_segments(out: &mut impl io::Write, segments: &[LogSegment], tty: bo } } out.flush()?; - Ok(()) + Ok(tty && segments.last().is_some_and(|segment| segment.redraw)) +} + +/// The "how to stop" hint kept on the row just below the newest log output of a +/// terminal follow, so it stays in view however much the log scrolls. +/// +/// While drawn, the cursor sits on the footer row. [`Self::erase`] puts it back +/// where the log output left off: on a fresh row, or, after a redraw segment +/// (a download bar), at the end of that still-open row, one row up. +struct FollowFooter { + text: String, + drawn: bool, + /// The row above the footer is an unfinished redraw row. + open_row: bool, +} + +impl FollowFooter { + fn new(service_id: &str) -> Self { + Self { + text: format!( + "Following {service_id} - Ctrl-C stops following and leaves the server running." + ), + drawn: false, + open_row: false, + } + } + + fn erase(&mut self, out: &mut impl io::Write) -> Result<()> { + if self.drawn { + write!(out, "\r\x1b[2K")?; + if self.open_row { + write!(out, "\x1b[1A")?; + } + self.drawn = false; + } + Ok(()) + } + + /// `cols` truncates the text so it never wraps onto a second row, which the + /// one-row erase could not clear. + fn draw( + &mut self, + out: &mut impl io::Write, + open_row: bool, + cols: Option, + ) -> Result<()> { + if open_row { + writeln!(out)?; + } + let shown: String = match cols { + Some(cols) => self.text.chars().take(cols.saturating_sub(1)).collect(), + None => self.text.clone(), + }; + write!(out, "\r\x1b[2K{shown}")?; + out.flush()?; + self.drawn = true; + self.open_row = open_row; + Ok(()) + } + + /// Write a batch of new output with the footer kept below it. A batch with + /// nothing in it leaves the screen alone (no flicker on an idle log). + fn write_batch( + &mut self, + out: &mut impl io::Write, + segments: &[LogSegment], + cols: Option, + ) -> Result<()> { + if segments.is_empty() { + return Ok(()); + } + self.erase(out)?; + let open_row = write_log_segments(out, segments, true)?; + self.draw(out, open_row, cols) + } +} + +fn terminal_cols() -> Option { + crossterm::terminal::size() + .ok() + .map(|(cols, _)| usize::from(cols)) } /// `rocm logs --service --follow`: print the recent output, then keep @@ -677,6 +763,7 @@ pub(crate) fn follow_service_logs(paths: &AppPaths, service_id: &str) -> Result< tty, LOG_FOLLOW_POLL, LOG_FOLLOW_LIVENESS_EVERY, + &terminal_cols, ) } @@ -687,6 +774,7 @@ fn follow_service_logs_with( tty: bool, poll: Duration, liveness_every: Duration, + cols: &dyn Fn() -> Option, ) -> Result<()> { let record = load_managed_service(paths, service_id)?; // Open the tail before the snapshot so output written in between is not lost. @@ -700,16 +788,27 @@ fn follow_service_logs_with( )?; return Ok(()); } - writeln!( - out, - "\nFollowing {service_id} - Ctrl-C stops following and leaves the server running." - )?; + // On a terminal the hint is a footer that stays under the newest output; a + // pipe or log file gets it once, since it has no row to keep it on. + let mut footer = tty.then(|| FollowFooter::new(service_id)); + match footer.as_mut() { + Some(footer) => footer.draw(out, false, cols())?, + None => writeln!( + out, + "\nFollowing {service_id} - Ctrl-C stops following and leaves the server running." + )?, + } out.flush()?; let mut since_check = Duration::ZERO; loop { thread::sleep(poll); - write_log_segments(out, &tail.poll(), tty)?; + match footer.as_mut() { + Some(footer) => footer.write_batch(out, &tail.poll(), cols())?, + None => { + write_log_segments(out, &tail.poll(), tty)?; + } + } since_check += poll; if since_check < liveness_every { continue; @@ -720,8 +819,13 @@ fn follow_service_logs_with( // Output written between the last poll and the exit. let mut last = tail.poll(); last.extend(tail.finish()); - write_log_segments(out, &last, tty)?; - if tty { + let mut open_row = false; + if let Some(footer) = footer.as_mut() { + footer.erase(out)?; + open_row = footer.open_row; + } + open_row = write_log_segments(out, &last, tty)? || (open_row && last.is_empty()); + if open_row { writeln!(out)?; } writeln!( @@ -1957,6 +2061,251 @@ mod tests { Ok(()) } + fn seg(text: &str, redraw: bool) -> LogSegment { + LogSegment { + text: text.into(), + redraw, + } + } + + const ERASE: &str = "\r\x1b[2K"; + + #[test] + fn follow_footer_is_redrawn_under_each_batch_of_new_lines() -> Result<()> { + let mut footer = FollowFooter::new("svc_a"); + let hint = footer.text.clone(); + let mut out = Vec::new(); + footer.draw(&mut out, false, None)?; + assert_eq!(String::from_utf8(out.clone())?, format!("{ERASE}{hint}")); + + out.clear(); + footer.write_batch(&mut out, &[seg("one", false), seg("two", false)], None)?; + // Footer row cleared, lines written over it, footer redrawn on the next row. + assert_eq!( + String::from_utf8(out)?, + format!("{ERASE}{ERASE}one\n{ERASE}two\n{ERASE}{hint}") + ); + Ok(()) + } + + #[test] + fn follow_footer_leaves_an_idle_screen_alone() -> Result<()> { + let mut footer = FollowFooter::new("svc_a"); + let mut out = Vec::new(); + footer.draw(&mut out, false, None)?; + out.clear(); + footer.write_batch(&mut out, &[], None)?; + assert!(out.is_empty(), "{out:?}"); + Ok(()) + } + + #[test] + fn follow_footer_keeps_a_download_bar_on_its_own_row_above_it() -> Result<()> { + let mut footer = FollowFooter::new("svc_a"); + let hint = footer.text.clone(); + let mut out = Vec::new(); + footer.draw(&mut out, false, None)?; + out.clear(); + + footer.write_batch(&mut out, &[seg("28%", true)], None)?; + // The bar row stays open; the footer goes on the row below it. + assert_eq!( + String::from_utf8(out.clone())?, + format!("{ERASE}{ERASE}28%\n{ERASE}{hint}") + ); + + out.clear(); + footer.write_batch(&mut out, &[seg("43%", true)], None)?; + // Erase the footer, step back up onto the bar row, redraw it in place. + assert_eq!( + String::from_utf8(out.clone())?, + format!("{ERASE}\x1b[1A{ERASE}43%\n{ERASE}{hint}") + ); + + out.clear(); + footer.write_batch(&mut out, &[seg("done", false)], None)?; + // A finished line replaces the bar, and the footer is back on a fresh row. + assert_eq!( + String::from_utf8(out)?, + format!("{ERASE}\x1b[1A{ERASE}done\n{ERASE}{hint}") + ); + Ok(()) + } + + #[test] + fn follow_footer_is_cut_to_one_terminal_row() -> Result<()> { + let mut footer = FollowFooter::new("svc_a"); + let mut out = Vec::new(); + footer.draw(&mut out, false, Some(11))?; + // One column short of the width, so it never wraps. + assert_eq!(String::from_utf8(out)?, format!("{ERASE}Following ")); + Ok(()) + } + + #[test] + fn follow_service_logs_keeps_the_hint_under_new_output_on_a_tty() -> Result<()> { + let (root, paths) = test_paths("follow-tty"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "svc_follow_tty", + "vllm", + "qwen3.5", + "Qwen/Qwen3.5", + "127.0.0.1", + 11438, + "managed", + std::process::id(), + None, + None, + None, + ); + record.status = "starting".to_owned(); + record.write()?; + fs::write(&record.log_path, "before\n")?; + + let log_path = record.log_path; + let manifest_paths = paths.clone(); + let writer = thread::spawn(move || -> Result<()> { + let appended = (|| -> Result<()> { + thread::sleep(Duration::from_millis(150)); + let mut file = fs::OpenOptions::new().append(true).open(&log_path)?; + std::io::Write::write_all(&mut file, b"Downloading 43%\n")?; + thread::sleep(Duration::from_millis(150)); + Ok(()) + })(); + let mut record = load_managed_service(&manifest_paths, "svc_follow_tty")?; + record.status = "stopped".to_owned(); + record.write()?; + appended + }); + let mut out = Vec::new(); + follow_service_logs_with( + &paths, + "svc_follow_tty", + &mut out, + true, + Duration::from_millis(20), + Duration::from_millis(40), + &|| None, + )?; + writer.join().expect("writer thread")?; + let out = String::from_utf8(out)?; + let hint = FollowFooter::new("svc_follow_tty").text; + // The hint is drawn again right under the line that scrolled it away, + // and only the stop message follows it, after the footer row is cleared. + assert!( + out.contains(&format!("Downloading 43%\n{ERASE}{hint}")), + "{out:?}" + ); + assert!( + out.ends_with(&format!( + "{hint}{ERASE}Service svc_follow_tty is stopped; stopped following.\n" + )), + "{out:?}" + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn follow_service_logs_ends_an_open_download_bar_row_before_the_stop_message() -> Result<()> { + let (root, paths) = test_paths("follow-tty-bar"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "svc_follow_bar", + "vllm", + "qwen3.5", + "Qwen/Qwen3.5", + "127.0.0.1", + 11440, + "managed", + std::process::id(), + None, + None, + None, + ); + record.status = "starting".to_owned(); + record.write()?; + fs::write(&record.log_path, "")?; + + let log_path = record.log_path; + let manifest_paths = paths.clone(); + let writer = thread::spawn(move || -> Result<()> { + let appended = (|| -> Result<()> { + thread::sleep(Duration::from_millis(150)); + let mut file = fs::OpenOptions::new().append(true).open(&log_path)?; + std::io::Write::write_all(&mut file, b"28%\r43%\r\r")?; + thread::sleep(Duration::from_millis(150)); + Ok(()) + })(); + let mut record = load_managed_service(&manifest_paths, "svc_follow_bar")?; + record.status = "stopped".to_owned(); + record.write()?; + appended + }); + let mut out = Vec::new(); + follow_service_logs_with( + &paths, + "svc_follow_bar", + &mut out, + true, + Duration::from_millis(20), + Duration::from_millis(40), + &|| None, + )?; + writer.join().expect("writer thread")?; + let out = String::from_utf8(out)?; + // The footer is erased and the cursor goes back up onto the bar's row; that row + // is then ended with a newline so the stop message starts on its own row. + assert!( + out.ends_with(&format!( + "{ERASE}\x1b[1A\nService svc_follow_bar is stopped; stopped following.\n" + )), + "{out:?}" + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn follow_service_logs_draws_no_footer_for_a_stopped_service_on_a_tty() -> Result<()> { + let (root, paths) = test_paths("follow-tty-stopped"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "svc_follow_tty_stopped", + "vllm", + "qwen3.5", + "Qwen/Qwen3.5", + "127.0.0.1", + 11439, + "managed", + 0, + None, + None, + None, + ); + record.status = "stopped".to_owned(); + record.write()?; + let mut out = Vec::new(); + follow_service_logs_with( + &paths, + "svc_follow_tty_stopped", + &mut out, + true, + Duration::from_millis(1), + Duration::from_millis(1), + &|| None, + )?; + let out = String::from_utf8(out)?; + assert!(!out.contains("Following"), "{out:?}"); + assert!(out.contains("not following."), "{out:?}"); + let _ = fs::remove_dir_all(root); + Ok(()) + } + #[test] fn follow_service_logs_prints_snapshot_and_returns_for_a_stopped_service() -> Result<()> { let (root, paths) = test_paths("follow-stopped"); @@ -1987,6 +2336,7 @@ mod tests { false, Duration::from_millis(1), Duration::from_millis(1), + &|| None, )?; let out = String::from_utf8(out)?; assert!(out.contains("Downloading 86%"), "{out}"); @@ -2045,6 +2395,7 @@ mod tests { false, Duration::from_millis(20), Duration::from_millis(40), + &|| None, )?; writer.join().expect("writer thread")?; let out = String::from_utf8(out)?; diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 5c80387e03..d74b6a1807 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -387,6 +387,9 @@ Expected result: output as it arrives (a download bar redraws on one terminal row), and returns by itself with `Service is ; stopped following.` once the server stops; Ctrl-C ends the follow and the server keeps running. + On a terminal, a `Following - Ctrl-C stops following ...` line stays on the + row directly under the newest output (and under the download bar) as the log + grows; with output piped to a file it is printed once instead. - `rocm serve --wait` keeps waiting past 45 seconds and returns once the server answers. If the server is killed or stopped meanwhile, it exits non-zero with `the server exited before it became ready` and a `rocm logs --service` From 486dcc3ee0edb293a50feaa8f15e6039b4ee1d57 Mon Sep 17 00:00:00 2001 From: juhovainio Date: Thu, 5 Nov 2026 11:46:32 +0200 Subject: [PATCH 07/12] ROCMAI-791: order serve-42 after serve-41 so scenario names stay sequential Signed-off-by: juhovainio --- .../features/model_serving.feature | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 473f15cba1..dbe74f32dd 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -646,17 +646,6 @@ Feature: Model serving Given a managed runtime is active When the user serves a GGUF model on lemonade with no automation enabled Then serving started no background helper - - # `--wait` makes a managed serve wait for readiness past the usual 45 seconds, - # so it cannot be combined with the attached modes, which already stay on the - # terminal. The waiting itself (a live server that becomes ready, one that - # exits first) is covered by `wait_past_readiness_budget_*` in - # `apps/rocm/src/service_launch.rs`, since a scenario would need a slow engine. - @id:serve-wait-conflicts-with-attached-modes - Scenario: serve-42 - Waiting for readiness cannot be combined with attached mode - When the user serves a model with both --wait and --verbose - Then the CLI refuses and names the conflicting flags - # The positive case for the OOM note (EAI-8059 review asked for it # explicitly, alongside the withholding the unit tests pin): when # THIS invocation launches a managed vLLM serve that OOMs and never becomes @@ -681,3 +670,14 @@ Feature: Model serving # suppression was untested: deleting the guard left the scenario green and a # `rocm daemon` outliving the checkout. And no background automation daemon was left behind + + + # `--wait` makes a managed serve wait for readiness past the usual 45 seconds, + # so it cannot be combined with the attached modes, which already stay on the + # terminal. The waiting itself (a live server that becomes ready, one that + # exits first) is covered by `wait_past_readiness_budget_*` in + # `apps/rocm/src/service_launch.rs`, since a scenario would need a slow engine. + @id:serve-wait-conflicts-with-attached-modes + Scenario: serve-42 - Waiting for readiness cannot be combined with attached mode + When the user serves a model with both --wait and --verbose + Then the CLI refuses and names the conflicting flags From f14f485b825e96bbaa6c6ba0ff372183deb05680 Mon Sep 17 00:00:00 2001 From: juhovainio Date: Thu, 5 Nov 2026 13:37:40 +0200 Subject: [PATCH 08/12] ROCMAI-791: wait on an already-starting server, tighten follow output serve --wait now waits on a matching server that is still starting instead of returning at once. Print the service id and follow command before waiting and clear the spinner when the wait fails. The follow footer leads with the keeps-running clause and is cut by display width, log text is stripped of control sequences, follow prints a ready line and exits non-zero when the service ended failed. Docs updated. Signed-off-by: juhovainio --- README.md | 11 +-- apps/rocm/src/cli_progress.rs | 2 +- apps/rocm/src/main.rs | 3 +- apps/rocm/src/serve_cmd.rs | 19 ++++- apps/rocm/src/serve_summary.rs | 2 +- apps/rocm/src/service_launch.rs | 144 ++++++++++++++++++++++++++------ apps/rocm/src/services_text.rs | 143 +++++++++++++++++++++++++++---- docs/manual-testing.md | 7 +- docs/testing.md | 7 +- 9 files changed, 283 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index e9a9a954fa..b7c7c908c0 100644 --- a/README.md +++ b/README.md @@ -918,8 +918,8 @@ seconds; if the server exits first (crashes, is killed, or is stopped with `rocm services stop`), it fails with an error that points at the server's logs instead of waiting forever. Ctrl-C ends the wait and leaves the server starting in the background. It cannot be combined with `--foreground` or `--verbose`, and -it applies only to a launch: when the model is already being served, `rocm serve` -reports its current status and returns without waiting. Watch the download with +when the model is already being served and still starting, it waits on that +server instead (an already-serving one is reported at once). Watch the download with `rocm logs --service --follow` from another terminal. `--no-smoke-test` skips the post-startup inference probe. @@ -1091,7 +1091,7 @@ Manage background servers started with `--managed`: ``` rocm services list [--all] [--json] -rocm services logs +rocm services logs [--follow] rocm services stop --yes rocm services restart --yes rocm services remove --yes @@ -1401,8 +1401,9 @@ rocm uninstall [--yes] [--dry-run] ``` `--follow` (`-f`) keeps printing a server's new log output, such as a model -download's progress, until the server stops. Ctrl-C ends the follow and leaves -the server running. On a terminal, a `Following - Ctrl-C stops following ...` +download's progress, until the server stops (a server that ends `failed` makes the command exit +non-zero), and prints `Service is ready.` when it finishes starting. Ctrl-C +ends the follow and leaves the server running. On a terminal, a `Ctrl-C stops following, the server keeps running. Following ` line stays on the row under the newest output; piped or redirected output prints it once. diff --git a/apps/rocm/src/cli_progress.rs b/apps/rocm/src/cli_progress.rs index 757476299b..65c5faa2a2 100644 --- a/apps/rocm/src/cli_progress.rs +++ b/apps/rocm/src/cli_progress.rs @@ -198,7 +198,7 @@ impl Spinner { /// CJK glyph occupies two terminal columns) to fit within `max_width` /// columns, appending an ellipsis when it doesn't already fit, so a repaint /// can never wrap to a second terminal row. -fn truncate_to_width(line: &str, max_width: usize) -> String { +pub(crate) fn truncate_to_width(line: &str, max_width: usize) -> String { if max_width == 0 { return String::new(); } diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index e414ee253d..09165bce3c 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -609,7 +609,8 @@ rocm serve qwen --verbose --device gpu_required")] managed: bool, /// Keep waiting until the server answers requests, however long a first-time /// model download takes, instead of returning `starting` after 45 seconds. - /// Fails with a pointer to the logs if the server exits first. Cannot be + /// Also waits on a matching server that is already starting. Fails with a + /// pointer to the logs if the server exits first. Cannot be /// combined with --foreground or --verbose, which already stay attached. #[arg(long, conflicts_with_all = ["foreground", "verbose"])] wait: bool, diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 2d8d210a9b..8fe6ac87d8 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -1515,10 +1515,17 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { } if background { + if wait { + // The spinner has no id on it, and a non-TTY stderr gets no spinner at all. + eprintln!( + "Waiting for {service_id}; Ctrl-C stops waiting, the server keeps running. \ + Follow: rocm logs --service {service_id} --follow" + ); + } let mut spinner = cli_progress::Spinner::new(format!("Starting {model} on {selected_engine}…")); spinner.tick(); - let report = start_managed_service( + let launched = start_managed_service( &selected_engine, &service_id, &model, @@ -1536,7 +1543,15 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { gpu_refusal_waived_for_reuse, wait, &mut |_elapsed| spinner.tick(), - )?; + ); + // `Spinner` has no `Drop`: clear its row so `Error:` is not glued to it. + let report = match launched { + Ok(report) => report, + Err(error) => { + spinner.clear(); + return Err(error); + } + }; // Not when the launch was fabricated. `simulate_oom_managed_launch` // writes a `starting` record for a process that never existed, so the // helper would be left watching a pid that will never appear, with diff --git a/apps/rocm/src/serve_summary.rs b/apps/rocm/src/serve_summary.rs index 2351644d2f..73db2a4b0d 100644 --- a/apps/rocm/src/serve_summary.rs +++ b/apps/rocm/src/serve_summary.rs @@ -176,7 +176,7 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { }; let _ = writeln!( out, - " note: {explanation} - check `rocm logs --service {}` or `rocm services list`", + " note: {explanation} - follow `rocm logs --service {} --follow` or check `rocm services list`", summary.service_id ); } diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 34bcd063e7..a15566fae3 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -880,6 +880,77 @@ fn simulate_oom_managed_launch( }) } +/// Probe `record`'s endpoint until it answers: once within the readiness +/// budget, or (with `wait_until_ready`) for as long as the server stays alive. +fn wait_for_record_ready( + paths: &AppPaths, + record: &ManagedServiceRecord, + wait_until_ready: bool, + endpoint_api_key: Option<&str>, + on_wait_tick: &mut dyn FnMut(Duration), +) -> EndpointReadiness { + let probe = |on_tick: &mut dyn FnMut(Duration)| { + wait_for_service_http_ready_with_progress( + &record.engine, + &record.host, + record.port, + &|| { + let mut latest = record.clone(); + let _ = latest.refresh_from_engine_state(); + latest.served_model_name().to_owned() + }, + endpoint_api_key, + READINESS_WAIT, + on_tick, + ) + }; + if wait_until_ready { + wait_past_readiness_budget( + || probe(&mut *on_wait_tick), + || service_still_live(paths, &record.service_id, service_pid_is_alive), + ) + } else { + probe(on_wait_tick) + } +} + +/// An equivalent service is already live. Without `--wait`, or once it is +/// serving, report it as is; otherwise wait for it like a fresh launch would. +fn wait_on_existing_service( + paths: &AppPaths, + mut report: ManagedLaunchReport, + wait_until_ready: bool, + on_wait_tick: &mut dyn FnMut(Duration), +) -> Result { + if !wait_until_ready || report.status == "ready" { + return Ok(report); + } + let Ok(mut record) = ManagedServiceRecord::read_manifest(paths, &report.service_id) else { + return Ok(report); + }; + // The key the caller generated belongs to a launch that did not happen; the + // existing service answers to its own. + let stored_key = endpoint_keys::endpoint_api_key(paths, &report.service_id); + let readiness = + wait_for_record_ready(paths, &record, true, stored_key.as_deref(), on_wait_tick); + if let Some(error) = died_during_wait( + paths, + &report.service_id, + true, + readiness, + service_pid_is_alive, + ) { + return Err(error); + } + report.status = status_for_readiness(readiness).to_owned(); + record.status.clone_from(&report.status); + if readiness == EndpointReadiness::Serving { + record.inference_verified_at_unix_ms = Some(rocm_core::unix_time_millis() as u64); + } + record.write()?; + Ok(report) +} + #[allow(clippy::too_many_arguments)] pub(crate) fn start_managed_service( engine: &str, @@ -933,7 +1004,12 @@ pub(crate) fn start_managed_service( require_api_key, gpu_refusal_waived_for_reuse, )? { - ManagedSpawn::AlreadyRunning(report) => return Ok(report), + ManagedSpawn::AlreadyRunning(report) => { + // `--wait` applies to a service that is still starting too: the + // natural retry after `rocm serve m` printed `starting`. + drop(launch_lock); + return wait_on_existing_service(&paths, report, wait_until_ready, on_wait_tick); + } ManagedSpawn::Spawned { record, child_pid } => (*record, child_pid), }; // The claiming service record is now persisted, so the selected GPU is @@ -945,29 +1021,13 @@ pub(crate) fn start_managed_service( #[cfg(windows)] thread::sleep(Duration::from_millis(200)); - let probe = |on_tick: &mut dyn FnMut(Duration)| { - wait_for_service_http_ready_with_progress( - engine, - host, - port, - &|| { - let mut latest = record.clone(); - let _ = latest.refresh_from_engine_state(); - latest.served_model_name().to_owned() - }, - endpoint_api_key, - READINESS_WAIT, - on_tick, - ) - }; - let readiness = if wait_until_ready { - wait_past_readiness_budget( - || probe(&mut *on_wait_tick), - || service_still_live(&paths, service_id, service_pid_is_alive), - ) - } else { - probe(on_wait_tick) - }; + let readiness = wait_for_record_ready( + &paths, + &record, + wait_until_ready, + endpoint_api_key, + on_wait_tick, + ); if let Some(error) = died_during_wait( &paths, service_id, @@ -1871,6 +1931,42 @@ mod tests { let _ = fs::remove_dir_all(root); } + #[test] + fn wait_on_existing_service_only_waits_for_one_that_is_not_serving_yet() { + // The probing path itself blocks for the readiness budget, so only the + // cases that must return at once are driven here; a manifest planted + // as `starting` would hang a wrong guard for 45 seconds and fail it. + let (root, paths) = test_paths("wait-existing"); + paths.ensure().expect("paths"); + let mut record = record_tracking_pid(&paths, 0, "starting"); + record.write().expect("write"); + let report = |status: &str| ManagedLaunchReport { + service_id: record.service_id.clone(), + endpoint_url: record.endpoint_url.clone(), + status: status.to_owned(), + already_running: true, + child_pid: None, + log_path: None, + manifest_path: None, + }; + let mut no_tick = |_: Duration| panic!("must not probe"); + + let untouched = wait_on_existing_service(&paths, report("starting"), false, &mut no_tick) + .expect("no --wait"); + assert_eq!(untouched.status, "starting"); + let ready = wait_on_existing_service(&paths, report("ready"), true, &mut no_tick) + .expect("already ready"); + assert_eq!(ready.status, "ready"); + record.status = "starting".to_owned(); + assert_eq!( + ManagedServiceRecord::read_manifest(&paths, &record.service_id) + .unwrap() + .status, + "starting" + ); + let _ = fs::remove_dir_all(root); + } + #[test] fn died_during_wait_fails_with_a_logs_pointer_and_records_the_death() { let (root, paths) = test_paths("wait-died"); diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index d0e4f2eafa..aae86c57c4 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -6,8 +6,10 @@ //! the internal sandbox-tool execution (stop/restart/list) behind the //! chat/LLM tool-calling path and the daemon-free managed-service lifecycle. //! -//! Mechanically relocated from `main.rs` with no behavior change — Phase 6f +//! Mostly relocated from `main.rs` with no behavior change — Phase 6f //! (ROCMAI-209) of the `main.rs` modularization (see `docs/architecture.md`). +//! The `--follow` tail (`LogTail`, `FollowFooter`, `follow_service_logs`) is new +//! code written here, not relocated. //! //! This module owns: the CLI/TUI logs browser (`render_logs_text`, //! `render_logs_browser_text`, `render_logs_browser_page_text`, @@ -623,7 +625,10 @@ impl LogTail { } if byte == b'\n' || byte == b'\r' { let mut redraw = byte == b'\r'; - let text = String::from_utf8_lossy(&self.pending[start..index]).into_owned(); + // The footer's row bookkeeping assumes the log text moves no cursor. + let text = rocm_core::terminal::strip_terminal_control_sequences( + &String::from_utf8_lossy(&self.pending[start..index]), + ); // `\r\n` is one line break, not a redraw followed by an empty line. if redraw && self.pending.get(index + 1) == Some(&b'\n') { redraw = false; @@ -684,12 +689,16 @@ struct FollowFooter { open_row: bool, } +/// Led by the part that says the server keeps running: service ids run 40+ +/// characters, so on a narrow terminal the end of the hint is what gets cut. +fn follow_hint(service_id: &str) -> String { + format!("Ctrl-C stops following, the server keeps running. Following {service_id}") +} + impl FollowFooter { fn new(service_id: &str) -> Self { Self { - text: format!( - "Following {service_id} - Ctrl-C stops following and leaves the server running." - ), + text: follow_hint(service_id), drawn: false, open_row: false, } @@ -718,7 +727,9 @@ impl FollowFooter { writeln!(out)?; } let shown: String = match cols { - Some(cols) => self.text.chars().take(cols.saturating_sub(1)).collect(), + Some(cols) => { + crate::cli_progress::truncate_to_width(&self.text, cols.saturating_sub(1)) + } None => self.text.clone(), }; write!(out, "\r\x1b[2K{shown}")?; @@ -769,6 +780,13 @@ pub(crate) fn follow_service_logs(paths: &AppPaths, service_id: &str) -> Result< ) } +/// True once, the first time `status` reads `ready`. +fn newly_ready(announced: &mut bool, status: &str) -> bool { + let ready = status == "ready" && !*announced; + *announced |= ready; + ready +} + fn follow_service_logs_with( paths: &AppPaths, service_id: &str, @@ -779,7 +797,9 @@ fn follow_service_logs_with( cols: &dyn Fn() -> Option, ) -> Result<()> { let record = load_managed_service(paths, service_id)?; - // Open the tail before the snapshot so output written in between is not lost. + // Open the tail before the snapshot: output written in between is then shown + // twice (in the snapshot and by the first poll) rather than lost, and the + // tail can start mid-line. let mut tail = LogTail::at_end(&record.log_path); write!(out, "{}", render_service_logs_text(paths, service_id)?)?; if !managed_service_is_live(&record) { @@ -795,13 +815,11 @@ fn follow_service_logs_with( let mut footer = tty.then(|| FollowFooter::new(service_id)); match footer.as_mut() { Some(footer) => footer.draw(out, false, cols())?, - None => writeln!( - out, - "\nFollowing {service_id} - Ctrl-C stops following and leaves the server running." - )?, + None => writeln!(out, "\n{}", follow_hint(service_id))?, } out.flush()?; + let mut announced_ready = record.status == "ready"; let mut since_check = Duration::ZERO; loop { thread::sleep(poll); @@ -817,6 +835,18 @@ fn follow_service_logs_with( } since_check = Duration::ZERO; let record = load_managed_service(paths, service_id)?; + if newly_ready(&mut announced_ready, &record.status) { + let line = [LogSegment { + text: format!("Service {service_id} is ready."), + redraw: false, + }]; + match footer.as_mut() { + Some(footer) => footer.write_batch(out, &line, cols())?, + None => { + write_log_segments(out, &line, tty)?; + } + } + } if !managed_service_is_live(&record) { // Output written between the last poll and the exit. let mut last = tail.poll(); @@ -835,6 +865,10 @@ fn follow_service_logs_with( "Service {service_id} is {}; stopped following.", record.status )?; + // Same outcome as `serve --wait`: a server that failed is not a success. + if record.status == "failed" { + bail!("service {service_id} failed; see the log output above"); + } return Ok(()); } } @@ -2396,9 +2430,28 @@ pub(crate) mod tests { fn follow_footer_is_cut_to_one_terminal_row() -> Result<()> { let mut footer = FollowFooter::new("svc_a"); let mut out = Vec::new(); - footer.draw(&mut out, false, Some(11))?; - // One column short of the width, so it never wraps. - assert_eq!(String::from_utf8(out)?, format!("{ERASE}Following ")); + footer.draw(&mut out, false, Some(30))?; + // One column short of the width, so it never wraps, and what is kept is + // the head of the hint: the part saying the server keeps running. + let drawn = String::from_utf8(out)?; + let shown = drawn.strip_prefix(ERASE).expect("erase prefix"); + assert_eq!(unicode_width::UnicodeWidthStr::width(shown), 29, "{shown}"); + assert!(shown.starts_with("Ctrl-C stops following, the"), "{shown}"); + Ok(()) + } + + #[test] + fn log_tail_strips_cursor_escapes_from_engine_output() -> Result<()> { + let (root, _paths) = test_paths("tail-escapes"); + fs::create_dir_all(&root)?; + let log = root.join("engine.log"); + fs::write(&log, "")?; + let mut tail = LogTail::at_end(&log); + fs::write(&log, "before\x1b[1Aafter\n")?; + let segments = tail.poll(); + assert_eq!(segments.len(), 1); + assert_eq!(segments[0].text, "beforeafter"); + let _ = fs::remove_dir_all(root); Ok(()) } @@ -2608,6 +2661,68 @@ pub(crate) mod tests { Ok(()) } + #[test] + fn newly_ready_fires_once_on_the_first_ready_status() { + let mut announced = false; + assert!(!newly_ready(&mut announced, "starting")); + assert!(newly_ready(&mut announced, "ready")); + assert!(!newly_ready(&mut announced, "ready")); + // A follow that began on an already-ready server has announced nothing new. + let mut announced = true; + assert!(!newly_ready(&mut announced, "ready")); + } + + #[test] + fn follow_service_logs_fails_when_the_service_failed() -> Result<()> { + let (root, paths) = test_paths("follow-ready-failed"); + paths.ensure()?; + let mut record = ManagedServiceRecord::new( + &paths, + "svc_follow_fail", + "vllm", + "qwen3.5", + "Qwen/Qwen3.5", + "127.0.0.1", + 11438, + "managed", + std::process::id(), + None, + None, + None, + ); + record.status = "starting".to_owned(); + record.write()?; + fs::write(&record.log_path, "before\n")?; + + let manifest_paths = paths.clone(); + let writer = thread::spawn(move || -> Result<()> { + thread::sleep(Duration::from_millis(200)); + let mut record = load_managed_service(&manifest_paths, "svc_follow_fail")?; + record.status = "failed".to_owned(); + record.write() + }); + let mut out = Vec::new(); + let result = follow_service_logs_with( + &paths, + "svc_follow_fail", + &mut out, + false, + Duration::from_millis(20), + Duration::from_millis(40), + &|| None, + ); + writer.join().expect("writer thread")?; + let out = String::from_utf8(out)?; + assert!(out.contains("is failed; stopped following"), "{out}"); + let error = result.expect_err("a failed service must not exit 0"); + assert!( + error.to_string().contains("svc_follow_fail failed"), + "{error}" + ); + let _ = fs::remove_dir_all(root); + Ok(()) + } + #[test] fn follow_service_logs_streams_new_output_until_the_service_stops() -> Result<()> { let (root, paths) = test_paths("follow-live"); diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 342dae703e..31c9625f55 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -403,7 +403,7 @@ rocm services rocm services list --all rocm services logs rocm services logs --follow -rocm serve --wait # a fresh launch: waits past 45s and returns once the server answers (an already-running model returns its current status at once) +rocm serve --wait # waits past 45s and returns once the server answers (also for a matching server still starting; an already-serving one returns at once) ``` Expected result: @@ -424,8 +424,9 @@ Expected result: - `rocm services logs --follow` prints the recent output, then new output as it arrives (a download bar redraws on one terminal row), and returns by itself with `Service is ; stopped following.` once the server - stops; Ctrl-C ends the follow and the server keeps running. - On a terminal, a `Following - Ctrl-C stops following ...` line stays on the + stops (exit non-zero if it ended `failed`), with a `Service is ready.` + line when it finishes starting; Ctrl-C ends the follow and the server keeps running. + On a terminal, a `Ctrl-C stops following, the server keeps running. Following ` line stays on the row directly under the newest output (and under the download bar) as the log grows; with output piped to a file it is printed once instead. - `rocm serve --wait` keeps waiting past 45 seconds and returns once the diff --git a/docs/testing.md b/docs/testing.md index 658bf623ac..a2b680c494 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1070,10 +1070,9 @@ rocm logs --service --follow # Ctrl-C to stop following While a first-time model download is running, `rocm serve --wait` keeps waiting past the usual 45 seconds until the server answers (or exits, which fails the -command), and -`--wait` with `--verbose` is refused (`serve-42`; `--foreground` is refused by the -argument parser, covered by a unit test). Run the -log follow above from a second terminal to watch the download. +command). `--wait` with `--verbose` is refused (`serve-42`); `--foreground` is refused +by the argument parser, covered by a unit test. Run the log follow above from a +second terminal to watch the download. The no-log first-run path should report `no CLI logs found yet` without creating the configured `data/logs` directory. From 125ba11c312ba59e651eb487f98afa17c8abdf5f Mon Sep 17 00:00:00 2001 From: juhovainio Date: Thu, 5 Nov 2026 14:50:54 +0200 Subject: [PATCH 09/12] ROCMAI-791: name the real service id in the --wait banner; strip escapes in follow finish The banner now comes from the launch once it knows which server it waits on, so attaching to an already-starting server prints that server's id and a valid logs hint. LogTail::finish strips terminal control sequences like poll does. Signed-off-by: juhovainio --- apps/rocm/src/serve_cmd.rs | 29 +++++++++++++++----------- apps/rocm/src/service_launch.rs | 36 ++++++++++++++++++++++++++++----- apps/rocm/src/services_text.rs | 20 +++++++++++++++++- docs/architecture.md | 2 +- 4 files changed, 68 insertions(+), 19 deletions(-) diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 8fe6ac87d8..b3467394aa 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -1515,16 +1515,10 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { } if background { - if wait { - // The spinner has no id on it, and a non-TTY stderr gets no spinner at all. - eprintln!( - "Waiting for {service_id}; Ctrl-C stops waiting, the server keeps running. \ - Follow: rocm logs --service {service_id} --follow" - ); - } - let mut spinner = - cli_progress::Spinner::new(format!("Starting {model} on {selected_engine}…")); - spinner.tick(); + let spinner = std::cell::RefCell::new(cli_progress::Spinner::new(format!( + "Starting {model} on {selected_engine}…" + ))); + spinner.borrow_mut().tick(); let launched = start_managed_service( &selected_engine, &service_id, @@ -1542,16 +1536,27 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { require_api_key, gpu_refusal_waived_for_reuse, wait, - &mut |_elapsed| spinner.tick(), + // Named with the id the launch resolved to: an already-starting + // server keeps its own, not the one generated above. The spinner + // has no id on it, and a non-TTY stderr gets no spinner at all. + &mut |waiting_on| { + spinner.borrow_mut().clear(); + eprintln!( + "Waiting for {waiting_on}; Ctrl-C stops waiting, the server keeps running. \ + Follow: rocm logs --service {waiting_on} --follow" + ); + }, + &mut |_elapsed| spinner.borrow_mut().tick(), ); // `Spinner` has no `Drop`: clear its row so `Error:` is not glued to it. let report = match launched { Ok(report) => report, Err(error) => { - spinner.clear(); + spinner.borrow_mut().clear(); return Err(error); } }; + let mut spinner = spinner.into_inner(); // Not when the launch was fabricated. `simulate_oom_managed_launch` // writes a `starting` record for a process that never existed, so the // helper would be left watching a pid that will never appear, with diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index a15566fae3..b6daabb487 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -920,6 +920,7 @@ fn wait_on_existing_service( paths: &AppPaths, mut report: ManagedLaunchReport, wait_until_ready: bool, + on_wait_begin: &mut dyn FnMut(&str), on_wait_tick: &mut dyn FnMut(Duration), ) -> Result { if !wait_until_ready || report.status == "ready" { @@ -928,6 +929,9 @@ fn wait_on_existing_service( let Ok(mut record) = ManagedServiceRecord::read_manifest(paths, &report.service_id) else { return Ok(report); }; + // The id of the server being waited on, which is not the caller's freshly + // generated one when it attached to an existing server. + on_wait_begin(&report.service_id); // The key the caller generated belongs to a launch that did not happen; the // existing service answers to its own. let stored_key = endpoint_keys::endpoint_api_key(paths, &report.service_id); @@ -969,6 +973,7 @@ pub(crate) fn start_managed_service( require_api_key: bool, gpu_refusal_waived_for_reuse: bool, wait_until_ready: bool, + on_wait_begin: &mut dyn FnMut(&str), on_wait_tick: &mut dyn FnMut(Duration), ) -> Result { let paths = AppPaths::discover()?; @@ -1008,7 +1013,13 @@ pub(crate) fn start_managed_service( // `--wait` applies to a service that is still starting too: the // natural retry after `rocm serve m` printed `starting`. drop(launch_lock); - return wait_on_existing_service(&paths, report, wait_until_ready, on_wait_tick); + return wait_on_existing_service( + &paths, + report, + wait_until_ready, + on_wait_begin, + on_wait_tick, + ); } ManagedSpawn::Spawned { record, child_pid } => (*record, child_pid), }; @@ -1021,6 +1032,9 @@ pub(crate) fn start_managed_service( #[cfg(windows)] thread::sleep(Duration::from_millis(200)); + if wait_until_ready { + on_wait_begin(service_id); + } let readiness = wait_for_record_ready( &paths, &record, @@ -1951,11 +1965,23 @@ mod tests { }; let mut no_tick = |_: Duration| panic!("must not probe"); - let untouched = wait_on_existing_service(&paths, report("starting"), false, &mut no_tick) - .expect("no --wait"); + let untouched = wait_on_existing_service( + &paths, + report("starting"), + false, + &mut |_| panic!("no --wait, no banner"), + &mut no_tick, + ) + .expect("no --wait"); assert_eq!(untouched.status, "starting"); - let ready = wait_on_existing_service(&paths, report("ready"), true, &mut no_tick) - .expect("already ready"); + let ready = wait_on_existing_service( + &paths, + report("ready"), + true, + &mut |_| panic!("already ready, no banner"), + &mut no_tick, + ) + .expect("already ready"); assert_eq!(ready.status, "ready"); record.status = "starting".to_owned(); assert_eq!( diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index aae86c57c4..1377639224 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -650,8 +650,11 @@ impl LogTail { if bytes.last() == Some(&b'\r') { bytes.pop(); } + // Same stripping as `poll`: a cut-off write can end mid-escape. (!bytes.is_empty()).then(|| LogSegment { - text: String::from_utf8_lossy(&bytes).into_owned(), + text: rocm_core::terminal::strip_terminal_control_sequences(&String::from_utf8_lossy( + &bytes, + )), redraw: false, }) } @@ -2455,6 +2458,21 @@ pub(crate) mod tests { Ok(()) } + #[test] + fn log_tail_finish_strips_cursor_escapes_from_a_cut_off_write() -> Result<()> { + let (root, _paths) = test_paths("tail-finish-escapes"); + fs::create_dir_all(&root)?; + let log = root.join("engine.log"); + fs::write(&log, "")?; + let mut tail = LogTail::at_end(&log); + fs::write(&log, "last\x1b[1Aword")?; + assert!(tail.poll().is_empty()); + let segment = tail.finish().expect("unterminated output is returned"); + assert_eq!(segment.text, "lastword"); + let _ = fs::remove_dir_all(root); + Ok(()) + } + #[test] fn follow_service_logs_keeps_the_hint_under_new_output_on_a_tty() -> Result<()> { let (root, paths) = test_paths("follow-tty"); diff --git a/docs/architecture.md b/docs/architecture.md index 2570bddc44..f243215fe8 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -76,7 +76,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`service_launch.rs`](../apps/rocm/src/service_launch.rs) — mechanical relocation, owns types (Phase 6b, ROCMAI-91): spawning/attaching to a managed service process and streaming its log, plus the endpoint-key/auth checks that gate a public bind — `AttachOutcome`/`RawModeGuard`/`ManagedSpawn`/`AutostartClaim` are its private types; `ManagedLaunchReport` is `pub(crate)`, since [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) reads its fields directly for the interactive summary. Most of this module's functions are `pub(crate)` for the same reason — other modules, including [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`automations.rs`](../apps/rocm/src/automations.rs), and [`services_text.rs`](../apps/rocm/src/services_text.rs), call into it; `main.rs` does not. `ServicesCommand` stayed in [`main.rs`](../apps/rocm/src/main.rs) rather than moving alongside, and `services()` lives in [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs); the record removal and pruning it dispatches to now lives in [`service_records.rs`](../apps/rocm/src/service_records.rs), and only stop/restart are also reachable from the chat/LLM sandbox tool-calling path, out of scope for Phase 6b. - [`service_records.rs`](../apps/rocm/src/service_records.rs) — mechanical relocation, owns types: removing one managed-service record and every file it owns (`remove_managed_service_record`, `service_record_artifacts`) and the `rocm services prune` plan/render/apply steps (`build_service_prune_plan`, `render_service_prune_plan`, `apply_service_prune_plan`, `prune_managed_service_records`) with the age gate behind them (`DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS`, `service_prune_min_age_hours`). It reads record state through [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs) and [`services_text.rs`](../apps/rocm/src/services_text.rs) and builds its advice text with [`service_hints.rs`](../apps/rocm/src/service_hints.rs); none of the three depends on it. `remove_managed_service_record`, `prune_managed_service_records`, `service_prune_min_age_hours` and `DEFAULT_SERVICE_PRUNE_MIN_AGE_HOURS` are `pub(crate)` (`services()` in [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs) calls the first three; the `ServicesCommand::Prune` clap definition in [`main.rs`](../apps/rocm/src/main.rs) uses the constant, which `main.rs` re-imports), and `ServicePruneOutcome`, the return type of `prune_managed_service_records`, is `pub(crate)` with `pub(crate)` `text`, `removed_records`, `removed_files` and `failures` fields that `services()` reads. Every other item is private. The cluster's regression tests live in this module's own `mod tests`, reaching the shared `test_paths`, `plant_service_record` and `plant_service_record_without_its_key` fixtures through `crate::tests::` (the last two widened to `pub(crate)` for that). - [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm services`' command handler (`services()`, which dispatches `list`, `logs`, `stop`, `restart`, `remove` and `prune`) and `run_approved_service_action`, the approval-gated stop/restart path. `ServicesCommand` stays in [`main.rs`](../apps/rocm/src/main.rs). Both functions are `pub(crate)`: `dispatch()` calls `services()` through a re-import, and [`advised_commands.rs`](../apps/rocm/src/advised_commands.rs)'s test reaches `run_approved_service_action` through a test-only re-import in `main.rs`. The module reads from [`services_text.rs`](../apps/rocm/src/services_text.rs), [`service_records.rs`](../apps/rocm/src/service_records.rs), [`service_hints.rs`](../apps/rocm/src/service_hints.rs) and [`cli_audit.rs`](../apps/rocm/src/cli_audit.rs); none of them calls back into it. The cluster's one regression test lives in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::`. -- [`services_text.rs`](../apps/rocm/src/services_text.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-209): the CLI/TUI logs browser (`render_logs_text`, `render_logs_browser_text`, `render_logs_browser_page_text`, `render_logs_browser_page_text_with_options`, `logs_browser_total_pages`, the `LogBrowserEntry` type, `collect_log_browser_entries`, `filter_log_browser_entries`, `log_browser_source_label`, `write_past_attempts_hint`), the `rocm services`/`rocm services logs` status surfaces (`render_services_text`, `render_services_json`, `render_service_logs_text`, `render_service_logs_text_with_options`, `render_service_action_result`, `read_managed_service_manifest`, `load_managed_service`), the `--follow` tail behind `rocm logs --service --follow` and `rocm services logs --follow` (`follow_service_logs`, `follow_service_logs_with`, the `LogTail` and `LogSegment` types, `write_log_segments`), the internal `rocmd`-compatible status/sandbox surface (`render_internal_status_text`, `run_internal_sandbox_tool`), and the managed-service stop/restart implementation behind both the sandbox tool and `rocm services stop|restart` (`terminate_recorded_service_pids`, the `MANAGED_STOP_GRACE` constant, `stop_internal_managed_service`, `stop_internal_managed_service_with`, `stop_internal_managed_service_confirmed_with`, `unload_lemonade_service_model`, `apply_restart_runtime_pin`, `load_service_for_restart`, `restart_child_argv`, `restart_internal_managed_service`, `restart_internal_managed_service_with`, `update_restarting_record`, `record_restarted_child`, `record_restart_readiness`), plus `validate_service_id`, `list_log_files`, `read_optional_tail_lines`, and the CLI-lifecycle-log tail formatting helpers (`format_cli_lifecycle_tail_line`, `lifecycle_event_label`, `humanize_log_label`, `lifecycle_field`, `read_tail_lines`). Two items physically interleaved in this span in `main.rs` moved with it rather than staying behind, since neither has a caller outside the cluster: the `LogBrowserEntry` struct and the `MANAGED_STOP_GRACE` constant; both stay private. `dispatch()` calls directly into this module at several points — `render_internal_status_text`, `run_internal_sandbox_tool`, `render_service_logs_text`, `render_logs_browser_text`, and `render_logs_text` — all re-imported via `use crate::services_text::{...};`; [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs)'s `services()` calls `render_services_text`, `render_services_json`, `render_service_logs_text` and `follow_service_logs` through its own imports, and `dispatch()` calls `follow_service_logs` for `rocm logs --service --follow`. `render_service_action_result`, `run_internal_sandbox_tool`, and `validate_service_id` are also `pub(crate)` and read from `services_cmd.rs`'s `run_approved_service_action`; `validate_service_id` is also read from [`service_records.rs`](../apps/rocm/src/service_records.rs)'s `remove_managed_service_record`/`service_record_artifacts`, and `load_managed_service` from its `remove_managed_service_record`/`apply_service_prune_plan`. `load_managed_service` and `restart_internal_managed_service` are `pub(crate)`, read from [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) (reconciling live servers against a runtime activation) and from [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs)'s own test module; `read_tail_lines`, `render_logs_text`, `render_logs_browser_text`, `render_services_text`, `render_service_logs_text`, and `validate_service_id` are also `pub(crate)`, read from [`chat.rs`](../apps/rocm/src/chat.rs)'s deterministic tool-output summaries; `read_optional_tail_lines` and `render_service_action_result` are also `pub(crate)`, read from [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s managed-spawn/attach path. Every other item in this list has no caller outside this module and stays private, including `restart_internal_managed_service_with`, `restart_child_argv`, `load_service_for_restart`, `apply_restart_runtime_pin`, and `unload_lemonade_service_model` — see [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs)'s and [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s entries above for how those two modules' own doc comments now describe this module, rather than `main.rs`, as that caller. The cluster's regression tests live in this module's own `mod tests`, reaching shared fixtures (`test_paths`, `plant_service_record_on_runtime`, `OLD_RUNTIME_KEY`, `NEW_RUNTIME_KEY`) through `crate::tests::` the same way other mechanically relocated modules do, and `runtime_services::` through its own `use crate::runtime_services;` for the handful of tests that drive this cluster's restart/stop behavior through `runtime_services`'s public entry points rather than calling it directly. Two tests that call this cluster's `load_managed_service` only incidentally — their primary subject is `runtime_services::pin_service_record_to_runtime`/`restore_service_record_pin` and `runtime_services::restart_service_onto_runtime_with`'s outcome classification respectively — stay in `main.rs`'s shared `mod tests` instead. +- [`services_text.rs`](../apps/rocm/src/services_text.rs) — mechanical relocation, owns private types (Phase 6f, ROCMAI-209): the CLI/TUI logs browser (`render_logs_text`, `render_logs_browser_text`, `render_logs_browser_page_text`, `render_logs_browser_page_text_with_options`, `logs_browser_total_pages`, the `LogBrowserEntry` type, `collect_log_browser_entries`, `filter_log_browser_entries`, `log_browser_source_label`, `write_past_attempts_hint`), the `rocm services`/`rocm services logs` status surfaces (`render_services_text`, `render_services_json`, `render_service_logs_text`, `render_service_logs_text_with_options`, `render_service_action_result`, `read_managed_service_manifest`, `load_managed_service`), the `--follow` tail behind `rocm logs --service --follow` and `rocm services logs --follow` (`follow_service_logs`, `follow_service_logs_with`, the `LogTail`, `LogSegment` and `FollowFooter` types, `write_log_segments`, `newly_ready`; new code written in this module, not relocated), the internal `rocmd`-compatible status/sandbox surface (`render_internal_status_text`, `run_internal_sandbox_tool`), and the managed-service stop/restart implementation behind both the sandbox tool and `rocm services stop|restart` (`terminate_recorded_service_pids`, the `MANAGED_STOP_GRACE` constant, `stop_internal_managed_service`, `stop_internal_managed_service_with`, `stop_internal_managed_service_confirmed_with`, `unload_lemonade_service_model`, `apply_restart_runtime_pin`, `load_service_for_restart`, `restart_child_argv`, `restart_internal_managed_service`, `restart_internal_managed_service_with`, `update_restarting_record`, `record_restarted_child`, `record_restart_readiness`), plus `validate_service_id`, `list_log_files`, `read_optional_tail_lines`, and the CLI-lifecycle-log tail formatting helpers (`format_cli_lifecycle_tail_line`, `lifecycle_event_label`, `humanize_log_label`, `lifecycle_field`, `read_tail_lines`). Two items physically interleaved in this span in `main.rs` moved with it rather than staying behind, since neither has a caller outside the cluster: the `LogBrowserEntry` struct and the `MANAGED_STOP_GRACE` constant; both stay private. `dispatch()` calls directly into this module at several points — `render_internal_status_text`, `run_internal_sandbox_tool`, `render_service_logs_text`, `render_logs_browser_text`, and `render_logs_text` — all re-imported via `use crate::services_text::{...};`; [`services_cmd.rs`](../apps/rocm/src/services_cmd.rs)'s `services()` calls `render_services_text`, `render_services_json`, `render_service_logs_text` and `follow_service_logs` through its own imports, and `dispatch()` calls `follow_service_logs` for `rocm logs --service --follow`. `render_service_action_result`, `run_internal_sandbox_tool`, and `validate_service_id` are also `pub(crate)` and read from `services_cmd.rs`'s `run_approved_service_action`; `validate_service_id` is also read from [`service_records.rs`](../apps/rocm/src/service_records.rs)'s `remove_managed_service_record`/`service_record_artifacts`, and `load_managed_service` from its `remove_managed_service_record`/`apply_service_prune_plan`. `load_managed_service` and `restart_internal_managed_service` are `pub(crate)`, read from [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) (reconciling live servers against a runtime activation) and from [`managed_service_status.rs`](../apps/rocm/src/managed_service_status.rs)'s own test module; `read_tail_lines`, `render_logs_text`, `render_logs_browser_text`, `render_services_text`, `render_service_logs_text`, and `validate_service_id` are also `pub(crate)`, read from [`chat.rs`](../apps/rocm/src/chat.rs)'s deterministic tool-output summaries; `read_optional_tail_lines` and `render_service_action_result` are also `pub(crate)`, read from [`service_launch.rs`](../apps/rocm/src/service_launch.rs)'s managed-spawn/attach path. Every other item in this list has no caller outside this module and stays private, including `restart_internal_managed_service_with`, `restart_child_argv`, `load_service_for_restart`, `apply_restart_runtime_pin`, and `unload_lemonade_service_model` — see [`engine_runtime_select.rs`](../apps/rocm/src/engine_runtime_select.rs)'s and [`gpu_policy.rs`](../apps/rocm/src/gpu_policy.rs)'s entries above for how those two modules' own doc comments now describe this module, rather than `main.rs`, as that caller. The cluster's regression tests live in this module's own `mod tests`, reaching shared fixtures (`test_paths`, `plant_service_record_on_runtime`, `OLD_RUNTIME_KEY`, `NEW_RUNTIME_KEY`) through `crate::tests::` the same way other mechanically relocated modules do, and `runtime_services::` through its own `use crate::runtime_services;` for the handful of tests that drive this cluster's restart/stop behavior through `runtime_services`'s public entry points rather than calling it directly. Two tests that call this cluster's `load_managed_service` only incidentally — their primary subject is `runtime_services::pin_service_record_to_runtime`/`restore_service_record_pin` and `runtime_services::restart_service_onto_runtime_with`'s outcome classification respectively — stay in `main.rs`'s shared `mod tests` instead. - [`setup_cmd.rs`](../apps/rocm/src/setup_cmd.rs) — mechanical relocation, dispatch-adjacent, no owned types: `rocm setup`'s command handler (`setup()`, which runs `setup status` by default and `setup reset`), the text `rocm setup status` prints (`render_setup_status_text`) and the reset that clears the saved first-time prompt state (`reset_setup_prompt_state`). `SetupCommand` stays in [`main.rs`](../apps/rocm/src/main.rs). `setup()` and `render_setup_status_text` are `pub(crate)` and re-imported in `main.rs`: `dispatch()` calls the first, and [`chat.rs`](../apps/rocm/src/chat.rs)'s read-only handling of `rocm setup` and `rocm setup status` reaches the second as `crate::render_setup_status_text`; `reset_setup_prompt_state` is private. The three regression tests live in this module's own `mod tests`, reaching the shared `test_paths` fixture through `crate::tests::` and `write_test_pip_runtime` through `crate::test_support::`. - [`storage.rs`](../apps/rocm/src/storage.rs) — full domain extraction. - [`system_packages.rs`](../apps/rocm/src/system_packages.rs) — full domain extraction, a leaf of the crate (split out of [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs)): the host system-package installs that `rocm install sdk` and `rocm engines install vllm` run through `sudo` apt, dnf or zypper. It owns the distro-aware OpenMPI, libatomic and libnuma flows (`ensure_openmpi_for_vllm`, `ensure_libatomic_for_torch`, `ensure_libnuma_for_torch`), the choice between printing the commands and running them (`SystemPackageInstallAction`, `system_package_install_action`), the guard that refuses an `apt-get install` which would remove ROCm packages, and `real_os_release`, the real host's os-release that every one of these plans is built from. Production code uses only `std`, `anyhow` and `rocm_core`; the three `ensure_*` functions and `real_os_release` are `pub(crate)` and the rest is private. `runtime_install_finalize.rs` re-exports the three `ensure_*` names for [`install_cmd.rs`](../apps/rocm/src/install_cmd.rs) and [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs), and [`driver_install.rs`](../apps/rocm/src/driver_install.rs) imports `real_os_release` from here directly. `SdkInstallConsents` stays in `runtime_install_finalize.rs` (it carries a `therock::SdkInstallConsent`); the functions here take its plain `system_packages` bool. `torch_runtime_dep_checks_disabled` is the E2E hook that skips the torch dependency checks; it only does anything under `e2e-test-hooks`. Its tests cover the consent decision (reaching `consents_for_install_sdk_argv` in `runtime_install_finalize.rs`'s test module) and, under `e2e-test-hooks` on Linux, that `real_os_release` ignores a simulated host root. From d05bc0758ea965e7d5f27fc4beeb7c7be71b2382 Mon Sep 17 00:00:00 2001 From: juhovainio Date: Thu, 5 Nov 2026 15:47:14 +0200 Subject: [PATCH 10/12] ROCMAI-791: say --wait ends when the engine gives up, not only when the server answers The help, README and test docs promised to wait however long a download takes, but vLLM and AIM stop after 5 minutes by default. Name the cap and its override, and mention it in the died-before-ready error. Signed-off-by: juhovainio --- README.md | 7 ++++--- apps/rocm/src/main.rs | 6 ++++-- apps/rocm/src/service_launch.rs | 2 +- docs/manual-testing.md | 5 +++-- docs/testing.md | 5 +++-- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index b7c7c908c0..170fed4438 100644 --- a/README.md +++ b/README.md @@ -912,9 +912,10 @@ can: directly and exits 0; if that signal fails too, it says so on stderr. `--managed` is the explicit form of the default background behavior. -`--wait` keeps `rocm serve` waiting until the server answers requests, however -long a first-time model download takes, instead of returning `starting` after 45 -seconds; if the server exits first (crashes, is killed, or is stopped with +`--wait` keeps `rocm serve` waiting until the server answers requests or its +engine gives up (vLLM and AIM after 5 minutes by default; raise it with +`ROCM_CLI_VLLM_READY_TIMEOUT_SECS` / `ROCM_CLI_AIM_READY_TIMEOUT_SECS`), instead +of returning `starting` after 45 seconds; if the server exits first (crashes, is killed, or is stopped with `rocm services stop`), it fails with an error that points at the server's logs instead of waiting forever. Ctrl-C ends the wait and leaves the server starting in the background. It cannot be combined with `--foreground` or `--verbose`, and diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 09165bce3c..6939cf55f3 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -607,8 +607,10 @@ rocm serve qwen --verbose --device gpu_required")] /// Keep the server managed by ROCm CLI. #[arg(long)] managed: bool, - /// Keep waiting until the server answers requests, however long a first-time - /// model download takes, instead of returning `starting` after 45 seconds. + /// Keep waiting until the server answers requests or its engine gives up + /// (vLLM and AIM after 5 minutes by default, raised with + /// `ROCM_CLI_VLLM_READY_TIMEOUT_SECS` / `ROCM_CLI_AIM_READY_TIMEOUT_SECS`), + /// instead of returning `starting` after 45 seconds. /// Also waits on a matching server that is already starting. Fails with a /// pointer to the logs if the server exits first. Cannot be /// combined with --foreground or --verbose, which already stay attached. diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index b6daabb487..cc0b2b4040 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -774,7 +774,7 @@ fn died_during_wait( } settle_exited_service(paths, service_id); Some(anyhow!( - "the server exited before it became ready; see `rocm logs --service {service_id}` for its output" + "the server exited before it became ready (vLLM and AIM give up after 5 minutes by default; `ROCM_CLI_VLLM_READY_TIMEOUT_SECS` / `ROCM_CLI_AIM_READY_TIMEOUT_SECS` allow longer); see `rocm logs --service {service_id}` for its output" )) } diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 31c9625f55..42176864b2 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -403,7 +403,7 @@ rocm services rocm services list --all rocm services logs rocm services logs --follow -rocm serve --wait # waits past 45s and returns once the server answers (also for a matching server still starting; an already-serving one returns at once) +rocm serve --wait # waits past 45s and returns once the server answers or its engine gives up (also for a matching server still starting; an already-serving one returns at once) ``` Expected result: @@ -430,7 +430,8 @@ Expected result: row directly under the newest output (and under the download bar) as the log grows; with output piped to a file it is printed once instead. - `rocm serve --wait` keeps waiting past 45 seconds and returns once the - server answers. If the server is killed or stopped meanwhile, it exits non-zero + server answers or its engine gives up (vLLM and AIM: 5 minutes by default, override + with `ROCM_CLI_VLLM_READY_TIMEOUT_SECS` / `ROCM_CLI_AIM_READY_TIMEOUT_SECS`). If the server is killed or stopped meanwhile, it exits non-zero with `the server exited before it became ready` and a `rocm logs --service` pointer, and the record shows `stopped`. - The logs command shows the exact service failure or startup output. diff --git a/docs/testing.md b/docs/testing.md index a2b680c494..9e6048f759 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -1069,8 +1069,9 @@ rocm logs --service --follow # Ctrl-C to stop following ``` While a first-time model download is running, `rocm serve --wait` keeps -waiting past the usual 45 seconds until the server answers (or exits, which fails the -command). `--wait` with `--verbose` is refused (`serve-42`); `--foreground` is refused +waiting past the usual 45 seconds until the server answers or its engine gives up +(vLLM and AIM stop after 5 minutes by default, see `ROCM_CLI_VLLM_READY_TIMEOUT_SECS` +and `ROCM_CLI_AIM_READY_TIMEOUT_SECS`); either way an exit fails the command. `--wait` with `--verbose` is refused (`serve-42`); `--foreground` is refused by the argument parser, covered by a unit test. Run the log follow above from a second terminal to watch the download. From 2aeeea470497b78e5d52fd462df155ca2e89b5f0 Mon Sep 17 00:00:00 2001 From: juhovainio Date: Thu, 5 Nov 2026 16:20:39 +0200 Subject: [PATCH 11/12] ROCMAI-791: follow/wait review items: key before wait, fresh record write, piped notices on stderr - serve --wait prints a generated API key when the wait begins, not only after it ends. - the post-wait record write re-reads the manifest and changes only status and the verification latch; a --wait death adds the vLLM OOM guidance a plain serve gives. - follow: notices (hint, ready, stopped) go to stderr when piped; the footer and an open download-bar row are ended before the ready line and on error; an already failed server exits non-zero. New unit tests and scenario logs-07. Signed-off-by: juhovainio --- README.md | 9 +- apps/rocm/src/serve_cmd.rs | 20 +- apps/rocm/src/service_launch.rs | 155 ++++++++- apps/rocm/src/services_text.rs | 377 +++++++++++++++++++-- docs/manual-testing.md | 4 +- tests/e2e-cucumber/features/logs.feature | 10 + tests/e2e-cucumber/tests/e2e/logs_steps.rs | 44 ++- 7 files changed, 573 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index 170fed4438..ed685b71a8 100644 --- a/README.md +++ b/README.md @@ -918,7 +918,8 @@ engine gives up (vLLM and AIM after 5 minutes by default; raise it with of returning `starting` after 45 seconds; if the server exits first (crashes, is killed, or is stopped with `rocm services stop`), it fails with an error that points at the server's logs instead of waiting forever. Ctrl-C ends the wait and leaves the server starting -in the background. It cannot be combined with `--foreground` or `--verbose`, and +in the background; a generated API key (public bind) is printed when the wait +begins, so interrupting the wait does not lose it. It cannot be combined with `--foreground` or `--verbose`, and when the model is already being served and still starting, it waits on that server instead (an already-serving one is reported at once). Watch the download with `rocm logs --service --follow` from another terminal. @@ -1405,8 +1406,10 @@ rocm uninstall [--yes] [--dry-run] download's progress, until the server stops (a server that ends `failed` makes the command exit non-zero), and prints `Service is ready.` when it finishes starting. Ctrl-C ends the follow and leaves the server running. On a terminal, a `Ctrl-C stops following, the server keeps running. Following ` -line stays on the row under the newest output; piped or redirected output prints -it once. +line stays on the row under the newest output. Piped or redirected output keeps +stdout to the log text alone: the hint (once), `is ready.` and `stopped following.` +lines go to stderr. A server that is already `failed` when the follow starts exits +non-zero too, after printing its log. `rocm uninstall` removes four categories of files that ROCm CLI manages on this computer: its binaries, configuration, data, and caches. It lists what it will diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index b3467394aa..ce799d9b2e 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -71,6 +71,7 @@ use crate::service_launch::{ ensure_background_helper_running_quiet, ensure_public_bind_engine_supported, managed_service_reuse_refusal, no_usable_gpu_refusal, print_managed_launch_plain, resolve_endpoint_auth, run_attached_service, start_managed_service, validate_bind_host, + wait_begin_client_config, }; use crate::{ GatedEngineAction, cli_progress, device_policy_name, ensure_engine_allowed_on_os, @@ -1519,6 +1520,10 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { "Starting {model} on {selected_engine}…" ))); spinner.borrow_mut().tick(); + // The key is shown when the wait begins, since Ctrl-C or a failed wait + // would otherwise leave a running authenticated server whose key was never + // printed. It is then not repeated in the summary below. + let key_shown_before_wait = std::cell::Cell::new(false); let launched = start_managed_service( &selected_engine, &service_id, @@ -1545,6 +1550,19 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { "Waiting for {waiting_on}; Ctrl-C stops waiting, the server keeps running. \ Follow: rocm logs --service {waiting_on} --follow" ); + if let Some(config) = wait_begin_client_config( + waiting_on, + &service_id, + &host, + port, + endpoint_auth.as_deref(), + ) { + key_shown_before_wait.set(true); + // Same intentional one-time key display as + // `print_managed_launch_plain`; see its rationale. + // codeql[rust/cleartext-logging] + print!("{config}"); + } }, &mut |_elapsed| spinner.borrow_mut().tick(), ); @@ -1591,7 +1609,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { } // Safe to move `endpoint_auth` here: this branch always returns, so the // fall-through (attached) path below never observes it moved. - let launched_key = if report.already_running { + let launched_key = if report.already_running || key_shown_before_wait.get() { None } else { endpoint_auth diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index cc0b2b4040..0519678d68 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -773,9 +773,48 @@ fn died_during_wait( return None; } settle_exited_service(paths, service_id); - Some(anyhow!( + let mut message = format!( "the server exited before it became ready (vLLM and AIM give up after 5 minutes by default; `ROCM_CLI_VLLM_READY_TIMEOUT_SECS` / `ROCM_CLI_AIM_READY_TIMEOUT_SECS` allow longer); see `rocm logs --service {service_id}` for its output" - )) + ); + // A plain `serve` adds the memory guidance for a vLLM launch that ran out of + // VRAM; the wait's failure is the same launch, so it gets the same note. + // "starting" is the status the note keys on for "never became ready", which is + // what a `--wait` death is, whatever the settled record now says. + if let Ok(record) = ManagedServiceRecord::read_manifest(paths, service_id) { + for note in crate::serve_summary::append_oom_serve_note( + Vec::new(), + &record.engine, + false, + "starting", + Some(&record.log_path), + ) { + let _ = write!(message, "\n note: {note}"); + } + } + Some(anyhow!(message)) +} + +/// Record the outcome of a finished wait on the manifest as it is now, changing +/// only what the wait learned. The copy read at spawn time can be long stale: +/// `rocmd` may have recorded a restart or a new supervisor pid, and `rocm logs` +/// refreshes the record, so writing it back whole would revert those. +fn record_wait_outcome( + paths: &AppPaths, + held: &mut ManagedServiceRecord, + status: &str, + verified: bool, +) -> Result<()> { + let mut latest = ManagedServiceRecord::read_manifest(paths, &held.service_id) + .unwrap_or_else(|_| held.clone()); + status.clone_into(&mut latest.status); + if verified { + // Latch the verification the wait just performed, so the readiness checks + // behind `services list` and chat read it instead of re-probing. + latest.inference_verified_at_unix_ms = Some(rocm_core::unix_time_millis() as u64); + } + latest.write()?; + *held = latest; + Ok(()) } /// Test-only fault-injection switch, off (`const false`) in shipped binaries: @@ -947,11 +986,12 @@ fn wait_on_existing_service( return Err(error); } report.status = status_for_readiness(readiness).to_owned(); - record.status.clone_from(&report.status); - if readiness == EndpointReadiness::Serving { - record.inference_verified_at_unix_ms = Some(rocm_core::unix_time_millis() as u64); - } - record.write()?; + record_wait_outcome( + paths, + &mut record, + &report.status, + readiness == EndpointReadiness::Serving, + )?; Ok(report) } @@ -1052,13 +1092,12 @@ pub(crate) fn start_managed_service( return Err(error); } let launch_status = status_for_readiness(readiness); - record.status = launch_status.to_owned(); - if readiness == EndpointReadiness::Serving { - // Latch the verification the wait just performed, so the readiness checks - // behind `services list` and chat read it instead of re-probing. - record.inference_verified_at_unix_ms = Some(rocm_core::unix_time_millis() as u64); - } - record.write()?; + record_wait_outcome( + &paths, + &mut record, + launch_status, + readiness == EndpointReadiness::Serving, + )?; let endpoint_url = format!("{}/v1", format_http_base_url(host, port)); record_cli_audit_event( &paths, @@ -1126,6 +1165,23 @@ pub(crate) fn print_managed_launch_plain( println!(" readiness: {}", report.status); } +/// The client configuration to show when a `--wait` begins, so a key generated for +/// a public bind reaches the user even if the wait is interrupted or fails. +/// `None` unless the wait is on the launch that key belongs to: an attach to an +/// already-starting server (`waiting_on` differs from `service_id`) answers to +/// its own key, which is not shown here. +pub(crate) fn wait_begin_client_config( + waiting_on: &str, + service_id: &str, + host: &str, + port: u16, + api_key: Option<&str>, +) -> Option { + let key = api_key.filter(|_| waiting_on == service_id)?; + let endpoint_url = format!("{}/v1", format_http_base_url(host, port)); + Some(render_endpoint_client_config(&endpoint_url, key)) +} + /// Render the one-time secure client configuration for a public, authenticated /// endpoint. This is the *intended* channel for delivering the key to the user /// (unlike logs/status, which must never contain it) — it prints the key once at @@ -2030,6 +2086,77 @@ mod tests { let _ = fs::remove_dir_all(root); } + #[test] + fn died_during_wait_adds_the_oom_guidance_for_a_vllm_launch_only() { + let (root, paths) = test_paths("wait-died-oom"); + paths.ensure().expect("paths"); + let mut record = record_tracking_pid(&paths, std::process::id(), "starting"); + record.engine = "vllm".to_owned(); + record.write().expect("write"); + fs::write( + &record.log_path, + "torch.OutOfMemoryError: HIP out of memory. Tried to allocate 7.21 GiB.\n", + ) + .expect("log"); + let id = record.service_id; + let pending = EndpointReadiness::Unreachable; + let message = died_during_wait(&paths, &id, true, pending, |_| false) + .expect("died") + .to_string(); + assert!(message.contains("--gpu-memory-utilization"), "{message}"); + + // Same log, another engine: that engine has no such flag to suggest. + let mut record = ManagedServiceRecord::read_manifest(&paths, &id).unwrap(); + record.engine = "lemonade".to_owned(); + "starting".clone_into(&mut record.status); + record.write().expect("write"); + let message = died_during_wait(&paths, &id, true, pending, |_| false) + .expect("died") + .to_string(); + assert!(!message.contains("--gpu-memory-utilization"), "{message}"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn record_wait_outcome_keeps_what_changed_on_disk_during_the_wait() { + let (root, paths) = test_paths("wait-outcome"); + paths.ensure().expect("paths"); + let mut held = record_tracking_pid(&paths, 111, "starting"); + held.write().expect("write"); + // While the wait ran, `rocmd` recorded a restart and a new supervisor. + let mut elsewhere = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); + elsewhere.restart_count = 3; + elsewhere.supervisor_pid = 222; + elsewhere.write().expect("write"); + + record_wait_outcome(&paths, &mut held, "ready", true).expect("outcome"); + let after = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); + assert_eq!(after.status, "ready"); + assert!(after.inference_verified_at_unix_ms.is_some()); + assert_eq!(after.restart_count, 3, "the restart count was reverted"); + assert_eq!(after.supervisor_pid, 222, "the supervisor pid was reverted"); + + record_wait_outcome(&paths, &mut held, "starting", false).expect("outcome"); + let after = ManagedServiceRecord::read_manifest(&paths, &held.service_id).unwrap(); + assert_eq!(after.status, "starting"); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn wait_begin_shows_the_key_only_for_the_launch_it_belongs_to() { + let own = wait_begin_client_config("svc-new", "svc-new", "0.0.0.0", 8000, Some("k-secret")) + .expect("own launch shows its key"); + assert!(own.contains("api key: k-secret"), "{own}"); + assert!(own.contains("http://0.0.0.0:8000/v1/models"), "{own}"); + // Attached to an already-starting server: its key is not this one. + assert!( + wait_begin_client_config("svc-old", "svc-new", "0.0.0.0", 8000, Some("k-secret")) + .is_none() + ); + // Loopback binds have no key to show. + assert!(wait_begin_client_config("svc-new", "svc-new", "127.0.0.1", 8000, None).is_none()); + } + #[test] fn serve_bind_validation_requires_public_ack() { validate_bind_host("127.0.0.1", false).unwrap(); diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index 1377639224..e00d21fcdc 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -718,6 +718,32 @@ impl FollowFooter { Ok(()) } + /// Clear the footer and end an unfinished redraw row, leaving the cursor on + /// a fresh row. + fn finish(&mut self, out: &mut impl io::Write) -> Result<()> { + self.erase(out)?; + if self.open_row { + writeln!(out)?; + self.open_row = false; + } + out.flush()?; + Ok(()) + } + + /// One line of the follow's own, written below the log output with the + /// footer redrawn under it. An unfinished download bar row is ended first, + /// not overwritten. + fn write_notice( + &mut self, + out: &mut impl io::Write, + text: &str, + cols: Option, + ) -> Result<()> { + self.finish(out)?; + writeln!(out, "{text}")?; + self.draw(out, false, cols) + } + /// `cols` truncates the text so it never wraps onto a second row, which the /// one-row erase could not clear. fn draw( @@ -776,6 +802,7 @@ pub(crate) fn follow_service_logs(paths: &AppPaths, service_id: &str) -> Result< paths, service_id, &mut out, + &mut io::stderr(), tty, LOG_FOLLOW_POLL, LOG_FOLLOW_LIVENESS_EVERY, @@ -790,27 +817,82 @@ fn newly_ready(announced: &mut bool, status: &str) -> bool { ready } +/// A line about the follow itself (the hint, "is ready.", "stopped following."). +/// On a terminal it shares the log's stream and row layout; piped, it goes to +/// `err` so `rocm logs -f | grep error` sees log text only. +fn write_note( + out: &mut impl io::Write, + err: &mut impl io::Write, + tty: bool, + line: &str, +) -> Result<()> { + if tty { + writeln!(out, "{line}")?; + } else { + writeln!(err, "{line}")?; + } + Ok(()) +} + +#[allow(clippy::too_many_arguments)] fn follow_service_logs_with( paths: &AppPaths, service_id: &str, out: &mut impl io::Write, + err: &mut impl io::Write, tty: bool, poll: Duration, liveness_every: Duration, cols: &dyn Fn() -> Option, ) -> Result<()> { - let record = load_managed_service(paths, service_id)?; + follow_service_logs_via( + paths, + service_id, + &|| load_managed_service(paths, service_id), + out, + err, + tty, + poll, + liveness_every, + cols, + ) +} + +/// [`follow_service_logs_with`] with the record read injected: a `ready` status +/// only sticks while the endpoint answers, so a test cannot plant one on disk. +#[allow(clippy::too_many_arguments)] +fn follow_service_logs_via( + paths: &AppPaths, + service_id: &str, + load: &dyn Fn() -> Result, + out: &mut impl io::Write, + err: &mut impl io::Write, + tty: bool, + poll: Duration, + liveness_every: Duration, + cols: &dyn Fn() -> Option, +) -> Result<()> { + let record = load()?; // Open the tail before the snapshot: output written in between is then shown // twice (in the snapshot and by the first poll) rather than lost, and the // tail can start mid-line. let mut tail = LogTail::at_end(&record.log_path); write!(out, "{}", render_service_logs_text(paths, service_id)?)?; + out.flush()?; if !managed_service_is_live(&record) { - writeln!( + write_note( out, - "\nService {service_id} is {}; not following.", - record.status + err, + tty, + &format!( + "\nService {service_id} is {}; not following.", + record.status + ), )?; + // Same outcome as a follow that sees the server fail: not a success. + if record.status == "failed" { + bail!("service {service_id} failed; see the log output above"); + } return Ok(()); } // On a terminal the hint is a footer that stays under the newest output; a @@ -818,10 +900,46 @@ fn follow_service_logs_with( let mut footer = tty.then(|| FollowFooter::new(service_id)); match footer.as_mut() { Some(footer) => footer.draw(out, false, cols())?, - None => writeln!(out, "\n{}", follow_hint(service_id))?, + None => write_note(out, err, tty, &format!("\n{}", follow_hint(service_id)))?, } out.flush()?; + let result = follow_loop( + load, + &record, + &mut tail, + &mut footer, + out, + err, + tty, + poll, + liveness_every, + cols, + ); + if result.is_err() + && let Some(footer) = footer.as_mut() + { + // Leave the terminal on a clean row so the caller's `Error:` is not glued + // to the footer or to an unfinished download bar. + let _ = footer.finish(out); + } + result +} + +#[allow(clippy::too_many_arguments)] +fn follow_loop( + load: &dyn Fn() -> Result, + record: &ManagedServiceRecord, + tail: &mut LogTail, + footer: &mut Option, + out: &mut impl io::Write, + err: &mut impl io::Write, + tty: bool, + poll: Duration, + liveness_every: Duration, + cols: &dyn Fn() -> Option, +) -> Result<()> { + let service_id = record.service_id.as_str(); let mut announced_ready = record.status == "ready"; let mut since_check = Duration::ZERO; loop { @@ -837,17 +955,12 @@ fn follow_service_logs_with( continue; } since_check = Duration::ZERO; - let record = load_managed_service(paths, service_id)?; + let record = load()?; if newly_ready(&mut announced_ready, &record.status) { - let line = [LogSegment { - text: format!("Service {service_id} is ready."), - redraw: false, - }]; + let line = format!("Service {service_id} is ready."); match footer.as_mut() { - Some(footer) => footer.write_batch(out, &line, cols())?, - None => { - write_log_segments(out, &line, tty)?; - } + Some(footer) => footer.write_notice(out, &line, cols())?, + None => write_note(out, err, tty, &line)?, } } if !managed_service_is_live(&record) { @@ -863,10 +976,14 @@ fn follow_service_logs_with( if open_row { writeln!(out)?; } - writeln!( + write_note( out, - "Service {service_id} is {}; stopped following.", - record.status + err, + tty, + &format!( + "Service {service_id} is {}; stopped following.", + record.status + ), )?; // Same outcome as `serve --wait`: a server that failed is not a success. if record.status == "failed" { @@ -2511,10 +2628,12 @@ pub(crate) mod tests { appended }); let mut out = Vec::new(); + let mut err = Vec::new(); follow_service_logs_with( &paths, "svc_follow_tty", &mut out, + &mut err, true, Duration::from_millis(20), Duration::from_millis(40), @@ -2577,10 +2696,12 @@ pub(crate) mod tests { appended }); let mut out = Vec::new(); + let mut err = Vec::new(); follow_service_logs_with( &paths, "svc_follow_bar", &mut out, + &mut err, true, Duration::from_millis(20), Duration::from_millis(40), @@ -2621,10 +2742,12 @@ pub(crate) mod tests { record.status = "stopped".to_owned(); record.write()?; let mut out = Vec::new(); + let mut err = Vec::new(); follow_service_logs_with( &paths, "svc_follow_tty_stopped", &mut out, + &mut err, true, Duration::from_millis(1), Duration::from_millis(1), @@ -2660,20 +2783,25 @@ pub(crate) mod tests { fs::write(&record.log_path, "Downloading 86%\n")?; let mut out = Vec::new(); + let mut err = Vec::new(); follow_service_logs_with( &paths, "svc_follow_stopped", &mut out, + &mut err, false, Duration::from_millis(1), Duration::from_millis(1), &|| None, )?; let out = String::from_utf8(out)?; + let err = String::from_utf8(err)?; assert!(out.contains("Downloading 86%"), "{out}"); + // The notice is not log text, so a pipe keeps it off stdout. + assert!(!out.contains("not following"), "{out}"); assert!( - out.contains("Service svc_follow_stopped is stopped; not following."), - "{out}" + err.contains("Service svc_follow_stopped is stopped; not following."), + "{err}" ); let _ = fs::remove_dir_all(root); Ok(()) @@ -2720,18 +2848,20 @@ pub(crate) mod tests { record.write() }); let mut out = Vec::new(); + let mut err = Vec::new(); let result = follow_service_logs_with( &paths, "svc_follow_fail", &mut out, + &mut err, false, Duration::from_millis(20), Duration::from_millis(40), &|| None, ); writer.join().expect("writer thread")?; - let out = String::from_utf8(out)?; - assert!(out.contains("is failed; stopped following"), "{out}"); + let err = String::from_utf8(err)?; + assert!(err.contains("is failed; stopped following"), "{err}"); let error = result.expect_err("a failed service must not exit 0"); assert!( error.to_string().contains("svc_follow_fail failed"), @@ -2781,10 +2911,12 @@ pub(crate) mod tests { appended }); let mut out = Vec::new(); + let mut err = Vec::new(); follow_service_logs_with( &paths, "svc_follow_live", &mut out, + &mut err, false, Duration::from_millis(20), Duration::from_millis(40), @@ -2792,9 +2924,208 @@ pub(crate) mod tests { )?; writer.join().expect("writer thread")?; let out = String::from_utf8(out)?; - assert!(out.contains("Following svc_follow_live"), "{out}"); + let err = String::from_utf8(err)?; + // Piped: stdout is the log text alone, with no terminal escapes, and the + // follow's own lines go to stderr, the hint exactly once. assert!(out.contains("Downloading 43%"), "{out}"); - assert!(out.contains("stopped following"), "{out}"); + assert!(!out.contains('\x1b'), "{out:?}"); + assert!(!out.contains("Following"), "{out}"); + assert!(!out.contains("stopped following"), "{out}"); + assert_eq!(err.matches("Following svc_follow_live").count(), 1, "{err}"); + assert!(err.contains("stopped following"), "{err}"); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + fn starting_record(paths: &AppPaths, id: &str, port: u16) -> Result { + let mut record = ManagedServiceRecord::new( + paths, + id, + "vllm", + "qwen3.5", + "Qwen/Qwen3.5", + "127.0.0.1", + port, + "managed", + std::process::id(), + None, + None, + None, + ); + record.status = "starting".to_owned(); + record.write()?; + Ok(record) + } + + /// A record read that walks through `statuses` (the last one repeats). + fn scripted_statuses( + base: ManagedServiceRecord, + statuses: &'static [&'static str], + ) -> impl Fn() -> Result { + let calls = std::cell::Cell::new(0_usize); + move || { + let call = calls.get(); + calls.set(call + 1); + let mut record = base.clone(); + record.status = statuses[call.min(statuses.len() - 1)].to_owned(); + Ok(record) + } + } + + #[test] + fn follow_service_logs_announces_ready_once_when_the_server_comes_up() -> Result<()> { + let (root, paths) = test_paths("follow-ready-line"); + paths.ensure()?; + let record = starting_record(&paths, "svc_follow_ready", 11441)?; + fs::write(&record.log_path, "before\n")?; + // Initial read, then one read per liveness check. + let load = scripted_statuses( + record, + &["starting", "starting", "ready", "ready", "stopped"], + ); + let mut out = Vec::new(); + let mut err = Vec::new(); + follow_service_logs_via( + &paths, + "svc_follow_ready", + &load, + &mut out, + &mut err, + false, + Duration::from_millis(5), + Duration::from_millis(5), + &|| None, + )?; + let err = String::from_utf8(err)?; + assert_eq!( + err.matches("Service svc_follow_ready is ready.").count(), + 1, + "{err}" + ); + assert!(!String::from_utf8(out)?.contains("is ready.")); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn follow_service_logs_ends_a_download_bar_row_before_the_ready_line_on_a_tty() -> Result<()> { + let (root, paths) = test_paths("follow-ready-bar"); + paths.ensure()?; + let record = starting_record(&paths, "svc_follow_rbar", 11442)?; + fs::write(&record.log_path, "")?; + let log_path = record.log_path.clone(); + let appender = thread::spawn(move || -> Result<()> { + thread::sleep(Duration::from_millis(50)); + let mut file = fs::OpenOptions::new().append(true).open(&log_path)?; + std::io::Write::write_all(&mut file, b"28%\r43%\r")?; + Ok(()) + }); + // The bar is on screen when the status turns ready (liveness check 3 on). + let calls = std::cell::Cell::new(0_usize); + let base = record; + let load = move || -> Result { + let call = calls.get(); + calls.set(call + 1); + if call >= 3 { + thread::sleep(Duration::from_millis(30)); + } + let mut record = base.clone(); + record.status = match call { + 0..=7 => "starting", + 8..=9 => "ready", + _ => "stopped", + } + .to_owned(); + Ok(record) + }; + let mut out = Vec::new(); + let mut err = Vec::new(); + follow_service_logs_via( + &paths, + "svc_follow_rbar", + &load, + &mut out, + &mut err, + true, + Duration::from_millis(10), + Duration::from_millis(10), + &|| None, + )?; + appender.join().expect("appender thread")?; + let out = String::from_utf8(out)?; + // The bar's row is ended with a newline, then the ready line gets its own + // row; it never replaces the bar. + assert!( + out.contains(&format!( + "{ERASE}\x1b[1A\nService svc_follow_rbar is ready.\n" + )), + "{out:?}" + ); + assert!(err.is_empty(), "a terminal follow writes nothing to stderr"); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn follow_service_logs_clears_the_footer_when_the_follow_errors() -> Result<()> { + let (root, paths) = test_paths("follow-error"); + paths.ensure()?; + let record = starting_record(&paths, "svc_follow_gone", 11443)?; + fs::write(&record.log_path, "")?; + let manifest = record.manifest_path; + // The service record disappears mid-follow, so the next liveness read fails. + let remover = thread::spawn(move || -> Result<()> { + thread::sleep(Duration::from_millis(100)); + fs::remove_file(manifest)?; + Ok(()) + }); + let mut out = Vec::new(); + let mut err = Vec::new(); + let result = follow_service_logs_with( + &paths, + "svc_follow_gone", + &mut out, + &mut err, + true, + Duration::from_millis(20), + Duration::from_millis(40), + &|| None, + ); + remover.join().expect("remover thread")?; + assert!(result.is_err(), "a vanished record must end the follow"); + let out = String::from_utf8(out)?; + // The footer row is erased, so `Error:` starts on a clean row. + assert!(out.ends_with(ERASE), "{out:?}"); + let _ = fs::remove_dir_all(root); + Ok(()) + } + + #[test] + fn follow_service_logs_fails_for_a_service_that_is_already_failed() -> Result<()> { + let (root, paths) = test_paths("follow-already-failed"); + paths.ensure()?; + let mut record = starting_record(&paths, "svc_follow_dead", 11444)?; + record.status = "failed".to_owned(); + record.supervisor_pid = 0; + record.write()?; + let mut out = Vec::new(); + let mut err = Vec::new(); + let result = follow_service_logs_with( + &paths, + "svc_follow_dead", + &mut out, + &mut err, + false, + Duration::from_millis(1), + Duration::from_millis(1), + &|| None, + ); + let error = result.expect_err("a failed service must not exit 0"); + assert!( + error.to_string().contains("svc_follow_dead failed"), + "{error}" + ); + assert!(String::from_utf8(err)?.contains("is failed; not following.")); let _ = fs::remove_dir_all(root); Ok(()) } diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 42176864b2..9a5a68ccd5 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -428,7 +428,9 @@ Expected result: line when it finishes starting; Ctrl-C ends the follow and the server keeps running. On a terminal, a `Ctrl-C stops following, the server keeps running. Following ` line stays on the row directly under the newest output (and under the download bar) as the log - grows; with output piped to a file it is printed once instead. + grows; piped, stdout carries only the log text and the hint (once), ready and + stopped-following lines go to stderr. A server already `failed` when the follow + starts also exits non-zero. - `rocm serve --wait` keeps waiting past 45 seconds and returns once the server answers or its engine gives up (vLLM and AIM: 5 minutes by default, override with `ROCM_CLI_VLLM_READY_TIMEOUT_SECS` / `ROCM_CLI_AIM_READY_TIMEOUT_SECS`). If the server is killed or stopped meanwhile, it exits non-zero diff --git a/tests/e2e-cucumber/features/logs.feature b/tests/e2e-cucumber/features/logs.feature index f3cd6cd5ae..c58430162b 100644 --- a/tests/e2e-cucumber/features/logs.feature +++ b/tests/e2e-cucumber/features/logs.feature @@ -50,3 +50,13 @@ Feature: Log inspection Given a stopped local server whose log shows a download at 43 percent When the user follows that server's logs with rocm services logs -f Then the CLI prints the download line, says it is not following, and exits + + # A server that ended `failed` is not a success, whether `--follow` finds it + # already failed (this scenario) or sees it fail while following (covered by + # `follow_service_logs_fails_when_the_service_failed` in + # `apps/rocm/src/services_text.rs`). + @id:logs-follow-failed-service-exits-nonzero + Scenario: logs-07 - Following a failed server's logs prints them and exits with an error + Given a failed local server whose log shows a download at 43 percent + When the user follows that server's logs + Then the CLI prints the download line, says it is not following, and exits with an error diff --git a/tests/e2e-cucumber/tests/e2e/logs_steps.rs b/tests/e2e-cucumber/tests/e2e/logs_steps.rs index 0dc14673b2..e6e31444c2 100644 --- a/tests/e2e-cucumber/tests/e2e/logs_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/logs_steps.rs @@ -130,6 +130,15 @@ const FOLLOW_SERVICE_ID: &str = "vllm-e2e-follow"; #[given("a stopped local server whose log shows a download at 43 percent")] async fn plant_stopped_server_with_log(world: &mut E2eWorld) { + plant_server_with_log(world, "stopped"); +} + +#[given("a failed local server whose log shows a download at 43 percent")] +async fn plant_failed_server_with_log(world: &mut E2eWorld) { + plant_server_with_log(world, "failed"); +} + +fn plant_server_with_log(world: &mut E2eWorld, status: &str) { let root = world.isolated_root.as_ref().expect("no isolated root"); let data = root.path().join("data"); let services = data.join("services"); @@ -147,7 +156,7 @@ async fn plant_stopped_server_with_log(world: &mut E2eWorld) { "port": 9, "endpoint_url": "http://127.0.0.1:9/v1", "mode": "managed", - "status": "stopped", + "status": status, "supervisor_pid": 0, "engine_pid": null, "manifest_path": services.join(format!("{FOLLOW_SERVICE_ID}.json")), @@ -185,15 +194,42 @@ async fn follow_without_service(world: &mut E2eWorld) { #[then("the CLI prints the download line, says it is not following, and exits")] async fn prints_and_exits(world: &mut E2eWorld) { let out = ok_output(world); + let err = world.cli_stderr.clone().unwrap_or_default(); + assert!( + out.contains("Downloading model.safetensors: 43%"), + "expected the planted log line, got:\n{out}" + ); + // Piped, stdout is log text only; the notice about the follow goes to stderr. + let note = format!("Service {FOLLOW_SERVICE_ID} is stopped; not following."); + assert!(!out.contains(¬e), "notice leaked onto stdout:\n{out}"); + assert!( + err.contains(¬e), + "expected the not-following line, got:\n{err}" + ); +} + +#[then("the CLI prints the download line, says it is not following, and exits with an error")] +async fn prints_and_fails(world: &mut E2eWorld) { + let rc = world.cli_rc.expect("no command rc recorded"); + let out = world.cli_output.clone().unwrap_or_default(); + let err = world.cli_stderr.clone().unwrap_or_default(); + assert_ne!( + rc, 0, + "a failed server must not follow as success:\n{out}\n{err}" + ); assert!( out.contains("Downloading model.safetensors: 43%"), "expected the planted log line, got:\n{out}" ); assert!( - out.contains(&format!( - "Service {FOLLOW_SERVICE_ID} is stopped; not following." + err.contains(&format!( + "Service {FOLLOW_SERVICE_ID} is failed; not following." )), - "expected the not-following line, got:\n{out}" + "expected the not-following line, got:\n{err}" + ); + assert!( + err.contains(&format!("service {FOLLOW_SERVICE_ID} failed")), + "expected the failure error, got:\n{err}" ); } From 84b539ff1f08a671bb4cf93cb93e4a0037e09cd0 Mon Sep 17 00:00:00 2001 From: juhovainio Date: Thu, 5 Nov 2026 17:30:35 +0200 Subject: [PATCH 12/12] ROCMAI-791: keep the dummy api key out of wait-begin test failure messages CodeQL treats formatting the key-bearing config into an assert message as cleartext logging. The assertions are unchanged; only the messages are. Signed-off-by: juhovainio --- apps/rocm/src/service_launch.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/rocm/src/service_launch.rs b/apps/rocm/src/service_launch.rs index 0519678d68..4c1e1cd464 100644 --- a/apps/rocm/src/service_launch.rs +++ b/apps/rocm/src/service_launch.rs @@ -2146,8 +2146,13 @@ mod tests { fn wait_begin_shows_the_key_only_for_the_launch_it_belongs_to() { let own = wait_begin_client_config("svc-new", "svc-new", "0.0.0.0", 8000, Some("k-secret")) .expect("own launch shows its key"); - assert!(own.contains("api key: k-secret"), "{own}"); - assert!(own.contains("http://0.0.0.0:8000/v1/models"), "{own}"); + // No `{own}` in the failure messages: the text carries the dummy key, + // and formatting it into an assert is what CodeQL flags as logging it. + assert!(own.contains("api key: k-secret"), "key line missing"); + assert!( + own.contains("http://0.0.0.0:8000/v1/models"), + "models url missing" + ); // Attached to an already-starting server: its key is not this one. assert!( wait_begin_client_config("svc-old", "svc-new", "0.0.0.0", 8000, Some("k-secret"))