From 57aa80dd4a2d51c4115ad7652682a5772300262a Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 17:48:45 +0000 Subject: [PATCH 1/8] test(dash-daemon): pin how the loop retires managed services when the registry changes Drive the real run_loop through the managed-service registry and pin what the broadcast stream and the snapshots show when a service's record is removed, when the registry directory is gone, and when the directory exists but cannot be listed (a regular file in its place, and on unix a mode-000 directory). The first two are the intended behaviour: an empty or absent registry means no services, so the known one is announced gone and leaves the snapshots. The last two document a defect, not a design: a failed read is indistinguishable from an empty registry, so the loop retires every managed instance on a transient glitch. They pass on today's code; the follow-up change flips them. Signed-off-by: Jussi Elo --- .../tests/discovery_reconcile.rs | 127 +++++++++++++++++- 1 file changed, 124 insertions(+), 3 deletions(-) diff --git a/crates/rocm-dash-daemon/tests/discovery_reconcile.rs b/crates/rocm-dash-daemon/tests/discovery_reconcile.rs index bd7c7b5200..311c016e27 100644 --- a/crates/rocm-dash-daemon/tests/discovery_reconcile.rs +++ b/crates/rocm-dash-daemon/tests/discovery_reconcile.rs @@ -13,9 +13,10 @@ //! its endpoint changes. On Lemonade: the endpoint is announced once, scraped, //! and announced gone when it stops answering. //! -//! The registry's retirement of a vanished service and Lemonade's change of id -//! are not driven through the loop here; they are covered only by the unit tests -//! in `runner.rs`. The Docker source has no seam that works without a container +//! The registry's retirement of a vanished service is driven through the loop +//! here too, for an empty registry and for one whose directory cannot be read. +//! Lemonade's change of id is not driven through the loop here; it is covered only +//! by the unit tests in `runner.rs`. The Docker source has no seam that works without a container //! daemon, so its reconcile is covered only by the unit tests next to the shared //! helper in `runner.rs`. @@ -437,6 +438,126 @@ async fn managed_endpoint_change_starts_the_instance_clean() { ); } +// ── Managed-service registry: what the loop does when the registry changes ── + +/// A running loop with one managed service that has been scraped at least once, +/// so the tests below start from an instance that carries live fields. +struct LiveManaged { + rx: broadcast::Receiver, + services_dir: std::path::PathBuf, + // Held so the registry directory and the mock endpoint outlive the test body. + _tmp: tempfile::TempDir, + _server: MockServer, + _runner: tokio::task::JoinHandle<()>, +} + +async fn start_live_managed() -> LiveManaged { + let tmp = tempfile::TempDir::new().unwrap(); + let services_dir = tmp.path().join("services"); + let server = spawn_server(|_| vllm_metrics()).await; + write_record(&services_dir, server.port); + let (mut rx, runner) = spawn_runner(managed_opts(services_dir.clone())); + snapshot_where(&mut rx, |snap| { + instance(snap, SVC_ID) + .filter(|i| approx(i.kv_cache_usage_pct, KV_PCT)) + .map(|_| ()) + }) + .await; + LiveManaged { + rx, + services_dir, + _tmp: tmp, + _server: server, + _runner: runner, + } +} + +/// Wait until the service is announced gone and a snapshot no longer lists it: +/// the announcement and the snapshot are the two things a dashboard shows. +async fn assert_service_is_retired(rx: &mut broadcast::Receiver) { + next_event(rx, |ev| match ev { + Event::InstanceGone { container_id } if container_id == SVC_ID => Some(()), + _ => None, + }) + .await; + snapshot_where(rx, |snap| instance(snap, SVC_ID).is_none().then_some(())).await; +} + +/// Replace the registry directory with a regular file of the same name, so +/// listing it fails with something other than "not found". Portable: it needs no +/// permission change, so it behaves the same for a privileged user and on Windows. +fn replace_registry_dir_with_a_file(services_dir: &std::path::Path) { + std::fs::remove_dir_all(services_dir).unwrap(); + std::fs::write(services_dir, b"not a directory").unwrap(); +} + +/// A registry that is confirmed empty (the directory is there and holds no +/// record) retires the service it knew about. +#[tokio::test] +async fn managed_confirmed_empty_registry_retires_the_known_instance() { + let mut live = start_live_managed().await; + + std::fs::remove_file(live.services_dir.join(format!("{SVC_ID}.json"))).unwrap(); + + assert_service_is_retired(&mut live.rx).await; +} + +/// A registry directory that does not exist is also confirmed empty: it is +/// created the first time something is served, so its absence means no services. +#[tokio::test] +async fn managed_absent_registry_dir_retires_the_known_instance() { + let mut live = start_live_managed().await; + + std::fs::remove_dir_all(&live.services_dir).unwrap(); + + assert_service_is_retired(&mut live.rx).await; +} + +/// A registry directory that cannot be listed retires the known service today, +/// exactly as an empty one does: the loop cannot tell a failed read from a +/// registry with nothing in it. This pins that defect; the next commit makes the +/// read failure leave the instance alone and flips this test. +#[tokio::test] +async fn managed_unlistable_registry_dir_retires_the_known_instance_today() { + let mut live = start_live_managed().await; + + replace_registry_dir_with_a_file(&live.services_dir); + + assert_service_is_retired(&mut live.rx).await; +} + +/// Restores a directory's permissions on drop, so the temp dir can be cleaned +/// up even when the test fails part-way. +#[cfg(unix)] +struct RestorePermissions(std::path::PathBuf); + +#[cfg(unix)] +impl Drop for RestorePermissions { + fn drop(&mut self) { + use std::os::unix::fs::PermissionsExt; + let _ = std::fs::set_permissions(&self.0, std::fs::Permissions::from_mode(0o755)); + } +} + +/// The same defect through a permission error rather than a path error. Unix +/// only, because Windows has no mode bits to drop; skipped when the process can +/// still list the directory with its read bit removed (running as root). +#[cfg(unix)] +#[tokio::test] +async fn managed_unreadable_registry_dir_retires_the_known_instance_today() { + use std::os::unix::fs::PermissionsExt; + let mut live = start_live_managed().await; + let _restore = RestorePermissions(live.services_dir.clone()); + + std::fs::set_permissions(&live.services_dir, std::fs::Permissions::from_mode(0o000)).unwrap(); + if std::fs::read_dir(&live.services_dir).is_ok() { + eprintln!("skipped: this process can list a mode-000 directory (running as root)"); + return; + } + + assert_service_is_retired(&mut live.rx).await; +} + // ── Lemonade endpoint ─────────────────────────────────────────────────────── fn lemonade_responder(path: &str) -> String { From d1ea3d5b9657b4f21048f364e51fb7acfa135bfd Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 18:23:34 +0000 Subject: [PATCH 2/8] fix(dash-daemon): keep managed services when the registry cannot be read The daemon's discovery loop read the managed-service registry through a best-effort loader that turned an unreadable directory, or a record it could not read, into zero services, and then reconciled against that empty set. A transient read error therefore announced every managed instance gone and dropped it from the snapshots, the same storm Docker discovery already avoids by skipping the reconcile when its listing errors. read_service_records now returns a Result that tells a registry that is confirmed empty (no directory, or no records) from one that could not be read (the directory cannot be listed, or a record file cannot be read). On a failed read the loop reconciles nothing for that tick: the known services stay known, no gone or repeat discovered event is sent, and the instances keep the data they hold and keep being scraped. The header carries a warning for as long as the registry cannot be read, and the log carries one warning for the first failed pass of a streak and one line when reads succeed again. A record that reads but is not valid JSON is skipped and reported once, so it no longer hides the other services, and an unchanged set of corrupt records is not reported again. load_service_records keeps its best-effort contract for the one-shot readers in the dashboard. The two commit-one tests that pinned the old retirement now assert the new behaviour, with recovery, recovery into an empty registry, malformed-record and once-per-streak-logging tests beside them, and scenario dash-42 covers the on-screen behaviour. Signed-off-by: Jussi Elo --- crates/rocm-dash-daemon/src/registry.rs | 228 +++++++++++++++-- crates/rocm-dash-daemon/src/runner.rs | 239 +++++++++++++++++- .../tests/discovery_reconcile.rs | 233 +++++++++++++++-- .../tests/registry_read_logging.rs | 163 ++++++++++++ docs/architecture.md | 2 +- tests/e2e-cucumber/features/dash.feature | 21 ++ tests/e2e-cucumber/tests/e2e/dash_steps.rs | 74 ++++++ 7 files changed, 915 insertions(+), 45 deletions(-) create mode 100644 crates/rocm-dash-daemon/tests/registry_read_logging.rs diff --git a/crates/rocm-dash-daemon/src/registry.rs b/crates/rocm-dash-daemon/src/registry.rs index 44e6275fb5..aec1bdc67c 100644 --- a/crates/rocm-dash-daemon/src/registry.rs +++ b/crates/rocm-dash-daemon/src/registry.rs @@ -32,7 +32,8 @@ use std::collections::HashSet; use std::fs; -use std::path::Path; +use std::io::ErrorKind; +use std::path::{Path, PathBuf}; use rocm_dash_collectors::engine_registry::EngineKind; use rocm_dash_core::metrics::{InstanceStatus, StartupPhase}; @@ -119,30 +120,114 @@ fn instance_status_for_record(status: &str, startup_phase: Option<&str>) -> Inst } } -/// Load every managed-service record under `services_dir`, newest first. +/// What one read of the registry directory found. +#[derive(Debug, Default)] +pub struct RegistryRead { + /// Every well-formed record, newest first. + pub records: Vec, + /// `*.json` files that were read but are not a JSON object this module can + /// parse, sorted by path. They are left out of `records`, and nothing else is + /// hidden by them: one corrupt record does not stop the others loading. + pub malformed: Vec, +} + +/// The registry could not be read, so what it holds is unknown. This is not the +/// same as an empty registry: a caller that retires services which are missing +/// from a read must not do so on this. +#[derive(Debug)] +pub struct RegistryReadError { + /// The directory that could not be listed, or the record file that could not + /// be read. + pub path: PathBuf, + pub source: std::io::Error, +} + +impl std::fmt::Display for RegistryReadError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "cannot read {}: {}", self.path.display(), self.source) + } +} + +impl std::error::Error for RegistryReadError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.source) + } +} + +/// Read every managed-service record under `services_dir`, telling a registry +/// that is confirmed empty apart from one that could not be read. /// -/// Best-effort and side-effect-free: a missing directory yields an empty list, -/// and an individual unreadable/malformed `*.json` is skipped rather than -/// failing the whole load (mirrors the canonical rocm-cli `load_managed_services` -/// semantics, minus the engine-state status overlay which the supervisor owns). -pub fn load_service_records(services_dir: &Path) -> Vec { - let Ok(entries) = fs::read_dir(services_dir) else { - return Vec::new(); +/// * A directory that does not exist is confirmed empty: it is created the first +/// time something is served, so its absence means there are no services. A +/// directory that exists but cannot be listed (or a regular file where it +/// should be) is a read failure. +/// * A record file that vanishes between the listing and the read was removed +/// on purpose and is skipped. Any other failure to read a record (a permission +/// or I/O error) is a read failure for the whole pass: the record's service +/// cannot be told apart from one that is gone, and nothing is known about which +/// service the unreadable file belonged to. +/// * A record that reads but does not parse is malformed. It is reported in +/// [`RegistryRead::malformed`] and the other records still load, because the +/// rocm-cli writer replaces a record atomically, so a corrupt file is corrupt +/// for good rather than half-written. +/// +/// Side-effect-free, and the canonical rocm-cli `load_managed_services` semantics +/// minus the engine-state status overlay, which the supervisor owns. +pub fn read_service_records(services_dir: &Path) -> Result { + let entries = match fs::read_dir(services_dir) { + Ok(entries) => entries, + Err(source) => { + let absent = matches!( + fs::metadata(services_dir), + Err(ref e) if e.kind() == ErrorKind::NotFound + ); + return if absent { + Ok(RegistryRead::default()) + } else { + Err(RegistryReadError { + path: services_dir.to_path_buf(), + source, + }) + }; + } }; - let mut records = Vec::new(); - for entry in entries.flatten() { + let mut read = RegistryRead::default(); + for entry in entries { + let entry = entry.map_err(|source| RegistryReadError { + path: services_dir.to_path_buf(), + source, + })?; let path = entry.path(); - if path.extension().and_then(|v| v.to_str()) != Some("json") { + if path.extension().and_then(|v| v.to_str()) != Some("json") || path.is_dir() { continue; } - if let Ok(bytes) = fs::read(&path) - && let Ok(record) = serde_json::from_slice::(&bytes) - { - records.push(record); + let bytes = match fs::read(&path) { + Ok(bytes) => bytes, + Err(e) if e.kind() == ErrorKind::NotFound => continue, + Err(source) => return Err(RegistryReadError { path, source }), + }; + match serde_json::from_slice::(&bytes) { + Ok(record) => read.records.push(record), + Err(_) => read.malformed.push(path), } } - records.sort_by_key(|record| std::cmp::Reverse(record.created_at_unix_ms)); - records + read.records + .sort_by_key(|record| std::cmp::Reverse(record.created_at_unix_ms)); + read.malformed.sort(); + Ok(read) +} + +/// Load every managed-service record under `services_dir`, newest first. +/// +/// Best-effort, for the one-shot readers (the dashboard's launcher and exit +/// prompt) that have nothing to retire: a directory that cannot be read yields an +/// empty list, and a malformed record is skipped. The daemon's discovery loop +/// must not use this, because it cannot tell an unreadable registry from an empty +/// one; it calls [`read_service_records`]. +pub fn load_service_records(services_dir: &Path) -> Vec { + read_service_records(services_dir) + .map(|read| read.records) + .unwrap_or_default() } /// Convert a managed-service record into a [`DiscoveredService`] for the @@ -339,6 +424,113 @@ mod tests { let _ = fs::remove_dir_all(&dir); } + #[test] + fn read_missing_dir_is_confirmed_empty() { + let dir = test_dir("read-absent").join("nope"); + let read = read_service_records(&dir).expect("an absent registry is confirmed empty"); + assert!(read.records.is_empty()); + assert!(read.malformed.is_empty()); + } + + #[test] + fn read_empty_dir_is_confirmed_empty() { + let dir = test_dir("read-empty"); + let read = read_service_records(&dir).expect("an empty registry is confirmed empty"); + assert!(read.records.is_empty()); + assert!(read.malformed.is_empty()); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn read_regular_file_where_the_dir_should_be_is_a_read_failure() { + let dir = test_dir("read-file-not-dir"); + let not_a_dir = dir.join("services"); + fs::write(¬_a_dir, "not a directory").unwrap(); + + let err = read_service_records(¬_a_dir) + .expect_err("a file where the registry should be must not read as empty"); + assert_eq!(err.path, not_a_dir); + assert!(err.to_string().contains("cannot read"), "{err}"); + // The best-effort wrapper keeps its old answer for the one-shot readers. + assert!(load_service_records(¬_a_dir).is_empty()); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn read_keeps_well_formed_records_beside_a_malformed_one() { + let dir = test_dir("read-malformed"); + fs::write( + dir.join("a.json"), + record_json("svc-a", "vllm", 8000, "running", 100), + ) + .unwrap(); + fs::write( + dir.join("b.json"), + record_json("svc-b", "vllm", 8001, "running", 200), + ) + .unwrap(); + fs::write(dir.join("bad.json"), "{ not valid").unwrap(); + fs::write(dir.join("also-bad.json"), "42").unwrap(); + fs::write(dir.join("notes.txt"), "ignore me").unwrap(); + + let read = read_service_records(&dir).expect("a malformed record is not a read failure"); + let ids: Vec<&str> = read.records.iter().map(|r| r.service_id.as_str()).collect(); + assert_eq!(ids, ["svc-b", "svc-a"]); + assert_eq!( + read.malformed, + [dir.join("also-bad.json"), dir.join("bad.json")] + ); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn read_skips_a_directory_named_like_a_record() { + let dir = test_dir("read-dir-named-json"); + fs::create_dir(dir.join("looks-like-a-record.json")).unwrap(); + fs::write( + dir.join("a.json"), + record_json("svc-a", "vllm", 8000, "running", 100), + ) + .unwrap(); + + let read = read_service_records(&dir).expect("a stray directory is not a read failure"); + assert_eq!(read.records.len(), 1); + assert!(read.malformed.is_empty()); + let _ = fs::remove_dir_all(&dir); + } + + /// Unix only (no mode bits on Windows), and skipped when the process can + /// read a mode-000 file anyway (running as root). + #[cfg(unix)] + #[test] + fn read_unreadable_record_file_is_a_read_failure() { + use std::os::unix::fs::PermissionsExt; + let dir = test_dir("read-unreadable-record"); + fs::write( + dir.join("a.json"), + record_json("svc-a", "vllm", 8000, "running", 100), + ) + .unwrap(); + let locked = dir.join("locked.json"); + fs::write( + &locked, + record_json("svc-locked", "vllm", 8001, "running", 200), + ) + .unwrap(); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).unwrap(); + if fs::read(&locked).is_ok() { + eprintln!("skipped: this process can read a mode-000 file (running as root)"); + let _ = fs::remove_dir_all(&dir); + return; + } + + let result = read_service_records(&dir); + let _ = fs::set_permissions(&locked, fs::Permissions::from_mode(0o644)); + let err = result.expect_err("an unreadable record must not read as a missing one"); + assert_eq!(err.path, locked); + let _ = fs::remove_dir_all(&dir); + } + #[test] fn discovered_uses_registry_port_as_authority() { let dir = test_dir("port-authority"); diff --git a/crates/rocm-dash-daemon/src/runner.rs b/crates/rocm-dash-daemon/src/runner.rs index 7d526b9577..c6d320173d 100644 --- a/crates/rocm-dash-daemon/src/runner.rs +++ b/crates/rocm-dash-daemon/src/runner.rs @@ -402,6 +402,7 @@ pub async fn run_loop( // holding them only for the tick that produced them makes the header ⚠ badge // blink on and off. Keep the last result until that cadence runs again. let mut docker_warn: Option = None; + let mut registry_warn: Option = None; let mut scrape_warns: Vec = Vec::new(); loop { @@ -588,19 +589,32 @@ pub async fn run_loop( // the registry's authority; non-vLLM engines are tracked so the vLLM // scrape below skips them. A model served via `rocm serve` thus appears // in the dashboard and is scraped for `gen_tps` without Docker. + // + // Like the Docker source above, a pass that could not read the registry + // reconciles nothing: what is known stays known (no `Gone`, no repeat + // `Discovered`), and the instances keep the data they hold. Only a + // registry that was read, even an empty one, retires what it lacks. if let Some(services_dir) = opts.services_dir.as_ref() && discovery_due { - let records = crate::registry::load_service_records(services_dir); - let disc = crate::registry::discover_managed_services(&records); - discovery.managed_non_vllm = disc.non_vllm; - reconcile.reconcile_set( - DiscoverySource::Managed, - &disc.svcs, - &discovery.known_services, - &disc.seen, - ); - discovery.known_services = disc.seen; + let read = crate::registry::read_service_records(services_dir); + let change = discovery.registry_health.observe(&read); + change.log(); + match read { + Ok(read) => { + let disc = crate::registry::discover_managed_services(&read.records); + discovery.managed_non_vllm = disc.non_vllm; + reconcile.reconcile_set( + DiscoverySource::Managed, + &disc.svcs, + &discovery.known_services, + &disc.seen, + ); + discovery.known_services = disc.seen; + registry_warn = None; + } + Err(e) => registry_warn = Some(registry_read_warning(&e)), + } } if tick_count.is_multiple_of(instance_ticks) { scrape_warns.clear(); @@ -862,6 +876,7 @@ pub async fn run_loop( } } warnings.extend(docker_warn.iter().cloned()); + warnings.extend(registry_warn.iter().cloned()); warnings.extend(scrape_warns.iter().cloned()); let snap = Snapshot { timestamp: cycle_at, @@ -1020,6 +1035,89 @@ struct DiscoveryState { managed_non_vllm: HashSet, /// The id of the currently-live Lemonade instance, if any. lemonade_id: Option, + /// How the registry reads have been going, so a condition that lasts many + /// passes is logged once rather than every discovery tick. + registry_health: RegistryHealth, +} + +/// What the managed-service registry's reads have reported so far. A failure +/// that lasts is logged on its first pass only, the recovery once, and a set of +/// malformed records each time it changes; the dashboard's warning badge, which +/// is cheap and does not scroll, carries the failure for as long as it lasts. +#[derive(Default)] +struct RegistryHealth { + /// The previous pass could not read the registry. + failing: bool, + /// The malformed records the last successful pass reported, so an unchanged + /// set is not reported again. + reported_malformed: Vec, +} + +/// What a discovery pass should say about the registry, apart from the services +/// it found. Kept as data so [`RegistryHealth::observe`] can be tested without a +/// log subscriber. +#[derive(Debug, Default, PartialEq, Eq)] +struct RegistryChange { + /// The first failed pass of a streak, with the error. + failed: Option, + /// The first successful pass after a streak of failures. + recovered: bool, + /// The malformed records, when the set differs from the one last reported + /// and is not empty. + malformed: Vec, +} + +impl RegistryHealth { + fn observe( + &mut self, + read: &Result, + ) -> RegistryChange { + let mut change = RegistryChange::default(); + match read { + Err(e) => { + if !self.failing { + self.failing = true; + change.failed = Some(e.to_string()); + } + } + Ok(read) => { + if self.failing { + self.failing = false; + change.recovered = true; + } + if read.malformed != self.reported_malformed { + self.reported_malformed.clone_from(&read.malformed); + change.malformed.clone_from(&read.malformed); + } + } + } + change + } +} + +impl RegistryChange { + fn log(&self) { + if let Some(error) = &self.failed { + warn!( + error = %error, + "managed-service registry could not be read; keeping the known services until it can be" + ); + } + if self.recovered { + info!("managed-service registry is readable again"); + } + if !self.malformed.is_empty() { + warn!( + files = ?self.malformed, + "skipping managed-service records that are not valid JSON" + ); + } + } +} + +/// The header warning for a pass that could not read the registry. +fn registry_read_warning(error: &crate::registry::RegistryReadError) -> String { + format!("managed registry: {error}; keeping the last known services") } /// Which discovery source a service came from. It only selects the log wording @@ -2002,4 +2100,125 @@ mod tests { pass.reconcile().reconcile_lemonade(None, &mut current); assert!(pass.announced().is_empty(), "nothing left to retire"); } + + // Registry read health: one test per row of the state matrix (first + // failure, repeated failure, recovery, empty, populated, malformed new, + // malformed unchanged, malformed cleared). + + fn registry_read( + malformed: &[&str], + ) -> Result { + Ok(crate::registry::RegistryRead { + records: Vec::new(), + malformed: malformed.iter().map(PathBuf::from).collect(), + }) + } + + fn registry_failure( + why: &str, + ) -> Result { + Err(crate::registry::RegistryReadError { + path: PathBuf::from("services"), + source: std::io::Error::other(why), + }) + } + + #[test] + fn registry_health_reports_the_first_failure_of_a_streak_once() { + let mut health = RegistryHealth::default(); + + let first = health.observe(®istry_failure("disk gone")); + assert_eq!( + first.failed.as_deref(), + Some("cannot read services: disk gone"), + "the first failed pass names the error" + ); + assert!(!first.recovered); + + for pass in 2..=5 { + assert_eq!( + health.observe(®istry_failure("disk gone")), + RegistryChange::default(), + "failed pass {pass} of the same streak must stay quiet" + ); + } + } + + #[test] + fn registry_health_reports_recovery_once_and_a_new_streak_again() { + let mut health = RegistryHealth::default(); + health.observe(®istry_failure("one")); + health.observe(®istry_failure("one")); + + let recovered = health.observe(®istry_read(&[])); + assert!(recovered.recovered); + assert!(recovered.failed.is_none()); + assert_eq!( + health.observe(®istry_read(&[])), + RegistryChange::default(), + "only the first good pass after a streak is a recovery" + ); + + let again = health.observe(®istry_failure("two")); + assert_eq!(again.failed.as_deref(), Some("cannot read services: two")); + } + + #[test] + fn registry_health_is_quiet_for_an_empty_or_populated_registry() { + let mut health = RegistryHealth::default(); + // A healthy read, empty or not, says nothing: there is no failure to + // report and no failure to have recovered from. + for _ in 0..3 { + assert_eq!( + health.observe(®istry_read(&[])), + RegistryChange::default() + ); + } + } + + #[test] + fn registry_health_reports_a_malformed_set_when_it_changes() { + let mut health = RegistryHealth::default(); + + let first = health.observe(®istry_read(&["a.json"])); + assert_eq!(first.malformed, [PathBuf::from("a.json")]); + assert_eq!( + health.observe(®istry_read(&["a.json"])), + RegistryChange::default(), + "an unchanged malformed set is not reported again" + ); + + let grown = health.observe(®istry_read(&["a.json", "b.json"])); + assert_eq!( + grown.malformed, + [PathBuf::from("a.json"), PathBuf::from("b.json")], + "a newly malformed record reports the whole set" + ); + + let cleared = health.observe(®istry_read(&[])); + assert!( + cleared.malformed.is_empty(), + "clearing the set is not a warning" + ); + let back = health.observe(®istry_read(&["a.json"])); + assert_eq!( + back.malformed, + [PathBuf::from("a.json")], + "a record that goes bad again after being fixed is reported again" + ); + } + + #[test] + fn registry_health_keeps_the_malformed_set_across_a_read_failure() { + let mut health = RegistryHealth::default(); + health.observe(®istry_read(&["a.json"])); + health.observe(®istry_failure("blip")); + + let after = health.observe(®istry_read(&["a.json"])); + assert!(after.recovered); + assert!( + after.malformed.is_empty(), + "the same corrupt record is not news just because a read failed in between" + ); + } } diff --git a/crates/rocm-dash-daemon/tests/discovery_reconcile.rs b/crates/rocm-dash-daemon/tests/discovery_reconcile.rs index 311c016e27..62f81ea1fe 100644 --- a/crates/rocm-dash-daemon/tests/discovery_reconcile.rs +++ b/crates/rocm-dash-daemon/tests/discovery_reconcile.rs @@ -158,15 +158,19 @@ fn lemonade_opts(port: u16) -> RunnerOptions { /// record: a torn read would look like the service vanishing and coming back, /// which is exactly what the tests below assert does not happen. fn write_record(services_dir: &std::path::Path, port: u16) { + write_record_for(services_dir, SVC_ID, port); +} + +fn write_record_for(services_dir: &std::path::Path, id: &str, port: u16) { std::fs::create_dir_all(services_dir).unwrap(); let json = format!( - r#"{{"service_id":"{SVC_ID}","engine":"vllm","model_ref":"Reconcile/1B","canonical_model_id":"Reconcile/1B","host":"127.0.0.1","port":{port},"status":"ready","created_at_unix_ms":0}}"# + r#"{{"service_id":"{id}","engine":"vllm","model_ref":"Reconcile/1B","canonical_model_id":"Reconcile/1B","host":"127.0.0.1","port":{port},"status":"ready","created_at_unix_ms":0}}"# ); // The registry only reads `*.json`, so the `.tmp` suffix keeps the // half-written file invisible to it. - let tmp = services_dir.join(format!("{SVC_ID}.json.tmp")); + let tmp = services_dir.join(format!("{id}.json.tmp")); std::fs::write(&tmp, json).unwrap(); - std::fs::rename(&tmp, services_dir.join(format!("{SVC_ID}.json"))).unwrap(); + std::fs::rename(&tmp, services_dir.join(format!("{id}.json"))).unwrap(); } /// Read events until `pick` returns a value, or fail at the deadline. @@ -445,12 +449,19 @@ async fn managed_endpoint_change_starts_the_instance_clean() { struct LiveManaged { rx: broadcast::Receiver, services_dir: std::path::PathBuf, - // Held so the registry directory and the mock endpoint outlive the test body. + server: MockServer, + // Held so the registry directory outlives the test body. _tmp: tempfile::TempDir, - _server: MockServer, _runner: tokio::task::JoinHandle<()>, } +impl LiveManaged { + /// The port the service's record names, for writing the record back. + const fn last_port(&self) -> u16 { + self.server.port + } +} + async fn start_live_managed() -> LiveManaged { let tmp = tempfile::TempDir::new().unwrap(); let services_dir = tmp.path().join("services"); @@ -466,8 +477,8 @@ async fn start_live_managed() -> LiveManaged { LiveManaged { rx, services_dir, + server, _tmp: tmp, - _server: server, _runner: runner, } } @@ -513,17 +524,107 @@ async fn managed_absent_registry_dir_retires_the_known_instance() { assert_service_is_retired(&mut live.rx).await; } -/// A registry directory that cannot be listed retires the known service today, -/// exactly as an empty one does: the loop cannot tell a failed read from a -/// registry with nothing in it. This pins that defect; the next commit makes the -/// read failure leave the instance alone and flips this test. +/// What the stream and the snapshots showed over a window of snapshots. +#[derive(Default)] +struct Window { + /// Every discovered/gone announcement for [`SVC_ID`], in order. + announcements: Vec<&'static str>, + snapshots: usize, + /// Snapshots in the window that did not list the service. + missing: usize, + /// The service as the last snapshot in the window listed it. + last: Option, + /// The warnings the last snapshot in the window carried. + warnings: Vec, +} + +/// Watch `count` snapshots, recording what was announced and what each snapshot +/// listed, under one deadline. +async fn watch(rx: &mut broadcast::Receiver, count: usize) -> Window { + let mut window = Window::default(); + observe( + rx, + "a window of snapshots", + &mut window, + |w, ev| { + match ev { + Event::InstanceDiscovered(inst) if inst.container_id == SVC_ID => { + w.announcements.push("discovered"); + } + Event::InstanceGone { container_id } if container_id == SVC_ID => { + w.announcements.push("gone"); + } + Event::Snapshot(snap) => { + w.snapshots += 1; + w.last = instance(snap, SVC_ID).cloned(); + w.missing += usize::from(w.last.is_none()); + w.warnings.clone_from(&snap.warnings); + } + _ => {} + } + w.snapshots >= count + }, + |w| { + format!( + "{} snapshots, {} without the service, announcements {:?}", + w.snapshots, w.missing, w.announcements + ) + }, + ) + .await; + window +} + +fn registry_warning(window: &Window) -> Option<&String> { + window + .warnings + .iter() + .find(|w| w.starts_with("managed registry")) +} + +/// Snapshots to watch across a read failure: several discovery ticks, so a loop +/// that retired the service would have done so well before the window ends. +const OUTAGE_SNAPSHOTS: usize = 8; + +/// The service as a still-scraped managed instance should look: present, with +/// the live fields its endpoint reports. +fn assert_still_live(inst: &Instance) { + assert!(inst.port.is_some(), "the instance lost its endpoint"); + assert!( + approx(inst.kv_cache_usage_pct, KV_PCT), + "the instance lost its kv cache reading: {:?}", + inst.kv_cache_usage_pct + ); + assert_eq!( + inst.running_reqs, + Some(1), + "the instance lost its request count" + ); +} + +/// A registry directory that cannot be listed says nothing about the services in +/// it, so the known service stays: not announced gone, still in every snapshot +/// with the data it holds, and the header says why the registry is not being +/// followed. #[tokio::test] -async fn managed_unlistable_registry_dir_retires_the_known_instance_today() { +async fn managed_unlistable_registry_dir_keeps_the_known_instance() { let mut live = start_live_managed().await; replace_registry_dir_with_a_file(&live.services_dir); + let window = watch(&mut live.rx, OUTAGE_SNAPSHOTS).await; - assert_service_is_retired(&mut live.rx).await; + assert_eq!( + window.announcements, + Vec::<&str>::new(), + "a failed read must not announce the service gone or discovered" + ); + assert_eq!(window.missing, 0, "a snapshot dropped the service"); + assert_still_live(window.last.as_ref().expect("the service is listed")); + let warning = registry_warning(&window).expect("the header should say the registry is unread"); + assert!( + warning.contains("keeping the last known services"), + "{warning}" + ); } /// Restores a directory's permissions on drop, so the temp dir can be cleaned @@ -539,12 +640,12 @@ impl Drop for RestorePermissions { } } -/// The same defect through a permission error rather than a path error. Unix -/// only, because Windows has no mode bits to drop; skipped when the process can -/// still list the directory with its read bit removed (running as root). +/// The same through a permission error rather than a path error. Unix only, +/// because Windows has no mode bits to drop; skipped when the process can still +/// list the directory with its read bit removed (running as root). #[cfg(unix)] #[tokio::test] -async fn managed_unreadable_registry_dir_retires_the_known_instance_today() { +async fn managed_unreadable_registry_dir_keeps_the_known_instance() { use std::os::unix::fs::PermissionsExt; let mut live = start_live_managed().await; let _restore = RestorePermissions(live.services_dir.clone()); @@ -554,10 +655,110 @@ async fn managed_unreadable_registry_dir_retires_the_known_instance_today() { eprintln!("skipped: this process can list a mode-000 directory (running as root)"); return; } + let window = watch(&mut live.rx, OUTAGE_SNAPSHOTS).await; + + assert_eq!(window.announcements, Vec::<&str>::new()); + assert_eq!(window.missing, 0, "a snapshot dropped the service"); + assert_still_live(window.last.as_ref().expect("the service is listed")); + assert!(registry_warning(&window).is_some(), "{:?}", window.warnings); +} + +/// When the registry becomes readable again with the same service in it, the +/// service was never gone: nothing is announced, and the warning clears. +#[tokio::test] +async fn managed_registry_recovery_does_not_reannounce_the_known_instance() { + let mut live = start_live_managed().await; + let port = live.last_port(); + + replace_registry_dir_with_a_file(&live.services_dir); + let outage = watch(&mut live.rx, OUTAGE_SNAPSHOTS).await; + assert!(registry_warning(&outage).is_some()); + + std::fs::remove_file(&live.services_dir).unwrap(); + write_record(&live.services_dir, port); + let after = watch(&mut live.rx, OUTAGE_SNAPSHOTS).await; + + assert_eq!( + after.announcements, + Vec::<&str>::new(), + "recovery must not read as a discovery storm" + ); + assert_eq!(after.missing, 0); + assert_still_live(after.last.as_ref().expect("the service is listed")); + assert_eq!( + registry_warning(&after), + None, + "the warning must clear once the registry reads again" + ); +} + +/// A registry that comes back readable and empty is a confirmed-empty registry: +/// the service that was kept through the outage is retired now. +#[tokio::test] +async fn managed_registry_recovering_empty_retires_the_instance_kept_through_the_outage() { + let mut live = start_live_managed().await; + + replace_registry_dir_with_a_file(&live.services_dir); + let outage = watch(&mut live.rx, OUTAGE_SNAPSHOTS).await; + assert_eq!(outage.missing, 0); + + std::fs::remove_file(&live.services_dir).unwrap(); + std::fs::create_dir(&live.services_dir).unwrap(); assert_service_is_retired(&mut live.rx).await; } +/// One corrupt record does not hide the others: the well-formed service stays +/// and a new well-formed one beside the corrupt file is still discovered. +#[tokio::test] +async fn managed_malformed_record_does_not_hide_the_other_services() { + let mut live = start_live_managed().await; + let port = live.last_port(); + + std::fs::write(live.services_dir.join("corrupt.json"), "{ not valid").unwrap(); + write_record_for(&live.services_dir, "second-svc", port); + + let second_announced = std::cell::Cell::new(false); + let mut window = Window::default(); + observe( + &mut live.rx, + "the second service to be announced beside a corrupt record", + &mut window, + |w, ev| { + match ev { + Event::InstanceDiscovered(inst) if inst.container_id == "second-svc" => { + second_announced.set(true); + } + Event::InstanceGone { container_id } => { + panic!("{container_id} was announced gone beside a corrupt record") + } + Event::Snapshot(snap) => { + w.snapshots += 1; + w.missing += usize::from(instance(snap, SVC_ID).is_none()); + w.warnings.clone_from(&snap.warnings); + } + _ => {} + } + second_announced.get() && w.snapshots >= SETTLE_SNAPSHOTS + }, + |w| { + format!( + "{} snapshots, second announced: {}", + w.snapshots, + second_announced.get() + ) + }, + ) + .await; + + assert_eq!(window.missing, 0, "the well-formed service was dropped"); + assert_eq!( + registry_warning(&window), + None, + "a corrupt record is not an unreadable registry" + ); +} + // ── Lemonade endpoint ─────────────────────────────────────────────────────── fn lemonade_responder(path: &str) -> String { diff --git a/crates/rocm-dash-daemon/tests/registry_read_logging.rs b/crates/rocm-dash-daemon/tests/registry_read_logging.rs new file mode 100644 index 0000000000..a40daa8db2 --- /dev/null +++ b/crates/rocm-dash-daemon/tests/registry_read_logging.rs @@ -0,0 +1,163 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! What the run loop logs about the managed-service registry's health. +//! +//! A read failure that lasts many discovery ticks is logged once, its recovery +//! once, and a second outage once more: one line per streak, not one per tick. +//! +//! This is its own test binary, with a single test, on purpose. `tracing` caches +//! whether a callsite is enabled the first time any thread reaches it; when a +//! test thread installs a per-thread subscriber after another thread has already +//! run the same `warn!` with none installed, the cached answer drops the event +//! and the capture comes up empty. In `discovery_reconcile.rs` the tests share a +//! process, so that is what happened there. Here nothing else logs first. + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use rocm_dash_core::protocol::Event; +use tokio::sync::broadcast; +use tokio::time::timeout; + +use rocm_dash_daemon::bench_ring::BenchRing; +use rocm_dash_daemon::runner::{self, RunnerOptions}; +use rocm_dash_daemon::snapshot_ring::SnapshotRing; + +const TICK: Duration = Duration::from_millis(100); +const DEADLINE: Duration = Duration::from_secs(20); +const SVC_ID: &str = "logging-svc"; +/// Snapshots to watch across a streak: several discovery ticks. +const STREAK_SNAPSHOTS: usize = 8; +const FAILED: &str = "managed-service registry could not be read"; +const RECOVERED: &str = "managed-service registry is readable again"; + +/// A shared buffer a `tracing` subscriber writes to, so the test can count what +/// the loop logged. +#[derive(Clone, Default)] +struct LogCapture(Arc>>); + +impl std::io::Write for LogCapture { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl LogCapture { + fn text(&self) -> String { + String::from_utf8_lossy(&self.0.lock().unwrap()).into_owned() + } + + fn lines_containing(&self, needle: &str) -> usize { + self.text() + .lines() + .filter(|line| line.contains(needle)) + .count() + } +} + +fn write_record(services_dir: &std::path::Path) { + std::fs::create_dir_all(services_dir).unwrap(); + // Nothing listens on the port, so the service is never scraped: this test is + // about discovery only. A rename keeps the tick from reading half a record. + let json = format!( + r#"{{"service_id":"{SVC_ID}","engine":"vllm","model_ref":"Logging/1B","canonical_model_id":"Logging/1B","host":"127.0.0.1","port":9,"status":"ready","created_at_unix_ms":0}}"# + ); + let tmp = services_dir.join(format!("{SVC_ID}.json.tmp")); + std::fs::write(&tmp, json).unwrap(); + std::fs::rename(&tmp, services_dir.join(format!("{SVC_ID}.json"))).unwrap(); +} + +/// Replace the registry directory with a regular file, so listing it fails. +fn break_registry(services_dir: &std::path::Path) { + std::fs::remove_dir_all(services_dir).unwrap(); + std::fs::write(services_dir, b"not a directory").unwrap(); +} + +/// Wait for `count` more snapshots, under one deadline. +async fn watch(rx: &mut broadcast::Receiver, count: usize) { + let deadline = tokio::time::Instant::now() + DEADLINE; + let mut seen = 0; + while seen < count { + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + match timeout(remaining, rx.recv()).await { + Ok(Ok(Event::Snapshot(_))) => seen += 1, + Ok(Ok(_) | Err(broadcast::error::RecvError::Lagged(_))) => {} + Ok(Err(broadcast::error::RecvError::Closed)) => panic!("broadcast channel closed"), + Err(elapsed) => { + panic!("only {seen} of {count} snapshots within {DEADLINE:?} ({elapsed})") + } + } + } +} + +/// The default subscriber is per thread and `#[tokio::test]` runs the loop on +/// the test's own thread, so the capture sees the loop's output. +#[tokio::test] +async fn registry_read_failure_is_logged_once_per_streak() { + let capture = LogCapture::default(); + let writer = capture.clone(); + let subscriber = tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .with_ansi(false) + .with_writer(move || writer.clone()) + .finish(); + let _guard = tracing::subscriber::set_default(subscriber); + + let tmp = tempfile::TempDir::new().unwrap(); + let services_dir = tmp.path().join("services"); + write_record(&services_dir); + let (tx, mut rx) = broadcast::channel::(1024); + let _runner = tokio::spawn(runner::run_loop( + Some(TICK), + tx, + Arc::new(Mutex::new(SnapshotRing::new(16))), + Arc::new(Mutex::new(BenchRing::new(4))), + None, + RunnerOptions { + services_dir: Some(services_dir.clone()), + discovery_tick: TICK, + instance_tick: TICK * 2, + gpu_tick: TICK, + ..RunnerOptions::default() + }, + )); + watch(&mut rx, 3).await; + assert_eq!( + capture.lines_containing(FAILED), + 0, + "a readable registry logs nothing: {}", + capture.text() + ); + + break_registry(&services_dir); + watch(&mut rx, STREAK_SNAPSHOTS).await; + assert_eq!( + capture.lines_containing(FAILED), + 1, + "a streak of failed ticks must log one warning: {}", + capture.text() + ); + assert_eq!(capture.lines_containing(RECOVERED), 0); + + std::fs::remove_file(&services_dir).unwrap(); + write_record(&services_dir); + watch(&mut rx, STREAK_SNAPSHOTS).await; + assert_eq!(capture.lines_containing(RECOVERED), 1, "{}", capture.text()); + assert_eq!(capture.lines_containing(FAILED), 1); + + break_registry(&services_dir); + watch(&mut rx, STREAK_SNAPSHOTS).await; + assert_eq!( + capture.lines_containing(FAILED), + 2, + "a second streak is a new warning: {}", + capture.text() + ); + assert_eq!(capture.lines_containing(RECOVERED), 1); +} diff --git a/docs/architecture.md b/docs/architecture.md index 4dca1d18f4..e1a4d1f9ee 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -278,7 +278,7 @@ The other three dashboard crates are described at crate level here, not module b - [`rocm-dash-collectors`](../crates/rocm-dash-collectors/src/lib.rs) — the IO-heavy collector implementations behind `rocm-dash-core`'s traits (amd-smi, Docker, host metrics, Lemonade and vLLM scrapers, and on Windows the [`winperf.rs`](../crates/rocm-dash-collectors/src/winperf.rs) perf-counter/ADLX GPU collector) plus the benchmark load generator; used by `rocm-dash-daemon` and, on Windows, by `rocm-dash-tui` in-process. - [`rocm-dash-core`](../crates/rocm-dash-core/src/lib.rs) — pure types, traits, schemas and the reducer, with no rendering dependencies. -- [`rocm-dash-daemon`](../crates/rocm-dash-daemon/src/lib.rs) — the daemon library: [`runner.rs`](../crates/rocm-dash-daemon/src/runner.rs) drives the collectors on tick cadences and broadcasts snapshots, and [`server.rs`](../crates/rocm-dash-daemon/src/server.rs) streams them to TUI clients as NDJSON. `apps/rocm`'s [`dash.rs`](../apps/rocm/src/dash.rs) drives it. +- [`rocm-dash-daemon`](../crates/rocm-dash-daemon/src/lib.rs) — the daemon library: [`runner.rs`](../crates/rocm-dash-daemon/src/runner.rs) drives the collectors on tick cadences and broadcasts snapshots, and [`server.rs`](../crates/rocm-dash-daemon/src/server.rs) streams them to TUI clients as NDJSON. [`registry.rs`](../crates/rocm-dash-daemon/src/registry.rs) reads the managed-service records `rocm serve` writes and tells a registry that is confirmed empty (no directory, or no records) from one that could not be read: the run loop retires a managed service that an empty registry lacks, but when a pass cannot read the registry it keeps the known services and their data, as it does when Docker discovery errors, and the header warns until a read succeeds. `apps/rocm`'s [`dash.rs`](../apps/rocm/src/dash.rs) drives it. ### `crates/rocm-engine-protocol` — engine IPC protocol diff --git a/tests/e2e-cucumber/features/dash.feature b/tests/e2e-cucumber/features/dash.feature index 668c6bef96..57052e9221 100644 --- a/tests/e2e-cucumber/features/dash.feature +++ b/tests/e2e-cucumber/features/dash.feature @@ -723,3 +723,24 @@ Feature: Interactive dashboard Then the Home view counts the managed model as running and shows it as loading When the user quits the dashboard Then the dashboard exits successfully + + # The dashboard follows the managed-service registry on a timer. A pass that + # cannot read the registry directory says nothing about the services in it, so + # it must not retire the model that is on screen: the header warns that the + # registry is not being read, and the model stays. The directory is swapped for + # a regular file, which no path or permission setup of the runner can make + # readable. A registry that reads but is empty does retire a service; that + # contrast, the recovery path and the once-per-streak log line are asserted by + # the daemon's own tests in `crates/rocm-dash-daemon/tests/`, where the loop + # runs without a terminal. + @id:dash-managed-model-survives-unreadable-registry @requires-os:linux + Scenario: dash-42 - A managed model stays on screen while its registry cannot be read + Given a running managed model is available locally + When the user opens the dashboard + And the user opens the Observe view + Then the managed model is displayed + When the managed-service registry becomes unreadable + Then the header warns that the registry is not being read + And the managed model stays displayed + When the user quits the dashboard + Then the dashboard exits successfully diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index 744f3f5d12..10c99be738 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -2496,6 +2496,80 @@ async fn metrics_endpoint_fails(world: &mut E2eWorld) { }); } +/// Replace the managed-service registry directory with a regular file of the +/// same name, so listing it fails the way a broken mount or a permission change +/// would, and wait for the failure to reach the screen. The header warnings +/// badge growing is the evidence that a frame the daemon assembled after the +/// failed pass is on display (the daemon adds a warning for as long as the +/// registry cannot be read); the count is read before the swap because other +/// warnings, such as no amd-smi on a GPU-less runner, are already present and +/// constant. +#[when("the managed-service registry becomes unreadable")] +async fn managed_registry_becomes_unreadable(world: &mut E2eWorld) { + let before = warning_badge_count(&session(world).screen_text()); + let services_dir = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("data") + .join("services"); + std::fs::remove_dir_all(&services_dir).expect("failed to remove the registry directory"); + std::fs::write(&services_dir, b"not a directory") + .expect("failed to put a file where the registry directory was"); + + session(world) + .wait_for_screen_where( + &format!("the header warnings badge grows past {before}"), + |screen| warning_badge_count(screen) > before, + default_timeout(), + ) + .await + .unwrap_or_else(|e| { + panic!( + "the unreadable registry never reached the header warnings badge, so no \ + frame here is known to postdate the failed read: {e}" + ) + }); +} + +/// The badge growing is what the `When` step waited for; this reads it back so +/// the scenario states the expectation, and a failure here names the half that +/// broke: the registry failure went unreported, rather than the model dropped. +#[then("the header warns that the registry is not being read")] +async fn header_warns_registry_unread(world: &mut E2eWorld) { + assert!( + warning_badge_count(&session(world).screen_text()) > 0, + "the header shows no warning after the registry became unreadable" + ); +} + +/// The model must be on screen in the frame that carries the warning and stay +/// on every frame for a few seconds after it. A daemon that retired the model +/// on a failed read would drop it in the same frame as the warning appears, so +/// the first sample already catches it; the dwell covers a retirement that +/// trails by a discovery tick. +#[then("the managed model stays displayed")] +async fn managed_model_stays_displayed(world: &mut E2eWorld) { + let model = world + .model_name + .as_deref() + .expect("no model name set") + .to_string(); + let deadline = Instant::now() + Duration::from_secs(3); + loop { + let screen = session(world).screen_text(); + assert!( + screen.contains(&model), + "the managed model left the screen while its registry could not be read:\n{screen}" + ); + if Instant::now() >= deadline { + return; + } + tokio::time::sleep(Duration::from_millis(200)).await; + } +} + /// EAI-7960 principal regression assertion. /// /// The frame under assertion is provably post-failure (the header warnings From 5ac34a40a0364f05e5ffb14178e23715635371e2 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 21:31:42 +0000 Subject: [PATCH 3/8] test(dash-daemon): cover every state a Lemonade probe can observe Add one test per lifecycle row of the Lemonade source: never answered, first answer, the same endpoint again, a host change, a port change, the endpoint going quiet, coming back elsewhere, the same answer twice in one tick, and moving away and back. Each row asserts the announcements, the instances a snapshot would carry, the per-id telemetry and the (id, port) pairs the vLLM scrape would poll. The vLLM scrape target selection moves out of run_loop into vllm_scrape_targets, unchanged, so the tests can ask which endpoints would be polled. The host-change, port-change and move-away-and-back rows pin what the code does today: the superseded instance is neither announced gone nor removed and is still a scrape target. A following change flips them. Signed-off-by: Jussi Elo --- crates/rocm-dash-daemon/src/runner.rs | 295 +++++++++++++++++++++++++- 1 file changed, 287 insertions(+), 8 deletions(-) diff --git a/crates/rocm-dash-daemon/src/runner.rs b/crates/rocm-dash-daemon/src/runner.rs index 7d526b9577..af2f117d51 100644 --- a/crates/rocm-dash-daemon/src/runner.rs +++ b/crates/rocm-dash-daemon/src/runner.rs @@ -614,14 +614,11 @@ pub async fn run_loop( && !runner.state.instances.is_empty() && tick_count.is_multiple_of(instance_ticks) { - let targets: Vec<(String, u16)> = runner - .state - .instances - .values() - .filter(|i| Some(i.container_id.as_str()) != discovery.lemonade_id.as_deref()) - .filter(|i| !discovery.managed_non_vllm.contains(&i.container_id)) - .filter_map(|i| i.port.map(|p| (i.container_id.clone(), p))) - .collect(); + let targets = vllm_scrape_targets( + &runner.state.instances, + discovery.lemonade_id.as_deref(), + &discovery.managed_non_vllm, + ); let prom = prom.clone(); let results = parallel_scrape(targets, move |(id, port)| { let prom = prom.clone(); @@ -1174,6 +1171,22 @@ impl Reconcile<'_> { } } +/// The `(id, port)` pairs the per-instance vLLM Prometheus scrape polls: every +/// instance with a port except the live Lemonade instance (its own collector +/// scrapes it) and the managed services whose engine is not vLLM. +fn vllm_scrape_targets( + instances: &HashMap, + lemonade_id: Option<&str>, + managed_non_vllm: &HashSet, +) -> Vec<(String, u16)> { + instances + .values() + .filter(|i| Some(i.container_id.as_str()) != lemonade_id) + .filter(|i| !managed_non_vllm.contains(&i.container_id)) + .filter_map(|i| i.port.map(|p| (i.container_id.clone(), p))) + .collect() +} + /// Set `vram_used_mb`/`vram_total_mb` on each instance from the per-process /// attribution map, falling back to device-summed VRAM over the instance's /// GPUs when its container has no per-process entry. Pure — the runner does the @@ -1962,6 +1975,272 @@ mod tests { assert!(pass.telemetry.prev_ttft.contains_key("a")); } + // ── Lemonade state matrix ─────────────────────────────────────────────── + // + // One Lemonade endpoint is probed per discovery pass. Its id encodes the host + // and port, so the states a probe can observe are: nothing ever answered; the + // first answer; the same endpoint again; the host or the port changed; the + // endpoint went quiet; it came back somewhere else; and the same answer + // arriving twice in one tick. Each row below checks the announcements, the + // instances a snapshot would carry, the telemetry kept per id, and the + // `(id, port)` pairs the vLLM scrape would poll, because a stale instance is + // visible on all four. + + fn lemonade_svc(host: &str, port: u16) -> DiscoveredService { + DiscoveredService { + container_id: format!("lemonade-{host}-{port}"), + container_name: "lemonade".into(), + model_name: "model".into(), + port: Some(port), + ..Default::default() + } + } + + impl Pass { + /// The ids a snapshot carries. The loop builds a snapshot's instances from + /// `state.instances`, so this is the dashboard's view of the pass. + fn instance_ids(&self) -> Vec { + let mut ids: Vec = self.state.instances.keys().cloned().collect(); + ids.sort(); + ids + } + + /// The ids the vLLM Prometheus scrape would poll while `current` is the + /// live Lemonade id. + fn scrape_targets(&self, current: Option<&str>) -> Vec { + let mut ids: Vec = + vllm_scrape_targets(&self.state.instances, current, &HashSet::new()) + .into_iter() + .map(|(id, _port)| id) + .collect(); + ids.sort(); + ids + } + + /// A live KV reading, latency baselines and a rate tracker for `id`. + fn give_live_telemetry(&mut self, id: &str) { + self.scrape(id); + self.telemetry + .gen_trackers + .insert(id.into(), make_tracker(2)); + } + + fn has_telemetry(&self, id: &str) -> bool { + self.telemetry.gen_trackers.contains_key(id) + || self.telemetry.prev_ttft.contains_key(id) + || self.telemetry.prev_tpot.contains_key(id) + } + } + + fn ids(ids: &[&str]) -> Vec { + ids.iter().map(|s| (*s).to_string()).collect() + } + + /// Discover `old`, give it live telemetry, then probe `new`. Returns the pass, + /// the loop's current Lemonade id and what the second probe announced. + fn lemonade_identity_change( + old: &DiscoveredService, + new: &DiscoveredService, + ) -> (Pass, Option, Vec) { + let mut pass = Pass::new(); + let mut current: Option = None; + pass.reconcile() + .reconcile_lemonade(Some(old.clone()), &mut current); + pass.give_live_telemetry(&old.container_id); + pass.announced(); + + pass.reconcile() + .reconcile_lemonade(Some(new.clone()), &mut current); + let announced = pass.announced(); + (pass, current, announced) + } + + #[test] + fn lemonade_that_never_answers_leaves_no_trace() { + let mut pass = Pass::new(); + let mut current: Option = None; + + pass.reconcile().reconcile_lemonade(None, &mut current); + pass.reconcile().reconcile_lemonade(None, &mut current); + + assert!(pass.announced().is_empty()); + assert_eq!(current, None); + assert!(pass.instance_ids().is_empty()); + assert!(pass.scrape_targets(current.as_deref()).is_empty()); + } + + #[test] + fn lemonade_first_answer_is_announced_once_and_left_to_its_own_collector() { + let mut pass = Pass::new(); + let mut current: Option = None; + + pass.reconcile() + .reconcile_lemonade(Some(lemonade_svc("127.0.0.1", 13305)), &mut current); + + assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13305"]); + assert_eq!(current.as_deref(), Some("lemonade-127.0.0.1-13305")); + assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13305"])); + assert!( + pass.scrape_targets(current.as_deref()).is_empty(), + "the live Lemonade instance is not a vLLM scrape target" + ); + } + + #[test] + fn lemonade_same_endpoint_again_keeps_its_instance_and_live_fields() { + let svc = lemonade_svc("127.0.0.1", 13305); + let id = svc.container_id.clone(); + let mut pass = Pass::new(); + let mut current: Option = None; + pass.reconcile() + .reconcile_lemonade(Some(svc.clone()), &mut current); + pass.give_live_telemetry(&id); + pass.announced(); + + pass.reconcile().reconcile_lemonade(Some(svc), &mut current); + + assert!(pass.announced().is_empty()); + assert_eq!(pass.instance_ids(), ids(&[&id])); + assert_eq!(pass.kv(&id), Some(25.0)); + assert!(pass.has_telemetry(&id)); + assert!(pass.scrape_targets(current.as_deref()).is_empty()); + } + + #[test] + fn lemonade_host_change_pins_the_stale_old_instance() { + let old = lemonade_svc("127.0.0.1", 13305); + let new = lemonade_svc("10.0.0.7", 13305); + let (pass, current, announced) = lemonade_identity_change(&old, &new); + + assert_eq!(current.as_deref(), Some(new.container_id.as_str())); + assert_eq!(announced, ["discovered lemonade-10.0.0.7-13305"]); + assert!(!pass.has_telemetry(&old.container_id)); + // Pins current behaviour, not a goal: the old instance is neither + // announced gone nor removed, stays in every snapshot, and is polled as + // a vLLM target because the scrape filter only skips the current id. + assert_eq!( + pass.instance_ids(), + ids(&["lemonade-10.0.0.7-13305", "lemonade-127.0.0.1-13305"]) + ); + assert_eq!( + pass.scrape_targets(current.as_deref()), + ids(&[&old.container_id]) + ); + } + + #[test] + fn lemonade_port_change_pins_the_stale_old_instance() { + let old = lemonade_svc("127.0.0.1", 13305); + let new = lemonade_svc("127.0.0.1", 13306); + let (pass, current, announced) = lemonade_identity_change(&old, &new); + + assert_eq!(current.as_deref(), Some(new.container_id.as_str())); + assert_eq!(announced, ["discovered lemonade-127.0.0.1-13306"]); + assert!(!pass.has_telemetry(&old.container_id)); + // Pins current behaviour, not a goal: see the host-change row. + assert_eq!( + pass.instance_ids(), + ids(&["lemonade-127.0.0.1-13305", "lemonade-127.0.0.1-13306"]) + ); + assert_eq!( + pass.scrape_targets(current.as_deref()), + ids(&[&old.container_id]) + ); + } + + #[test] + fn lemonade_endpoint_going_quiet_is_announced_gone_and_leaves_no_trace() { + let svc = lemonade_svc("127.0.0.1", 13305); + let id = svc.container_id.clone(); + let mut pass = Pass::new(); + let mut current: Option = None; + pass.reconcile().reconcile_lemonade(Some(svc), &mut current); + pass.give_live_telemetry(&id); + pass.announced(); + + pass.reconcile().reconcile_lemonade(None, &mut current); + + assert_eq!(pass.announced(), [format!("gone {id}")]); + assert_eq!(current, None); + assert!(pass.instance_ids().is_empty()); + assert!(!pass.has_telemetry(&id)); + assert!(pass.scrape_targets(current.as_deref()).is_empty()); + } + + #[test] + fn lemonade_coming_back_elsewhere_after_going_quiet_is_the_only_instance() { + let old = lemonade_svc("127.0.0.1", 13305); + let new = lemonade_svc("127.0.0.1", 13306); + let mut pass = Pass::new(); + let mut current: Option = None; + pass.reconcile() + .reconcile_lemonade(Some(old.clone()), &mut current); + pass.give_live_telemetry(&old.container_id); + pass.reconcile().reconcile_lemonade(None, &mut current); + assert_eq!( + pass.announced(), + [ + "discovered lemonade-127.0.0.1-13305", + "gone lemonade-127.0.0.1-13305" + ] + ); + + pass.reconcile() + .reconcile_lemonade(Some(new.clone()), &mut current); + + assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13306"]); + assert_eq!(current.as_deref(), Some(new.container_id.as_str())); + assert_eq!(pass.instance_ids(), ids(&[&new.container_id])); + assert!(!pass.has_telemetry(&old.container_id)); + assert!(pass.scrape_targets(current.as_deref()).is_empty()); + } + + #[test] + fn lemonade_answer_repeated_within_one_tick_is_announced_once() { + let svc = lemonade_svc("127.0.0.1", 13305); + let mut pass = Pass::new(); + let mut current: Option = None; + + pass.reconcile() + .reconcile_lemonade(Some(svc.clone()), &mut current); + pass.reconcile().reconcile_lemonade(Some(svc), &mut current); + + assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13305"]); + assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13305"])); + assert!(pass.scrape_targets(current.as_deref()).is_empty()); + } + + #[test] + fn lemonade_endpoint_moving_away_and_back_pins_the_stale_instances() { + let a = lemonade_svc("127.0.0.1", 13305); + let b = lemonade_svc("127.0.0.1", 13306); + let mut pass = Pass::new(); + let mut current: Option = None; + pass.reconcile() + .reconcile_lemonade(Some(a.clone()), &mut current); + pass.announced(); + + pass.reconcile() + .reconcile_lemonade(Some(b.clone()), &mut current); + assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13306"]); + pass.reconcile() + .reconcile_lemonade(Some(a.clone()), &mut current); + + // Pins current behaviour, not a goal: the first endpoint is announced + // as discovered for a second time without ever being announced gone, and + // the endpoint that was left stays behind as a vLLM scrape target. + assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13305"]); + assert_eq!(current.as_deref(), Some(a.container_id.as_str())); + assert_eq!( + pass.instance_ids(), + ids(&[&a.container_id, &b.container_id]) + ); + assert_eq!( + pass.scrape_targets(current.as_deref()), + ids(&[&b.container_id]) + ); + } + #[test] fn reconcile_lemonade_follows_one_endpoint_through_its_lifecycle() { let mut pass = Pass::new(); From a8aceb6a265f63958b7646119081110305db5e87 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 21:41:55 +0000 Subject: [PATCH 4/8] fix(dash-daemon): retire the old Lemonade instance when its host or port changes The Lemonade id encodes host and port, so moving the endpoint yields a new id. The old id's telemetry was purged and the new id announced, but the old instance was never retired: it stayed in the state and in every snapshot, no InstanceGone was sent for it, and the vLLM scrape kept polling its port, because the scrape filter only skips the current Lemonade id. On an id change the old id is now retired first (InstanceGone, removed from the state, telemetry purged) and then the new id is upserted and announced, so a snapshot carries exactly one Lemonade instance and nothing polls the endpoint it left. The rows of the Lemonade state matrix that pinned the stale instance, and the pin in the lifecycle test, now assert the retired behaviour. Signed-off-by: Jussi Elo --- crates/rocm-dash-daemon/src/runner.rs | 102 +++++++++++++------------- 1 file changed, 53 insertions(+), 49 deletions(-) diff --git a/crates/rocm-dash-daemon/src/runner.rs b/crates/rocm-dash-daemon/src/runner.rs index af2f117d51..5b03a64642 100644 --- a/crates/rocm-dash-daemon/src/runner.rs +++ b/crates/rocm-dash-daemon/src/runner.rs @@ -1141,10 +1141,12 @@ impl Reconcile<'_> { /// Reconcile the probe-based Lemonade source, which has at most one service. /// /// Its `container_id` encodes the host and port, so a port or host change - /// yields a new id: the old id's telemetry is purged so a stale rate - /// baseline, TTFT and TPOT do not bleed across, and the new id is announced. - /// The old instance is not retired here. `None` means the endpoint did not - /// answer, which retires the current one. + /// yields a new id. The old id is retired first (its telemetry is purged, so + /// a stale rate baseline, TTFT and TPOT do not bleed across; it leaves the + /// state and is announced gone), then the new id is announced, so the state + /// and every snapshot carry one Lemonade instance and the vLLM scrape never + /// polls the old endpoint. `None` means the endpoint did not answer, which + /// retires the current one. fn reconcile_lemonade( &mut self, probed: Option, @@ -1153,8 +1155,8 @@ impl Reconcile<'_> { match probed { Some(svc) => { let is_new_id = current.as_deref() != Some(svc.container_id.as_str()); - if is_new_id && let Some(old_id) = current.as_deref() { - self.telemetry.purge(old_id); + if is_new_id && let Some(old_id) = current.take() { + self.retire(DiscoverySource::Lemonade, &old_id); } self.upsert(&svc, !is_new_id); if is_new_id { @@ -2107,44 +2109,46 @@ mod tests { } #[test] - fn lemonade_host_change_pins_the_stale_old_instance() { + fn lemonade_host_change_retires_the_old_instance_and_announces_the_new_one() { let old = lemonade_svc("127.0.0.1", 13305); let new = lemonade_svc("10.0.0.7", 13305); let (pass, current, announced) = lemonade_identity_change(&old, &new); assert_eq!(current.as_deref(), Some(new.container_id.as_str())); - assert_eq!(announced, ["discovered lemonade-10.0.0.7-13305"]); - assert!(!pass.has_telemetry(&old.container_id)); - // Pins current behaviour, not a goal: the old instance is neither - // announced gone nor removed, stays in every snapshot, and is polled as - // a vLLM target because the scrape filter only skips the current id. assert_eq!( - pass.instance_ids(), - ids(&["lemonade-10.0.0.7-13305", "lemonade-127.0.0.1-13305"]) + announced, + [ + "gone lemonade-127.0.0.1-13305", + "discovered lemonade-10.0.0.7-13305" + ] ); - assert_eq!( - pass.scrape_targets(current.as_deref()), - ids(&[&old.container_id]) + assert!(!pass.has_telemetry(&old.container_id)); + assert_eq!(pass.instance_ids(), ids(&["lemonade-10.0.0.7-13305"])); + assert!( + pass.scrape_targets(current.as_deref()).is_empty(), + "the old endpoint must not be polled as a vLLM target" ); } #[test] - fn lemonade_port_change_pins_the_stale_old_instance() { + fn lemonade_port_change_retires_the_old_instance_and_announces_the_new_one() { let old = lemonade_svc("127.0.0.1", 13305); let new = lemonade_svc("127.0.0.1", 13306); let (pass, current, announced) = lemonade_identity_change(&old, &new); assert_eq!(current.as_deref(), Some(new.container_id.as_str())); - assert_eq!(announced, ["discovered lemonade-127.0.0.1-13306"]); - assert!(!pass.has_telemetry(&old.container_id)); - // Pins current behaviour, not a goal: see the host-change row. assert_eq!( - pass.instance_ids(), - ids(&["lemonade-127.0.0.1-13305", "lemonade-127.0.0.1-13306"]) + announced, + [ + "gone lemonade-127.0.0.1-13305", + "discovered lemonade-127.0.0.1-13306" + ] ); - assert_eq!( - pass.scrape_targets(current.as_deref()), - ids(&[&old.container_id]) + assert!(!pass.has_telemetry(&old.container_id)); + assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13306"])); + assert!( + pass.scrape_targets(current.as_deref()).is_empty(), + "the old endpoint must not be polled as a vLLM target" ); } @@ -2211,7 +2215,7 @@ mod tests { } #[test] - fn lemonade_endpoint_moving_away_and_back_pins_the_stale_instances() { + fn lemonade_endpoint_moving_away_and_back_is_retired_each_time() { let a = lemonade_svc("127.0.0.1", 13305); let b = lemonade_svc("127.0.0.1", 13306); let mut pass = Pass::new(); @@ -2220,25 +2224,27 @@ mod tests { .reconcile_lemonade(Some(a.clone()), &mut current); pass.announced(); - pass.reconcile() - .reconcile_lemonade(Some(b.clone()), &mut current); - assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13306"]); + pass.reconcile().reconcile_lemonade(Some(b), &mut current); + assert_eq!( + pass.announced(), + [ + "gone lemonade-127.0.0.1-13305", + "discovered lemonade-127.0.0.1-13306" + ] + ); pass.reconcile() .reconcile_lemonade(Some(a.clone()), &mut current); - // Pins current behaviour, not a goal: the first endpoint is announced - // as discovered for a second time without ever being announced gone, and - // the endpoint that was left stays behind as a vLLM scrape target. - assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13305"]); - assert_eq!(current.as_deref(), Some(a.container_id.as_str())); assert_eq!( - pass.instance_ids(), - ids(&[&a.container_id, &b.container_id]) - ); - assert_eq!( - pass.scrape_targets(current.as_deref()), - ids(&[&b.container_id]) + pass.announced(), + [ + "gone lemonade-127.0.0.1-13306", + "discovered lemonade-127.0.0.1-13305" + ] ); + assert_eq!(current.as_deref(), Some(a.container_id.as_str())); + assert_eq!(pass.instance_ids(), ids(&[&a.container_id])); + assert!(pass.scrape_targets(current.as_deref()).is_empty()); } #[test] @@ -2260,18 +2266,16 @@ mod tests { ); assert_eq!(pass.kv("l1"), Some(25.0), "live fields carry over"); - // A new id (the host or port changed) purges the old id's telemetry and - // announces the new one. + // A new id (the host or port changed) retires the old id, telemetry + // included, and announces the new one. pass.reconcile() .reconcile_lemonade(Some(discovered("l2", 13306)), &mut current); - assert_eq!(pass.announced(), ["discovered l2"]); + assert_eq!(pass.announced(), ["gone l1", "discovered l2"]); assert_eq!(current.as_deref(), Some("l2")); assert!(!pass.telemetry.prev_ttft.contains_key("l1")); - // Pins current behaviour, not a goal: the superseded instance stays in - // state (and so in every snapshot) because only its telemetry is purged. - // Retiring it later is a deliberate behaviour change, and this assertion - // is the one to update when that happens. - assert!(pass.state.instances.contains_key("l1")); + // The superseded instance leaves the state (and so every snapshot), not + // only its telemetry. + assert!(!pass.state.instances.contains_key("l1")); pass.reconcile().reconcile_lemonade(None, &mut current); assert_eq!(pass.announced(), ["gone l2"]); From 90852f04f3691b9599c4722e949765f489f0bcb1 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 06:01:08 +0000 Subject: [PATCH 5/8] fix(dash-daemon): skip only the unreadable record in the best-effort registry loader The compatibility loader behind the dashboard's launcher, exit prompt and services overlay was a thin wrapper over the strict reader, so a single record file that could not be read (a permission or I/O error) made it discard every readable service. The previous best-effort loader skipped only the failing record. Both readers now share one scan that differs only in what it does with a record file that cannot be read: the strict reader the daemon's discovery loop uses still fails the pass, the best-effort loader skips that file and keeps the rest. The record reader is injected into the scan, so the two behaviours are tested on every platform with a record that the reader refuses, alongside the Unix mode-000 test. A registry with one corrupt record and two readable ones is covered for both readers. Signed-off-by: Jussi Elo --- crates/rocm-dash-daemon/src/registry.rs | 187 ++++++++++++++++++++++-- 1 file changed, 177 insertions(+), 10 deletions(-) diff --git a/crates/rocm-dash-daemon/src/registry.rs b/crates/rocm-dash-daemon/src/registry.rs index aec1bdc67c..29aba916e8 100644 --- a/crates/rocm-dash-daemon/src/registry.rs +++ b/crates/rocm-dash-daemon/src/registry.rs @@ -154,6 +154,15 @@ impl std::error::Error for RegistryReadError { } } +/// What a scan does with a record file that exists but cannot be read. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum UnreadableRecord { + /// The whole pass fails: nothing is known about the service the file held. + Fail, + /// The file is skipped and the other records still load. + Skip, +} + /// Read every managed-service record under `services_dir`, telling a registry /// that is confirmed empty apart from one that could not be read. /// @@ -174,6 +183,26 @@ impl std::error::Error for RegistryReadError { /// Side-effect-free, and the canonical rocm-cli `load_managed_services` semantics /// minus the engine-state status overlay, which the supervisor owns. pub fn read_service_records(services_dir: &Path) -> Result { + read_service_records_with(services_dir, |path| fs::read(path)) +} + +/// [`read_service_records`] with the record reader injected, so a test can make +/// one record unreadable on every platform (a mode-000 file works only on Unix, +/// and not as root). +fn read_service_records_with( + services_dir: &Path, + read_record: impl Fn(&Path) -> std::io::Result>, +) -> Result { + scan_registry(services_dir, read_record, UnreadableRecord::Fail) +} + +/// The scan behind [`read_service_records`] and [`load_service_records`]; the two +/// differ only in what they do with a record file that cannot be read. +fn scan_registry( + services_dir: &Path, + read_record: impl Fn(&Path) -> std::io::Result>, + on_unreadable: UnreadableRecord, +) -> Result { let entries = match fs::read_dir(services_dir) { Ok(entries) => entries, Err(source) => { @@ -193,17 +222,24 @@ pub fn read_service_records(services_dir: &Path) -> Result entry, + Err(_) if on_unreadable == UnreadableRecord::Skip => continue, + Err(source) => { + return Err(RegistryReadError { + path: services_dir.to_path_buf(), + source, + }); + } + }; let path = entry.path(); if path.extension().and_then(|v| v.to_str()) != Some("json") || path.is_dir() { continue; } - let bytes = match fs::read(&path) { + let bytes = match read_record(&path) { Ok(bytes) => bytes, Err(e) if e.kind() == ErrorKind::NotFound => continue, + Err(_) if on_unreadable == UnreadableRecord::Skip => continue, Err(source) => return Err(RegistryReadError { path, source }), }; match serde_json::from_slice::(&bytes) { @@ -220,12 +256,23 @@ pub fn read_service_records(services_dir: &Path) -> Result Vec { - read_service_records(services_dir) + load_service_records_with(services_dir, |path| fs::read(path)) +} + +/// [`load_service_records`] with the record reader injected (see +/// [`read_service_records_with`]). +fn load_service_records_with( + services_dir: &Path, + read_record: impl Fn(&Path) -> std::io::Result>, +) -> Vec { + scan_registry(services_dir, read_record, UnreadableRecord::Skip) .map(|read| read.records) .unwrap_or_default() } @@ -531,6 +578,126 @@ mod tests { let _ = fs::remove_dir_all(&dir); } + /// The one-shot readers (launcher, exit prompt, services overlay) stay + /// best-effort per record: one unreadable record file hides only itself, + /// while the strict reader the daemon's loop uses fails on the same + /// registry. Unix only for the real mode-000 file, and skipped as root; the + /// portable twin below injects the failing read. + #[cfg(unix)] + #[test] + fn load_skips_one_unreadable_record_and_keeps_the_readable_ones() { + use std::os::unix::fs::PermissionsExt; + let dir = test_dir("load-unreadable-record"); + fs::write( + dir.join("a.json"), + record_json("svc-a", "vllm", 8000, "running", 100), + ) + .unwrap(); + fs::write( + dir.join("b.json"), + record_json("svc-b", "vllm", 8001, "running", 300), + ) + .unwrap(); + let locked = dir.join("locked.json"); + fs::write( + &locked, + record_json("svc-locked", "vllm", 8002, "running", 200), + ) + .unwrap(); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).unwrap(); + if fs::read(&locked).is_ok() { + eprintln!("skipped: this process can read a mode-000 file (running as root)"); + let _ = fs::remove_dir_all(&dir); + return; + } + + let loaded = load_service_records(&dir); + let strict = read_service_records(&dir); + let _ = fs::set_permissions(&locked, fs::Permissions::from_mode(0o644)); + let ids: Vec<&str> = loaded.iter().map(|r| r.service_id.as_str()).collect(); + assert_eq!(ids, ["svc-b", "svc-a"]); + assert_eq!( + strict.expect_err("the strict reader must still fail").path, + locked + ); + let _ = fs::remove_dir_all(&dir); + } + + /// A registry with three records, one of which the injected reader refuses + /// with a permission error. Runs on every platform, unlike the mode-000 test. + fn registry_with_one_record_refused() -> (PathBuf, impl Fn(&Path) -> std::io::Result>) { + let dir = test_dir("scan-refused-record"); + for (name, id, created) in [ + ("a", "svc-a", 100), + ("locked", "svc-locked", 200), + ("b", "svc-b", 300), + ] { + fs::write( + dir.join(format!("{name}.json")), + record_json(id, "vllm", 8000, "running", created), + ) + .unwrap(); + } + let refused = dir.join("locked.json"); + let read_record = move |path: &Path| { + if path == refused { + Err(std::io::Error::from(ErrorKind::PermissionDenied)) + } else { + fs::read(path) + } + }; + (dir, read_record) + } + + #[test] + fn load_keeps_the_readable_records_beside_one_that_cannot_be_read() { + let (dir, read_record) = registry_with_one_record_refused(); + let loaded = load_service_records_with(&dir, read_record); + let ids: Vec<&str> = loaded.iter().map(|r| r.service_id.as_str()).collect(); + assert_eq!(ids, ["svc-b", "svc-a"]); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn read_fails_the_pass_on_a_record_that_cannot_be_read() { + let (dir, read_record) = registry_with_one_record_refused(); + let err = read_service_records_with(&dir, read_record) + .expect_err("the strict reader must not read past an unreadable record"); + assert_eq!(err.path, dir.join("locked.json")); + assert_eq!(err.source.kind(), ErrorKind::PermissionDenied); + let _ = fs::remove_dir_all(&dir); + } + + /// The compatibility loader and the strict reader share one scan, and a + /// registry with one corrupt record and two readable ones gives both the two + /// readable records (a corrupt record is malformed in both, never a failure). + #[test] + fn load_and_read_agree_on_a_registry_with_one_corrupt_record() { + let dir = test_dir("load-read-corrupt"); + fs::write( + dir.join("a.json"), + record_json("svc-a", "vllm", 8000, "running", 100), + ) + .unwrap(); + fs::write(dir.join("corrupt.json"), "{ not valid").unwrap(); + fs::write( + dir.join("b.json"), + record_json("svc-b", "vllm", 8001, "running", 200), + ) + .unwrap(); + + let loaded: Vec = load_service_records(&dir) + .into_iter() + .map(|r| r.service_id) + .collect(); + let read = read_service_records(&dir).expect("a corrupt record is not a read failure"); + let strict: Vec = read.records.into_iter().map(|r| r.service_id).collect(); + assert_eq!(loaded, ["svc-b", "svc-a"]); + assert_eq!(strict, loaded); + assert_eq!(read.malformed, [dir.join("corrupt.json")]); + let _ = fs::remove_dir_all(&dir); + } + #[test] fn discovered_uses_registry_port_as_authority() { let dir = test_dir("port-authority"); From b3d599a46f3f7cc4a5d953ed58dfa2e01fb0f076 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 06:02:18 +0000 Subject: [PATCH 6/8] test(dash): make dash-43 fail the registry read without ever exposing an empty registry The step made the registry unreadable by deleting its directory and then writing a regular file in its place. The dashboard is a separate process that polls the registry on its own schedule, so a poll between the two calls could legitimately read an absent registry (confirmed empty), retire the model, and leave the later failed reads preserving an empty known set: the scenario could fail against a correct daemon. The step now adds one record that cannot be opened, a symbolic link to itself, in a single atomic step that leaves the model's own record and the directory in place. It fails with a loop error rather than not-found, so the strict reader fails the pass, and it works when the runner is privileged. The scenario stays Linux-only; the non-Unix build keeps a stub that panics with that reason. The in-process daemon tests keep the delete-then-write swap, which is safe there because the runner shares their current-thread runtime and cannot run between the two calls; the helper now says so. Signed-off-by: Jussi Elo --- .../tests/discovery_reconcile.rs | 6 +++ tests/e2e-cucumber/features/dash.feature | 18 ++++---- tests/e2e-cucumber/tests/e2e/dash_steps.rs | 41 ++++++++++++++----- 3 files changed, 46 insertions(+), 19 deletions(-) diff --git a/crates/rocm-dash-daemon/tests/discovery_reconcile.rs b/crates/rocm-dash-daemon/tests/discovery_reconcile.rs index 62f81ea1fe..6038bab436 100644 --- a/crates/rocm-dash-daemon/tests/discovery_reconcile.rs +++ b/crates/rocm-dash-daemon/tests/discovery_reconcile.rs @@ -497,6 +497,12 @@ async fn assert_service_is_retired(rx: &mut broadcast::Receiver) { /// Replace the registry directory with a regular file of the same name, so /// listing it fails with something other than "not found". Portable: it needs no /// permission change, so it behaves the same for a privileged user and on Windows. +/// +/// The directory is briefly absent between the two calls, which would read as a +/// confirmed-empty registry. The runner is a task on this test's own +/// current-thread runtime and reads the registry synchronously, so it cannot run +/// between them: nothing here awaits. A test that moves to a multi-thread runtime +/// must swap atomically instead. fn replace_registry_dir_with_a_file(services_dir: &std::path::Path) { std::fs::remove_dir_all(services_dir).unwrap(); std::fs::write(services_dir, b"not a directory").unwrap(); diff --git a/tests/e2e-cucumber/features/dash.feature b/tests/e2e-cucumber/features/dash.feature index 5e6ae31cad..f5e0bfef79 100644 --- a/tests/e2e-cucumber/features/dash.feature +++ b/tests/e2e-cucumber/features/dash.feature @@ -742,14 +742,16 @@ Feature: Interactive dashboard Then the dashboard exits successfully # The dashboard follows the managed-service registry on a timer. A pass that - # cannot read the registry directory says nothing about the services in it, so - # it must not retire the model that is on screen: the header warns that the - # registry is not being read, and the model stays. The directory is swapped for - # a regular file, which no path or permission setup of the runner can make - # readable. A registry that reads but is empty does retire a service; that - # contrast, the recovery path and the once-per-streak log line are asserted by - # the daemon's own tests in `crates/rocm-dash-daemon/tests/`, where the loop - # runs without a terminal. + # cannot read the registry says nothing about the services in it, so it must + # not retire the model that is on screen: the header warns that the registry + # is not being read, and the model stays. The registry is made unreadable by + # adding one record that cannot be opened (a link to itself), in a single step + # that leaves the model's own record in place, so no poll of the registry can + # see it empty or absent, and no permission setup of the runner (running as + # root) can make it readable. A registry that reads but is empty does retire + # a service; that contrast, the recovery path and the once-per-streak log + # line are asserted by the daemon's own tests in + # `crates/rocm-dash-daemon/tests/`, where the loop runs without a terminal. @id:dash-managed-model-survives-unreadable-registry @requires-os:linux Scenario: dash-43 - A managed model stays on screen while its registry cannot be read Given a running managed model is available locally diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index 048e321df9..9bfd8c35be 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -2610,14 +2610,34 @@ async fn metrics_endpoint_fails(world: &mut E2eWorld) { }); } -/// Replace the managed-service registry directory with a regular file of the -/// same name, so listing it fails the way a broken mount or a permission change -/// would, and wait for the failure to reach the screen. The header warnings -/// badge growing is the evidence that a frame the daemon assembled after the -/// failed pass is on display (the daemon adds a warning for as long as the -/// registry cannot be read); the count is read before the swap because other -/// warnings, such as no amd-smi on a GPU-less runner, are already present and -/// constant. +/// Create `path` as a symbolic link to itself, so opening it fails with a loop +/// error. The scenario that uses it is `@requires-os:linux`. +#[cfg(unix)] +fn make_unreadable_record(path: &std::path::Path) { + std::os::unix::fs::symlink(path, path).expect("failed to plant the unreadable record"); +} + +#[cfg(not(unix))] +fn make_unreadable_record(_path: &std::path::Path) { + panic!("the unreadable-registry scenario is Linux-only (@requires-os:linux)"); +} + +/// Make the managed-service registry unreadable and wait for the failure to +/// reach the screen. The registry is made unreadable by adding one record file +/// that cannot be opened: a symbolic link to itself, which fails with a loop +/// error rather than "not found" (a dangling link would read as a record that +/// was removed on purpose, and be skipped). That is a single atomic step that +/// leaves the existing record and the directory in place, and it works for a +/// privileged user, which a permission change does not. The dashboard process +/// polls the registry on its own schedule, so the step must never expose a +/// registry with the records gone: deleting the directory before putting +/// something in its place would let a poll in between read an absent registry, +/// which is a confirmed-empty one, and retire the model the scenario expects to +/// survive. The header warnings badge growing is the evidence that a frame the +/// daemon assembled after the failed pass is on display (the daemon adds a +/// warning for as long as the registry cannot be read); the count is read before +/// the change because other warnings, such as no amd-smi on a GPU-less runner, +/// are already present and constant. #[when("the managed-service registry becomes unreadable")] async fn managed_registry_becomes_unreadable(world: &mut E2eWorld) { let before = warning_badge_count(&session(world).screen_text()); @@ -2628,9 +2648,8 @@ async fn managed_registry_becomes_unreadable(world: &mut E2eWorld) { .path() .join("data") .join("services"); - std::fs::remove_dir_all(&services_dir).expect("failed to remove the registry directory"); - std::fs::write(&services_dir, b"not a directory") - .expect("failed to put a file where the registry directory was"); + let looping = services_dir.join("unreadable.json"); + make_unreadable_record(&looping); session(world) .wait_for_screen_where( From 913a53e5572b7b2795711d098c1851a53bd563ef Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 11:25:02 +0000 Subject: [PATCH 7/8] fix(dash-daemon): keep adding services that read while one registry record cannot be opened A record file the daemon cannot open (a root-owned file left by a privileged serve, a symlink loop) failed every discovery pass, so the managed registry stopped following new services for as long as the file stayed. The rule that a failed read never retires anything is unchanged, but the failure now carries the records that did read, and the loop upserts and announces those services while retiring none. read_service_records keeps scanning past an unreadable record and returns the first failure with the partial read. The best-effort loader no longer needs its own mode: it uses the partial read of a failed scan. In the run loop, a failed pass upserts the services it saw and adds them to the known set, so a later complete read can retire them; the services it did not see stay known and keep their non-vLLM marks. A service that stopped, or whose record was removed, keeps its row until the registry reads in full, and the header warning names the unreadable path throughout; the module docs state this trade-off. State matrix, each row asserting the announcements and the snapshot together: one record unreadable with a new valid record added, with a known record removed (then fixed, which retires it and clears the warning), and with the known service's own record becoming unreadable (then readable again, announcing nothing). Shown red by removing the upsert on a failed pass, by retiring on a failed pass, by not adding the seen services to the known set, by not refreshing the non-vLLM marks and by dropping the partial read. Signed-off-by: Jussi Elo --- crates/rocm-dash-daemon/src/registry.rs | 113 +++++++---- crates/rocm-dash-daemon/src/runner.rs | 108 +++++++++- .../tests/discovery_reconcile.rs | 185 ++++++++++++++++++ docs/architecture.md | 2 +- 4 files changed, 357 insertions(+), 51 deletions(-) diff --git a/crates/rocm-dash-daemon/src/registry.rs b/crates/rocm-dash-daemon/src/registry.rs index 29aba916e8..1bb31a4512 100644 --- a/crates/rocm-dash-daemon/src/registry.rs +++ b/crates/rocm-dash-daemon/src/registry.rs @@ -29,6 +29,21 @@ //! (the same assumption the Docker-discovery path already makes). A managed //! service on a non-loopback host is surfaced but its metrics are scraped at the //! local address. Per-host scrape targeting is tracked as a D7 follow-up. +//! +//! **A registry that cannot be read in full (trade-off):** a record file that +//! exists but cannot be opened (a root-owned file left by `sudo rocm serve`, a +//! symlink loop) makes [`read_service_records`] fail the pass, because the +//! service that file held cannot be told apart from one that is gone, and the +//! daemon's rule is that a failed read never retires anything. The failure +//! carries what the pass did read ([`RegistryReadError::partial`]), so the daemon +//! still adds and refreshes those services: a new `rocm serve` appears on the +//! next discovery tick even beside a stale unreadable record. What stays frozen, +//! for as long as the file is unreadable (there is no timeout), is retirement: +//! a service that has stopped or whose record was removed keeps its row (with its +//! last live fields) until the registry reads in full, and the header warning +//! names the unreadable path throughout. Retiring on a deadline would reintroduce +//! the wrongly vanishing row this rule exists to prevent, so the freeze is +//! visible rather than bounded. use std::collections::HashSet; use std::fs; @@ -131,15 +146,22 @@ pub struct RegistryRead { pub malformed: Vec, } -/// The registry could not be read, so what it holds is unknown. This is not the -/// same as an empty registry: a caller that retires services which are missing -/// from a read must not do so on this. +/// The registry could not be read in full, so what it holds is unknown. +/// +/// This is not the same as an empty registry: a caller that retires services +/// which are missing from a read must not do so on this. It may still use what was read +/// around the failure ([`RegistryReadError::partial`]) to add or refresh services, +/// because a record that was read is evidence that its service exists, whereas a +/// record that was not read is no evidence of anything. #[derive(Debug)] pub struct RegistryReadError { - /// The directory that could not be listed, or the record file that could not - /// be read. + /// The directory that could not be listed, or the first record file that + /// could not be read. pub path: PathBuf, pub source: std::io::Error, + /// What the pass did read: every well-formed record it could open, and the + /// malformed ones. Empty when the directory itself could not be listed. + pub partial: RegistryRead, } impl std::fmt::Display for RegistryReadError { @@ -154,27 +176,19 @@ impl std::error::Error for RegistryReadError { } } -/// What a scan does with a record file that exists but cannot be read. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum UnreadableRecord { - /// The whole pass fails: nothing is known about the service the file held. - Fail, - /// The file is skipped and the other records still load. - Skip, -} - /// Read every managed-service record under `services_dir`, telling a registry /// that is confirmed empty apart from one that could not be read. /// /// * A directory that does not exist is confirmed empty: it is created the first /// time something is served, so its absence means there are no services. A /// directory that exists but cannot be listed (or a regular file where it -/// should be) is a read failure. +/// should be) is a read failure with nothing partial to offer. /// * A record file that vanishes between the listing and the read was removed /// on purpose and is skipped. Any other failure to read a record (a permission -/// or I/O error) is a read failure for the whole pass: the record's service -/// cannot be told apart from one that is gone, and nothing is known about which -/// service the unreadable file belonged to. +/// or I/O error) is a read failure for the pass: the record's service cannot be +/// told apart from one that is gone, and nothing is known about which service +/// the unreadable file belonged to. The scan still reads the remaining records, +/// so [`RegistryReadError::partial`] holds every record that could be read. /// * A record that reads but does not parse is malformed. It is reported in /// [`RegistryRead::malformed`] and the other records still load, because the /// rocm-cli writer replaces a record atomically, so a corrupt file is corrupt @@ -192,16 +206,6 @@ pub fn read_service_records(services_dir: &Path) -> Result std::io::Result>, -) -> Result { - scan_registry(services_dir, read_record, UnreadableRecord::Fail) -} - -/// The scan behind [`read_service_records`] and [`load_service_records`]; the two -/// differ only in what they do with a record file that cannot be read. -fn scan_registry( - services_dir: &Path, - read_record: impl Fn(&Path) -> std::io::Result>, - on_unreadable: UnreadableRecord, ) -> Result { let entries = match fs::read_dir(services_dir) { Ok(entries) => entries, @@ -216,20 +220,21 @@ fn scan_registry( Err(RegistryReadError { path: services_dir.to_path_buf(), source, + partial: RegistryRead::default(), }) }; } }; let mut read = RegistryRead::default(); + // The first failure; later ones add nothing a caller can act on. + let mut failure: Option<(PathBuf, std::io::Error)> = None; for entry in entries { let entry = match entry { Ok(entry) => entry, - Err(_) if on_unreadable == UnreadableRecord::Skip => continue, Err(source) => { - return Err(RegistryReadError { - path: services_dir.to_path_buf(), - source, - }); + // The rest of the listing is unknown, so stop reading it. + failure.get_or_insert_with(|| (services_dir.to_path_buf(), source)); + break; } }; let path = entry.path(); @@ -239,8 +244,10 @@ fn scan_registry( let bytes = match read_record(&path) { Ok(bytes) => bytes, Err(e) if e.kind() == ErrorKind::NotFound => continue, - Err(_) if on_unreadable == UnreadableRecord::Skip => continue, - Err(source) => return Err(RegistryReadError { path, source }), + Err(source) => { + failure.get_or_insert((path, source)); + continue; + } }; match serde_json::from_slice::(&bytes) { Ok(record) => read.records.push(record), @@ -250,7 +257,14 @@ fn scan_registry( read.records .sort_by_key(|record| std::cmp::Reverse(record.created_at_unix_ms)); read.malformed.sort(); - Ok(read) + match failure { + None => Ok(read), + Some((path, source)) => Err(RegistryReadError { + path, + source, + partial: read, + }), + } } /// Load every managed-service record under `services_dir`, newest first. @@ -260,8 +274,8 @@ fn scan_registry( /// an empty list, and a record that cannot be read or parsed is skipped on its /// own, so one bad file never hides the services beside it. The daemon's /// discovery loop must not use this, because it cannot tell an unreadable -/// registry from an empty one; it calls [`read_service_records`], which fails the -/// pass on an unreadable record instead. +/// registry from an empty one; it calls [`read_service_records`] and keeps the +/// services it could not confirm gone. pub fn load_service_records(services_dir: &Path) -> Vec { load_service_records_with(services_dir, |path| fs::read(path)) } @@ -272,9 +286,9 @@ fn load_service_records_with( services_dir: &Path, read_record: impl Fn(&Path) -> std::io::Result>, ) -> Vec { - scan_registry(services_dir, read_record, UnreadableRecord::Skip) - .map(|read| read.records) - .unwrap_or_default() + read_service_records_with(services_dir, read_record) + .unwrap_or_else(|failed| failed.partial) + .records } /// Convert a managed-service record into a [`DiscoveredService`] for the @@ -498,6 +512,10 @@ mod tests { .expect_err("a file where the registry should be must not read as empty"); assert_eq!(err.path, not_a_dir); assert!(err.to_string().contains("cannot read"), "{err}"); + assert!( + err.partial.records.is_empty() && err.partial.malformed.is_empty(), + "a directory that cannot be listed has nothing partial to offer" + ); // The best-effort wrapper keeps its old answer for the one-shot readers. assert!(load_service_records(¬_a_dir).is_empty()); let _ = fs::remove_dir_all(&dir); @@ -575,6 +593,7 @@ mod tests { let _ = fs::set_permissions(&locked, fs::Permissions::from_mode(0o644)); let err = result.expect_err("an unreadable record must not read as a missing one"); assert_eq!(err.path, locked); + assert_eq!(err.partial.records.len(), 1, "the readable record is kept"); let _ = fs::remove_dir_all(&dir); } @@ -662,9 +681,19 @@ mod tests { fn read_fails_the_pass_on_a_record_that_cannot_be_read() { let (dir, read_record) = registry_with_one_record_refused(); let err = read_service_records_with(&dir, read_record) - .expect_err("the strict reader must not read past an unreadable record"); + .expect_err("the strict reader must not read an unreadable record as a missing one"); assert_eq!(err.path, dir.join("locked.json")); assert_eq!(err.source.kind(), ErrorKind::PermissionDenied); + // The failure does not stop the scan: both readable records are in the + // partial read, newest first, whichever side of the locked one they sort. + let ids: Vec<&str> = err + .partial + .records + .iter() + .map(|r| r.service_id.as_str()) + .collect(); + assert_eq!(ids, ["svc-b", "svc-a"]); + assert!(err.partial.malformed.is_empty()); let _ = fs::remove_dir_all(&dir); } diff --git a/crates/rocm-dash-daemon/src/runner.rs b/crates/rocm-dash-daemon/src/runner.rs index 3c212d30a7..fde02a1c37 100644 --- a/crates/rocm-dash-daemon/src/runner.rs +++ b/crates/rocm-dash-daemon/src/runner.rs @@ -590,10 +590,13 @@ pub async fn run_loop( // scrape below skips them. A model served via `rocm serve` thus appears // in the dashboard and is scraped for `gen_tps` without Docker. // - // Like the Docker source above, a pass that could not read the registry - // reconciles nothing: what is known stays known (no `Gone`, no repeat - // `Discovered`), and the instances keep the data they hold. Only a - // registry that was read, even an empty one, retires what it lacks. + // A pass that could not read the registry in full retires nothing, as + // with the Docker source above: what is known stays known (no `Gone`), + // and the instances keep the data they hold. Unlike Docker, whose listing + // is all or nothing, a registry can fail on one record while the rest + // read, so the readable records still add and refresh their services + // (announced once, as ever). Only a registry that was read in full, even + // an empty one, retires what it lacks. if let Some(services_dir) = opts.services_dir.as_ref() && discovery_due { @@ -613,7 +616,19 @@ pub async fn run_loop( discovery.known_services = disc.seen; registry_warn = None; } - Err(e) => registry_warn = Some(registry_read_warning(&e)), + Err(e) => { + // The records that did read still add and refresh their + // services; nothing is retired, because the one that did + // not read could be any of the known ones. + let disc = crate::registry::discover_managed_services(&e.partial.records); + reconcile.upsert_set( + DiscoverySource::Managed, + &disc.svcs, + &discovery.known_services, + ); + discovery.keep_through_partial_registry(disc); + registry_warn = Some(registry_read_warning(&e)); + } } } if tick_count.is_multiple_of(instance_ticks) { @@ -1037,6 +1052,19 @@ struct DiscoveryState { registry_health: RegistryHealth, } +impl DiscoveryState { + /// Record what a pass that read only part of the registry found. The services + /// it saw join the known set, so a later complete read can retire them, and + /// the known ones it did not see stay; the non-vLLM marks of the services it + /// saw are refreshed and the others kept, so a kept non-vLLM service is still + /// left out of the vLLM scrape. + fn keep_through_partial_registry(&mut self, disc: crate::registry::ManagedDiscovery) { + self.managed_non_vllm.retain(|id| !disc.seen.contains(id)); + self.managed_non_vllm.extend(disc.non_vllm); + self.known_services.extend(disc.seen); + } +} + /// What the managed-service registry's reads have reported so far. A failure /// that lasts is logged on its first pass only, the recovery once, and a set of /// malformed records each time it changes; the dashboard's warning badge, which @@ -1215,6 +1243,22 @@ impl Reconcile<'_> { svcs: &[DiscoveredService], known: &HashSet, seen: &HashSet, + ) { + self.upsert_set(source, svcs, known); + for gone in known.difference(seen) { + self.retire(source, gone); + } + } + + /// The half of [`Self::reconcile_set`] that adds and refreshes: upsert every + /// service in `svcs` and announce the ones `known` did not have. Retires + /// nothing, which is what a pass that could only see part of its source needs + /// (see [`DiscoveryState::keep_through_partial_registry`]). + fn upsert_set( + &mut self, + source: DiscoverySource, + svcs: &[DiscoveredService], + known: &HashSet, ) { for svc in svcs { let same_endpoint = self @@ -1231,9 +1275,6 @@ impl Reconcile<'_> { self.announce(source, svc); } } - for gone in known.difference(seen) { - self.retire(source, gone); - } } /// Reconcile the probe-based Lemonade source, which has at most one service. @@ -2403,9 +2444,60 @@ mod tests { Err(crate::registry::RegistryReadError { path: PathBuf::from("services"), source: std::io::Error::other(why), + partial: crate::registry::RegistryRead::default(), }) } + /// State matrix for what a partially read registry does to the discovery + /// state, one assertion group per row: a service it saw joins the known set + /// and its non-vLLM mark follows the record; one it did not see stays known + /// and keeps its mark; nothing leaves either set. + #[test] + fn partial_registry_read_adds_what_it_saw_and_forgets_nothing() { + let mut state = DiscoveryState { + known_services: id_set(&["old-vllm", "old-lemonade", "flips"]), + managed_non_vllm: id_set(&["old-lemonade", "flips"]), + ..DiscoveryState::default() + }; + + state.keep_through_partial_registry(crate::registry::ManagedDiscovery { + seen: id_set(&["new-lemonade", "new-vllm", "flips"]), + non_vllm: id_set(&["new-lemonade"]), + ..Default::default() + }); + + assert_eq!( + state.known_services, + id_set(&[ + "old-vllm", + "old-lemonade", + "flips", + "new-lemonade", + "new-vllm" + ]), + "the unseen services stay known and the new ones join them" + ); + assert_eq!( + state.managed_non_vllm, + id_set(&["old-lemonade", "new-lemonade"]), + "an unseen non-vLLM service keeps its mark, a seen one follows its record" + ); + } + + #[test] + fn partial_registry_read_that_saw_nothing_changes_nothing() { + let mut state = DiscoveryState { + known_services: id_set(&["a", "b"]), + managed_non_vllm: id_set(&["b"]), + ..DiscoveryState::default() + }; + + state.keep_through_partial_registry(crate::registry::ManagedDiscovery::default()); + + assert_eq!(state.known_services, id_set(&["a", "b"])); + assert_eq!(state.managed_non_vllm, id_set(&["b"])); + } + #[test] fn registry_health_reports_the_first_failure_of_a_streak_once() { let mut health = RegistryHealth::default(); diff --git a/crates/rocm-dash-daemon/tests/discovery_reconcile.rs b/crates/rocm-dash-daemon/tests/discovery_reconcile.rs index 6038bab436..570c0a656c 100644 --- a/crates/rocm-dash-daemon/tests/discovery_reconcile.rs +++ b/crates/rocm-dash-daemon/tests/discovery_reconcile.rs @@ -765,6 +765,191 @@ async fn managed_malformed_record_does_not_hide_the_other_services() { ); } +// ── Managed-service registry: one record that cannot be opened ────────────── +// +// State matrix for a registry where some record cannot be read, one test per +// row (the rows with no unreadable record, an empty registry, and a directory +// that cannot be listed are the tests above): +// +// | State | Announced | Snapshots list | Header warning | +// | -------------------------------------------------- | ---------------- | ----------------- | -------------- | +// | one record unreadable, a new valid record appears | the new one once | known + new | yes | +// | one record unreadable, a known record is removed | nothing | known stays | yes | +// | the same, then the unreadable record is fixed | gone, once | the removed one goes | clears | +// | the known service's own record becomes unreadable | nothing | known stays | yes | +// | ... and then reads again | nothing | known stays | clears | +// +// A record that cannot be opened is a symbolic link to itself, which fails with a +// loop error (not "not found", which would read as a record removed on purpose), +// needs no permission change and so also works when the tests run as root. + +/// Make `.json` in the registry something that cannot be opened. +#[cfg(unix)] +fn write_unreadable_record(services_dir: &std::path::Path, name: &str) { + let file = format!("{name}.json"); + // A relative target, so the link keeps pointing at itself when renamed. + std::os::unix::fs::symlink(&file, services_dir.join(&file)).unwrap(); +} + +/// What the stream and the snapshots showed, for every service, over a window of +/// snapshots. +#[cfg(unix)] +#[derive(Default)] +struct Roster { + /// Every discovered/gone announcement, in order, with the service id. + announcements: Vec<(String, &'static str)>, + snapshots: usize, + /// Snapshots in the window that did not list [`SVC_ID`]. + missing_known: usize, + /// The ids the last snapshot in the window listed, sorted. + ids: Vec, + /// [`SVC_ID`] as the last snapshot in the window listed it. + known: Option, + /// The warnings the last snapshot in the window carried. + warnings: Vec, +} + +#[cfg(unix)] +impl Roster { + fn registry_warning(&self) -> Option<&String> { + self.warnings + .iter() + .find(|w| w.starts_with("managed registry")) + } +} + +/// Watch `count` snapshots, recording every announcement and what the snapshots +/// listed, under one deadline. +#[cfg(unix)] +async fn watch_roster(rx: &mut broadcast::Receiver, count: usize) -> Roster { + let mut roster = Roster::default(); + observe( + rx, + "a window of snapshots", + &mut roster, + |r, ev| { + match ev { + Event::InstanceDiscovered(inst) => r + .announcements + .push((inst.container_id.clone(), "discovered")), + Event::InstanceGone { container_id } => { + r.announcements.push((container_id.clone(), "gone")); + } + Event::Snapshot(snap) => { + r.snapshots += 1; + r.known = instance(snap, SVC_ID).cloned(); + r.missing_known += usize::from(r.known.is_none()); + r.ids = snap + .instances + .iter() + .map(|i| i.container_id.clone()) + .collect(); + r.ids.sort(); + r.warnings.clone_from(&snap.warnings); + } + _ => {} + } + r.snapshots >= count + }, + |r| { + format!( + "{} snapshots, listing {:?}, announcements {:?}", + r.snapshots, r.ids, r.announcements + ) + }, + ) + .await; + roster +} + +/// A new service is added while one other record cannot be opened: the readable +/// records still add their services, announced once, and nothing is retired. +#[cfg(unix)] +#[tokio::test] +async fn managed_unreadable_record_does_not_hide_a_new_service() { + let mut live = start_live_managed().await; + let port = live.last_port(); + + write_unreadable_record(&live.services_dir, "stale"); + write_record_for(&live.services_dir, "second-svc", port); + let roster = watch_roster(&mut live.rx, OUTAGE_SNAPSHOTS).await; + + assert_eq!( + roster.announcements, + [("second-svc".to_string(), "discovered")], + "the new service is announced once, and nothing is announced gone" + ); + assert_eq!( + roster.missing_known, 0, + "a snapshot dropped the known service" + ); + assert_eq!(roster.ids, [SVC_ID.to_string(), "second-svc".to_string()]); + let warning = roster + .registry_warning() + .expect("the header should name the unreadable record"); + assert!(warning.contains("stale.json"), "{warning}"); +} + +/// While a record cannot be opened a service whose record is removed keeps its +/// row, because the unreadable record could be any known service's; once the +/// record reads again the registry is read in full and the removed service is +/// retired, with the warning cleared. +#[cfg(unix)] +#[tokio::test] +async fn managed_unreadable_record_defers_retirement_until_it_reads_again() { + let mut live = start_live_managed().await; + + write_unreadable_record(&live.services_dir, "stale"); + std::fs::remove_file(live.services_dir.join(format!("{SVC_ID}.json"))).unwrap(); + let during = watch_roster(&mut live.rx, OUTAGE_SNAPSHOTS).await; + + assert_eq!( + during.announcements, + Vec::<(String, &str)>::new(), + "a removed record must not retire anything while another cannot be read" + ); + assert_eq!(during.missing_known, 0, "a snapshot dropped the service"); + assert_eq!(during.ids, [SVC_ID.to_string()]); + assert!(during.registry_warning().is_some(), "{:?}", during.warnings); + + std::fs::remove_file(live.services_dir.join("stale.json")).unwrap(); + assert_service_is_retired(&mut live.rx).await; + let after = watch_roster(&mut live.rx, SETTLE_SNAPSHOTS).await; + assert_eq!(after.ids, Vec::::new()); + assert_eq!( + after.registry_warning(), + None, + "the warning must clear once the registry reads in full" + ); +} + +/// The known service's own record becomes unreadable (here it is replaced by a +/// self-referencing link, atomically): the service keeps its row and its live +/// fields, and is not announced again when the record reads again. +#[cfg(unix)] +#[tokio::test] +async fn managed_own_record_becoming_unreadable_keeps_the_instance_and_recovers_quietly() { + let mut live = start_live_managed().await; + let port = live.last_port(); + + let tmp_link = live.services_dir.join(format!("{SVC_ID}.json.tmp")); + std::os::unix::fs::symlink(format!("{SVC_ID}.json"), &tmp_link).unwrap(); + std::fs::rename(&tmp_link, live.services_dir.join(format!("{SVC_ID}.json"))).unwrap(); + let during = watch_roster(&mut live.rx, OUTAGE_SNAPSHOTS).await; + + assert_eq!(during.announcements, Vec::<(String, &str)>::new()); + assert_eq!(during.missing_known, 0, "a snapshot dropped the service"); + assert_still_live(during.known.as_ref().expect("the service is listed")); + assert!(during.registry_warning().is_some(), "{:?}", during.warnings); + + write_record(&live.services_dir, port); + let after = watch_roster(&mut live.rx, OUTAGE_SNAPSHOTS).await; + assert_eq!(after.announcements, Vec::<(String, &str)>::new()); + assert_eq!(after.missing_known, 0); + assert_eq!(after.ids, [SVC_ID.to_string()]); + assert_eq!(after.registry_warning(), None); +} + // ── Lemonade endpoint ─────────────────────────────────────────────────────── fn lemonade_responder(path: &str) -> String { diff --git a/docs/architecture.md b/docs/architecture.md index e7ddc2ad4d..0e7a675d38 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -280,7 +280,7 @@ The other three dashboard crates are described at crate level here, not module b - [`rocm-dash-collectors`](../crates/rocm-dash-collectors/src/lib.rs) — the IO-heavy collector implementations behind `rocm-dash-core`'s traits (amd-smi, Docker, host metrics, Lemonade and vLLM scrapers, and on Windows the [`winperf.rs`](../crates/rocm-dash-collectors/src/winperf.rs) perf-counter/ADLX GPU collector) plus the benchmark load generator; used by `rocm-dash-daemon` and, on Windows, by `rocm-dash-tui` in-process. - [`rocm-dash-core`](../crates/rocm-dash-core/src/lib.rs) — pure types, traits, schemas and the reducer, with no rendering dependencies. -- [`rocm-dash-daemon`](../crates/rocm-dash-daemon/src/lib.rs) — the daemon library: [`runner.rs`](../crates/rocm-dash-daemon/src/runner.rs) drives the collectors on tick cadences and broadcasts snapshots, and [`server.rs`](../crates/rocm-dash-daemon/src/server.rs) streams them to TUI clients as NDJSON. [`registry.rs`](../crates/rocm-dash-daemon/src/registry.rs) reads the managed-service records `rocm serve` writes and tells a registry that is confirmed empty (no directory, or no records) from one that could not be read: the run loop retires a managed service that an empty registry lacks, but when a pass cannot read the registry it keeps the known services and their data, as it does when Docker discovery errors, and the header warns until a read succeeds. `apps/rocm`'s [`dash.rs`](../apps/rocm/src/dash.rs) drives it. +- [`rocm-dash-daemon`](../crates/rocm-dash-daemon/src/lib.rs) — the daemon library: [`runner.rs`](../crates/rocm-dash-daemon/src/runner.rs) drives the collectors on tick cadences and broadcasts snapshots, and [`server.rs`](../crates/rocm-dash-daemon/src/server.rs) streams them to TUI clients as NDJSON. [`registry.rs`](../crates/rocm-dash-daemon/src/registry.rs) reads the managed-service records `rocm serve` writes and tells a registry that is confirmed empty (no directory, or no records) from one that could not be read: the run loop retires a managed service that an empty registry lacks, but when a pass cannot read the registry in full (a directory that cannot be listed, or a record file that cannot be opened) it retires nothing: the known services and their data stay, as when Docker discovery errors, the records that did read still add and refresh their services, and the header warns until a read succeeds. `apps/rocm`'s [`dash.rs`](../apps/rocm/src/dash.rs) drives it. ### `crates/rocm-engine-protocol` — engine IPC protocol From ea75b804125da86eb2bf8c7eb35fa9c6734c033d Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Fri, 6 Nov 2026 11:25:39 +0000 Subject: [PATCH 8/8] test(dash): add dash-44 for a new model appearing beside an unreadable registry record With the registry made unreadable by one record that cannot be opened, a second model whose record can be read is written into the registry, and the scenario waits for it to appear on screen while the first model stays. The second record points at a disposable process, because the harness teardown runs the real stop command for every record but the first model's, and a record naming the harness itself would signal the harness. Shown red by removing the daemon's upsert on a failed pass: the new model never appears. dash-43 is unchanged; the rest of the matrix (a removed record is not retired until the registry reads in full, quiet recovery) is asserted by the daemon's own tests. Signed-off-by: Jussi Elo --- tests/e2e-cucumber/features/dash.feature | 21 +++++++ tests/e2e-cucumber/tests/e2e/dash_steps.rs | 66 ++++++++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/tests/e2e-cucumber/features/dash.feature b/tests/e2e-cucumber/features/dash.feature index f5e0bfef79..227cbc16d6 100644 --- a/tests/e2e-cucumber/features/dash.feature +++ b/tests/e2e-cucumber/features/dash.feature @@ -763,3 +763,24 @@ Feature: Interactive dashboard And the managed model stays displayed When the user quits the dashboard Then the dashboard exits successfully + + # The same registry failure, from the other side: a service whose record can be + # read is still added while some other record cannot be opened. The dashboard + # reads what it can and retires nothing, so a model that is started beside a + # stale unreadable record appears on screen (and the first model stays). The + # daemon's own tests in `crates/rocm-dash-daemon/tests/` assert the rest of the + # matrix: a record removed during the failure is not retired until the registry + # reads in full, and a record that becomes unreadable is not announced again + # when it reads again. + @id:dash-new-model-appears-beside-unreadable-record @requires-os:linux + Scenario: dash-44 - A newly served model appears while another record in the registry cannot be read + Given a running managed model is available locally + When the user opens the dashboard + And the user opens the Observe view + Then the managed model is displayed + When the managed-service registry becomes unreadable + And another managed model is served + Then the other managed model is displayed + And the managed model stays displayed + When the user quits the dashboard + Then the dashboard exits successfully diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index b213855ac7..c19c9c139e 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -2703,6 +2703,72 @@ async fn managed_model_stays_displayed(world: &mut E2eWorld) { } } +/// The model `another managed model is served` adds beside the first one. +const OTHER_MODEL: &str = "TestModel/E2E-2B"; + +/// Serve a second model: write a record for it into the registry, as `rocm serve` +/// does, by copying the first model's record under another service id and name. +/// It is written to a temporary name and renamed into place, so a poll of the +/// registry never reads half a record (which would read as a malformed one, not +/// as the new service). +/// +/// The record points at a disposable `sleep` child, not at the harness, for the +/// reason `managed_model_record_points_at_disposable_process` gives: the World's +/// teardown runs `rocm services stop --yes` for every record but the first +/// model's, and that signals whatever process the record names. The child is kept +/// in `recorded_process`, which teardown reaps. +#[when("another managed model is served")] +async fn another_managed_model_is_served(world: &mut E2eWorld) { + let child = std::process::Command::new("sleep") + .arg("300") + .spawn() + .expect("failed to spawn the disposable process for the second model"); + let pid = child.id(); + let start_ticks = rocm_core::process_start_ticks(pid); + world.recorded_process = Some(child); + let services_dir = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("data") + .join("services"); + let first = std::fs::read(services_dir.join("e2e-mock.json")) + .expect("the first model's record should be in the registry"); + let mut record: serde_json::Value = + serde_json::from_slice(&first).expect("the first model's record should be JSON"); + record["service_id"] = "e2e-second".into(); + record["model_ref"] = OTHER_MODEL.into(); + record["canonical_model_id"] = OTHER_MODEL.into(); + record["supervisor_pid"] = pid.into(); + record["engine_pid"] = pid.into(); + record["supervisor_start_ticks"] = serde_json::to_value(start_ticks).unwrap(); + let staged = services_dir.join("e2e-second.json.tmp"); + std::fs::write( + &staged, + serde_json::to_vec_pretty(&record).expect("failed to serialize the second record"), + ) + .expect("failed to stage the second record"); + std::fs::rename(&staged, services_dir.join("e2e-second.json")) + .expect("failed to publish the second record"); +} + +/// The second model is on screen although the registry still holds a record that +/// cannot be opened. The step before it left the header warning in place, so the +/// registry is known to be failing while this waits. +#[then("the other managed model is displayed")] +async fn other_managed_model_displayed(world: &mut E2eWorld) { + session(world) + .wait_for_screen(OTHER_MODEL, default_timeout()) + .await + .unwrap_or_else(|e| { + panic!( + "the newly served model never appeared while one registry record could not \ + be read: {e}" + ) + }); +} + /// EAI-7960 principal regression assertion. /// /// The frame under assertion is provably post-failure (the header warnings