diff --git a/crates/rocm-dash-daemon/src/registry.rs b/crates/rocm-dash-daemon/src/registry.rs index 44e6275fb5..1bb31a4512 100644 --- a/crates/rocm-dash-daemon/src/registry.rs +++ b/crates/rocm-dash-daemon/src/registry.rs @@ -29,10 +29,26 @@ //! (the same assumption the Docker-discovery path already makes). A managed //! service on a non-loopback host is surfaced but its metrics are scraped at the //! local address. Per-host scrape targeting is tracked as a D7 follow-up. +//! +//! **A registry that cannot be read in full (trade-off):** a record file that +//! exists but cannot be opened (a root-owned file left by `sudo rocm serve`, a +//! symlink loop) makes [`read_service_records`] fail the pass, because the +//! service that file held cannot be told apart from one that is gone, and the +//! daemon's rule is that a failed read never retires anything. The failure +//! carries what the pass did read ([`RegistryReadError::partial`]), so the daemon +//! still adds and refreshes those services: a new `rocm serve` appears on the +//! next discovery tick even beside a stale unreadable record. What stays frozen, +//! for as long as the file is unreadable (there is no timeout), is retirement: +//! a service that has stopped or whose record was removed keeps its row (with its +//! last live fields) until the registry reads in full, and the header warning +//! names the unreadable path throughout. Retiring on a deadline would reintroduce +//! the wrongly vanishing row this rule exists to prevent, so the freeze is +//! visible rather than bounded. use std::collections::HashSet; use std::fs; -use std::path::Path; +use std::io::ErrorKind; +use std::path::{Path, PathBuf}; use rocm_dash_collectors::engine_registry::EngineKind; use rocm_dash_core::metrics::{InstanceStatus, StartupPhase}; @@ -119,30 +135,160 @@ fn instance_status_for_record(status: &str, startup_phase: Option<&str>) -> Inst } } -/// Load every managed-service record under `services_dir`, newest first. +/// What one read of the registry directory found. +#[derive(Debug, Default)] +pub struct RegistryRead { + /// Every well-formed record, newest first. + pub records: Vec, + /// `*.json` files that were read but are not a JSON object this module can + /// parse, sorted by path. They are left out of `records`, and nothing else is + /// hidden by them: one corrupt record does not stop the others loading. + pub malformed: Vec, +} + +/// The registry could not be read in full, so what it holds is unknown. /// -/// Best-effort and side-effect-free: a missing directory yields an empty list, -/// and an individual unreadable/malformed `*.json` is skipped rather than -/// failing the whole load (mirrors the canonical rocm-cli `load_managed_services` -/// semantics, minus the engine-state status overlay which the supervisor owns). -pub fn load_service_records(services_dir: &Path) -> Vec { - let Ok(entries) = fs::read_dir(services_dir) else { - return Vec::new(); +/// This is not the same as an empty registry: a caller that retires services +/// which are missing from a read must not do so on this. It may still use what was read +/// around the failure ([`RegistryReadError::partial`]) to add or refresh services, +/// because a record that was read is evidence that its service exists, whereas a +/// record that was not read is no evidence of anything. +#[derive(Debug)] +pub struct RegistryReadError { + /// The directory that could not be listed, or the first record file that + /// could not be read. + pub path: PathBuf, + pub source: std::io::Error, + /// What the pass did read: every well-formed record it could open, and the + /// malformed ones. Empty when the directory itself could not be listed. + pub partial: RegistryRead, +} + +impl std::fmt::Display for RegistryReadError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "cannot read {}: {}", self.path.display(), self.source) + } +} + +impl std::error::Error for RegistryReadError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + Some(&self.source) + } +} + +/// Read every managed-service record under `services_dir`, telling a registry +/// that is confirmed empty apart from one that could not be read. +/// +/// * A directory that does not exist is confirmed empty: it is created the first +/// time something is served, so its absence means there are no services. A +/// directory that exists but cannot be listed (or a regular file where it +/// should be) is a read failure with nothing partial to offer. +/// * A record file that vanishes between the listing and the read was removed +/// on purpose and is skipped. Any other failure to read a record (a permission +/// or I/O error) is a read failure for the pass: the record's service cannot be +/// told apart from one that is gone, and nothing is known about which service +/// the unreadable file belonged to. The scan still reads the remaining records, +/// so [`RegistryReadError::partial`] holds every record that could be read. +/// * A record that reads but does not parse is malformed. It is reported in +/// [`RegistryRead::malformed`] and the other records still load, because the +/// rocm-cli writer replaces a record atomically, so a corrupt file is corrupt +/// for good rather than half-written. +/// +/// Side-effect-free, and the canonical rocm-cli `load_managed_services` semantics +/// minus the engine-state status overlay, which the supervisor owns. +pub fn read_service_records(services_dir: &Path) -> Result { + read_service_records_with(services_dir, |path| fs::read(path)) +} + +/// [`read_service_records`] with the record reader injected, so a test can make +/// one record unreadable on every platform (a mode-000 file works only on Unix, +/// and not as root). +fn read_service_records_with( + services_dir: &Path, + read_record: impl Fn(&Path) -> std::io::Result>, +) -> Result { + let entries = match fs::read_dir(services_dir) { + Ok(entries) => entries, + Err(source) => { + let absent = matches!( + fs::metadata(services_dir), + Err(ref e) if e.kind() == ErrorKind::NotFound + ); + return if absent { + Ok(RegistryRead::default()) + } else { + Err(RegistryReadError { + path: services_dir.to_path_buf(), + source, + partial: RegistryRead::default(), + }) + }; + } }; - let mut records = Vec::new(); - for entry in entries.flatten() { + let mut read = RegistryRead::default(); + // The first failure; later ones add nothing a caller can act on. + let mut failure: Option<(PathBuf, std::io::Error)> = None; + for entry in entries { + let entry = match entry { + Ok(entry) => entry, + Err(source) => { + // The rest of the listing is unknown, so stop reading it. + failure.get_or_insert_with(|| (services_dir.to_path_buf(), source)); + break; + } + }; let path = entry.path(); - if path.extension().and_then(|v| v.to_str()) != Some("json") { + if path.extension().and_then(|v| v.to_str()) != Some("json") || path.is_dir() { continue; } - if let Ok(bytes) = fs::read(&path) - && let Ok(record) = serde_json::from_slice::(&bytes) - { - records.push(record); + let bytes = match read_record(&path) { + Ok(bytes) => bytes, + Err(e) if e.kind() == ErrorKind::NotFound => continue, + Err(source) => { + failure.get_or_insert((path, source)); + continue; + } + }; + match serde_json::from_slice::(&bytes) { + Ok(record) => read.records.push(record), + Err(_) => read.malformed.push(path), } } - records.sort_by_key(|record| std::cmp::Reverse(record.created_at_unix_ms)); - records + read.records + .sort_by_key(|record| std::cmp::Reverse(record.created_at_unix_ms)); + read.malformed.sort(); + match failure { + None => Ok(read), + Some((path, source)) => Err(RegistryReadError { + path, + source, + partial: read, + }), + } +} + +/// Load every managed-service record under `services_dir`, newest first. +/// +/// Best-effort, for the one-shot readers (the dashboard's launcher and exit +/// prompt) that have nothing to retire: a directory that cannot be listed yields +/// an empty list, and a record that cannot be read or parsed is skipped on its +/// own, so one bad file never hides the services beside it. The daemon's +/// discovery loop must not use this, because it cannot tell an unreadable +/// registry from an empty one; it calls [`read_service_records`] and keeps the +/// services it could not confirm gone. +pub fn load_service_records(services_dir: &Path) -> Vec { + load_service_records_with(services_dir, |path| fs::read(path)) +} + +/// [`load_service_records`] with the record reader injected (see +/// [`read_service_records_with`]). +fn load_service_records_with( + services_dir: &Path, + read_record: impl Fn(&Path) -> std::io::Result>, +) -> Vec { + read_service_records_with(services_dir, read_record) + .unwrap_or_else(|failed| failed.partial) + .records } /// Convert a managed-service record into a [`DiscoveredService`] for the @@ -339,6 +485,248 @@ mod tests { let _ = fs::remove_dir_all(&dir); } + #[test] + fn read_missing_dir_is_confirmed_empty() { + let dir = test_dir("read-absent").join("nope"); + let read = read_service_records(&dir).expect("an absent registry is confirmed empty"); + assert!(read.records.is_empty()); + assert!(read.malformed.is_empty()); + } + + #[test] + fn read_empty_dir_is_confirmed_empty() { + let dir = test_dir("read-empty"); + let read = read_service_records(&dir).expect("an empty registry is confirmed empty"); + assert!(read.records.is_empty()); + assert!(read.malformed.is_empty()); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn read_regular_file_where_the_dir_should_be_is_a_read_failure() { + let dir = test_dir("read-file-not-dir"); + let not_a_dir = dir.join("services"); + fs::write(¬_a_dir, "not a directory").unwrap(); + + let err = read_service_records(¬_a_dir) + .expect_err("a file where the registry should be must not read as empty"); + assert_eq!(err.path, not_a_dir); + assert!(err.to_string().contains("cannot read"), "{err}"); + assert!( + err.partial.records.is_empty() && err.partial.malformed.is_empty(), + "a directory that cannot be listed has nothing partial to offer" + ); + // The best-effort wrapper keeps its old answer for the one-shot readers. + assert!(load_service_records(¬_a_dir).is_empty()); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn read_keeps_well_formed_records_beside_a_malformed_one() { + let dir = test_dir("read-malformed"); + fs::write( + dir.join("a.json"), + record_json("svc-a", "vllm", 8000, "running", 100), + ) + .unwrap(); + fs::write( + dir.join("b.json"), + record_json("svc-b", "vllm", 8001, "running", 200), + ) + .unwrap(); + fs::write(dir.join("bad.json"), "{ not valid").unwrap(); + fs::write(dir.join("also-bad.json"), "42").unwrap(); + fs::write(dir.join("notes.txt"), "ignore me").unwrap(); + + let read = read_service_records(&dir).expect("a malformed record is not a read failure"); + let ids: Vec<&str> = read.records.iter().map(|r| r.service_id.as_str()).collect(); + assert_eq!(ids, ["svc-b", "svc-a"]); + assert_eq!( + read.malformed, + [dir.join("also-bad.json"), dir.join("bad.json")] + ); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn read_skips_a_directory_named_like_a_record() { + let dir = test_dir("read-dir-named-json"); + fs::create_dir(dir.join("looks-like-a-record.json")).unwrap(); + fs::write( + dir.join("a.json"), + record_json("svc-a", "vllm", 8000, "running", 100), + ) + .unwrap(); + + let read = read_service_records(&dir).expect("a stray directory is not a read failure"); + assert_eq!(read.records.len(), 1); + assert!(read.malformed.is_empty()); + let _ = fs::remove_dir_all(&dir); + } + + /// Unix only (no mode bits on Windows), and skipped when the process can + /// read a mode-000 file anyway (running as root). + #[cfg(unix)] + #[test] + fn read_unreadable_record_file_is_a_read_failure() { + use std::os::unix::fs::PermissionsExt; + let dir = test_dir("read-unreadable-record"); + fs::write( + dir.join("a.json"), + record_json("svc-a", "vllm", 8000, "running", 100), + ) + .unwrap(); + let locked = dir.join("locked.json"); + fs::write( + &locked, + record_json("svc-locked", "vllm", 8001, "running", 200), + ) + .unwrap(); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).unwrap(); + if fs::read(&locked).is_ok() { + eprintln!("skipped: this process can read a mode-000 file (running as root)"); + let _ = fs::remove_dir_all(&dir); + return; + } + + let result = read_service_records(&dir); + let _ = fs::set_permissions(&locked, fs::Permissions::from_mode(0o644)); + let err = result.expect_err("an unreadable record must not read as a missing one"); + assert_eq!(err.path, locked); + assert_eq!(err.partial.records.len(), 1, "the readable record is kept"); + let _ = fs::remove_dir_all(&dir); + } + + /// The one-shot readers (launcher, exit prompt, services overlay) stay + /// best-effort per record: one unreadable record file hides only itself, + /// while the strict reader the daemon's loop uses fails on the same + /// registry. Unix only for the real mode-000 file, and skipped as root; the + /// portable twin below injects the failing read. + #[cfg(unix)] + #[test] + fn load_skips_one_unreadable_record_and_keeps_the_readable_ones() { + use std::os::unix::fs::PermissionsExt; + let dir = test_dir("load-unreadable-record"); + fs::write( + dir.join("a.json"), + record_json("svc-a", "vllm", 8000, "running", 100), + ) + .unwrap(); + fs::write( + dir.join("b.json"), + record_json("svc-b", "vllm", 8001, "running", 300), + ) + .unwrap(); + let locked = dir.join("locked.json"); + fs::write( + &locked, + record_json("svc-locked", "vllm", 8002, "running", 200), + ) + .unwrap(); + fs::set_permissions(&locked, fs::Permissions::from_mode(0o000)).unwrap(); + if fs::read(&locked).is_ok() { + eprintln!("skipped: this process can read a mode-000 file (running as root)"); + let _ = fs::remove_dir_all(&dir); + return; + } + + let loaded = load_service_records(&dir); + let strict = read_service_records(&dir); + let _ = fs::set_permissions(&locked, fs::Permissions::from_mode(0o644)); + let ids: Vec<&str> = loaded.iter().map(|r| r.service_id.as_str()).collect(); + assert_eq!(ids, ["svc-b", "svc-a"]); + assert_eq!( + strict.expect_err("the strict reader must still fail").path, + locked + ); + let _ = fs::remove_dir_all(&dir); + } + + /// A registry with three records, one of which the injected reader refuses + /// with a permission error. Runs on every platform, unlike the mode-000 test. + fn registry_with_one_record_refused() -> (PathBuf, impl Fn(&Path) -> std::io::Result>) { + let dir = test_dir("scan-refused-record"); + for (name, id, created) in [ + ("a", "svc-a", 100), + ("locked", "svc-locked", 200), + ("b", "svc-b", 300), + ] { + fs::write( + dir.join(format!("{name}.json")), + record_json(id, "vllm", 8000, "running", created), + ) + .unwrap(); + } + let refused = dir.join("locked.json"); + let read_record = move |path: &Path| { + if path == refused { + Err(std::io::Error::from(ErrorKind::PermissionDenied)) + } else { + fs::read(path) + } + }; + (dir, read_record) + } + + #[test] + fn load_keeps_the_readable_records_beside_one_that_cannot_be_read() { + let (dir, read_record) = registry_with_one_record_refused(); + let loaded = load_service_records_with(&dir, read_record); + let ids: Vec<&str> = loaded.iter().map(|r| r.service_id.as_str()).collect(); + assert_eq!(ids, ["svc-b", "svc-a"]); + let _ = fs::remove_dir_all(&dir); + } + + #[test] + fn read_fails_the_pass_on_a_record_that_cannot_be_read() { + let (dir, read_record) = registry_with_one_record_refused(); + let err = read_service_records_with(&dir, read_record) + .expect_err("the strict reader must not read an unreadable record as a missing one"); + assert_eq!(err.path, dir.join("locked.json")); + assert_eq!(err.source.kind(), ErrorKind::PermissionDenied); + // The failure does not stop the scan: both readable records are in the + // partial read, newest first, whichever side of the locked one they sort. + let ids: Vec<&str> = err + .partial + .records + .iter() + .map(|r| r.service_id.as_str()) + .collect(); + assert_eq!(ids, ["svc-b", "svc-a"]); + assert!(err.partial.malformed.is_empty()); + let _ = fs::remove_dir_all(&dir); + } + + /// The compatibility loader and the strict reader share one scan, and a + /// registry with one corrupt record and two readable ones gives both the two + /// readable records (a corrupt record is malformed in both, never a failure). + #[test] + fn load_and_read_agree_on_a_registry_with_one_corrupt_record() { + let dir = test_dir("load-read-corrupt"); + fs::write( + dir.join("a.json"), + record_json("svc-a", "vllm", 8000, "running", 100), + ) + .unwrap(); + fs::write(dir.join("corrupt.json"), "{ not valid").unwrap(); + fs::write( + dir.join("b.json"), + record_json("svc-b", "vllm", 8001, "running", 200), + ) + .unwrap(); + + let loaded: Vec = load_service_records(&dir) + .into_iter() + .map(|r| r.service_id) + .collect(); + let read = read_service_records(&dir).expect("a corrupt record is not a read failure"); + let strict: Vec = read.records.into_iter().map(|r| r.service_id).collect(); + assert_eq!(loaded, ["svc-b", "svc-a"]); + assert_eq!(strict, loaded); + assert_eq!(read.malformed, [dir.join("corrupt.json")]); + let _ = fs::remove_dir_all(&dir); + } + #[test] fn discovered_uses_registry_port_as_authority() { let dir = test_dir("port-authority"); diff --git a/crates/rocm-dash-daemon/src/runner.rs b/crates/rocm-dash-daemon/src/runner.rs index 7d526b9577..fde02a1c37 100644 --- a/crates/rocm-dash-daemon/src/runner.rs +++ b/crates/rocm-dash-daemon/src/runner.rs @@ -402,6 +402,7 @@ pub async fn run_loop( // holding them only for the tick that produced them makes the header ⚠ badge // blink on and off. Keep the last result until that cadence runs again. let mut docker_warn: Option = None; + let mut registry_warn: Option = None; let mut scrape_warns: Vec = Vec::new(); loop { @@ -588,19 +589,47 @@ pub async fn run_loop( // the registry's authority; non-vLLM engines are tracked so the vLLM // scrape below skips them. A model served via `rocm serve` thus appears // in the dashboard and is scraped for `gen_tps` without Docker. + // + // A pass that could not read the registry in full retires nothing, as + // with the Docker source above: what is known stays known (no `Gone`), + // and the instances keep the data they hold. Unlike Docker, whose listing + // is all or nothing, a registry can fail on one record while the rest + // read, so the readable records still add and refresh their services + // (announced once, as ever). Only a registry that was read in full, even + // an empty one, retires what it lacks. if let Some(services_dir) = opts.services_dir.as_ref() && discovery_due { - let records = crate::registry::load_service_records(services_dir); - let disc = crate::registry::discover_managed_services(&records); - discovery.managed_non_vllm = disc.non_vllm; - reconcile.reconcile_set( - DiscoverySource::Managed, - &disc.svcs, - &discovery.known_services, - &disc.seen, - ); - discovery.known_services = disc.seen; + let read = crate::registry::read_service_records(services_dir); + let change = discovery.registry_health.observe(&read); + change.log(); + match read { + Ok(read) => { + let disc = crate::registry::discover_managed_services(&read.records); + discovery.managed_non_vllm = disc.non_vllm; + reconcile.reconcile_set( + DiscoverySource::Managed, + &disc.svcs, + &discovery.known_services, + &disc.seen, + ); + discovery.known_services = disc.seen; + registry_warn = None; + } + Err(e) => { + // The records that did read still add and refresh their + // services; nothing is retired, because the one that did + // not read could be any of the known ones. + let disc = crate::registry::discover_managed_services(&e.partial.records); + reconcile.upsert_set( + DiscoverySource::Managed, + &disc.svcs, + &discovery.known_services, + ); + discovery.keep_through_partial_registry(disc); + registry_warn = Some(registry_read_warning(&e)); + } + } } if tick_count.is_multiple_of(instance_ticks) { scrape_warns.clear(); @@ -614,14 +643,11 @@ pub async fn run_loop( && !runner.state.instances.is_empty() && tick_count.is_multiple_of(instance_ticks) { - let targets: Vec<(String, u16)> = runner - .state - .instances - .values() - .filter(|i| Some(i.container_id.as_str()) != discovery.lemonade_id.as_deref()) - .filter(|i| !discovery.managed_non_vllm.contains(&i.container_id)) - .filter_map(|i| i.port.map(|p| (i.container_id.clone(), p))) - .collect(); + let targets = vllm_scrape_targets( + &runner.state.instances, + discovery.lemonade_id.as_deref(), + &discovery.managed_non_vllm, + ); let prom = prom.clone(); let results = parallel_scrape(targets, move |(id, port)| { let prom = prom.clone(); @@ -862,6 +888,7 @@ pub async fn run_loop( } } warnings.extend(docker_warn.iter().cloned()); + warnings.extend(registry_warn.iter().cloned()); warnings.extend(scrape_warns.iter().cloned()); let snap = Snapshot { timestamp: cycle_at, @@ -1020,6 +1047,102 @@ struct DiscoveryState { managed_non_vllm: HashSet, /// The id of the currently-live Lemonade instance, if any. lemonade_id: Option, + /// How the registry reads have been going, so a condition that lasts many + /// passes is logged once rather than every discovery tick. + registry_health: RegistryHealth, +} + +impl DiscoveryState { + /// Record what a pass that read only part of the registry found. The services + /// it saw join the known set, so a later complete read can retire them, and + /// the known ones it did not see stay; the non-vLLM marks of the services it + /// saw are refreshed and the others kept, so a kept non-vLLM service is still + /// left out of the vLLM scrape. + fn keep_through_partial_registry(&mut self, disc: crate::registry::ManagedDiscovery) { + self.managed_non_vllm.retain(|id| !disc.seen.contains(id)); + self.managed_non_vllm.extend(disc.non_vllm); + self.known_services.extend(disc.seen); + } +} + +/// What the managed-service registry's reads have reported so far. A failure +/// that lasts is logged on its first pass only, the recovery once, and a set of +/// malformed records each time it changes; the dashboard's warning badge, which +/// is cheap and does not scroll, carries the failure for as long as it lasts. +#[derive(Default)] +struct RegistryHealth { + /// The previous pass could not read the registry. + failing: bool, + /// The malformed records the last successful pass reported, so an unchanged + /// set is not reported again. + reported_malformed: Vec, +} + +/// What a discovery pass should say about the registry, apart from the services +/// it found. Kept as data so [`RegistryHealth::observe`] can be tested without a +/// log subscriber. +#[derive(Debug, Default, PartialEq, Eq)] +struct RegistryChange { + /// The first failed pass of a streak, with the error. + failed: Option, + /// The first successful pass after a streak of failures. + recovered: bool, + /// The malformed records, when the set differs from the one last reported + /// and is not empty. + malformed: Vec, +} + +impl RegistryHealth { + fn observe( + &mut self, + read: &Result, + ) -> RegistryChange { + let mut change = RegistryChange::default(); + match read { + Err(e) => { + if !self.failing { + self.failing = true; + change.failed = Some(e.to_string()); + } + } + Ok(read) => { + if self.failing { + self.failing = false; + change.recovered = true; + } + if read.malformed != self.reported_malformed { + self.reported_malformed.clone_from(&read.malformed); + change.malformed.clone_from(&read.malformed); + } + } + } + change + } +} + +impl RegistryChange { + fn log(&self) { + if let Some(error) = &self.failed { + warn!( + error = %error, + "managed-service registry could not be read; keeping the known services until it can be" + ); + } + if self.recovered { + info!("managed-service registry is readable again"); + } + if !self.malformed.is_empty() { + warn!( + files = ?self.malformed, + "skipping managed-service records that are not valid JSON" + ); + } + } +} + +/// The header warning for a pass that could not read the registry. +fn registry_read_warning(error: &crate::registry::RegistryReadError) -> String { + format!("managed registry: {error}; keeping the last known services") } /// Which discovery source a service came from. It only selects the log wording @@ -1120,6 +1243,22 @@ impl Reconcile<'_> { svcs: &[DiscoveredService], known: &HashSet, seen: &HashSet, + ) { + self.upsert_set(source, svcs, known); + for gone in known.difference(seen) { + self.retire(source, gone); + } + } + + /// The half of [`Self::reconcile_set`] that adds and refreshes: upsert every + /// service in `svcs` and announce the ones `known` did not have. Retires + /// nothing, which is what a pass that could only see part of its source needs + /// (see [`DiscoveryState::keep_through_partial_registry`]). + fn upsert_set( + &mut self, + source: DiscoverySource, + svcs: &[DiscoveredService], + known: &HashSet, ) { for svc in svcs { let same_endpoint = self @@ -1136,18 +1275,17 @@ impl Reconcile<'_> { self.announce(source, svc); } } - for gone in known.difference(seen) { - self.retire(source, gone); - } } /// Reconcile the probe-based Lemonade source, which has at most one service. /// /// Its `container_id` encodes the host and port, so a port or host change - /// yields a new id: the old id's telemetry is purged so a stale rate - /// baseline, TTFT and TPOT do not bleed across, and the new id is announced. - /// The old instance is not retired here. `None` means the endpoint did not - /// answer, which retires the current one. + /// yields a new id. The old id is retired first (its telemetry is purged, so + /// a stale rate baseline, TTFT and TPOT do not bleed across; it leaves the + /// state and is announced gone), then the new id is announced, so the state + /// and every snapshot carry one Lemonade instance and the vLLM scrape never + /// polls the old endpoint. `None` means the endpoint did not answer, which + /// retires the current one. fn reconcile_lemonade( &mut self, probed: Option, @@ -1156,8 +1294,8 @@ impl Reconcile<'_> { match probed { Some(svc) => { let is_new_id = current.as_deref() != Some(svc.container_id.as_str()); - if is_new_id && let Some(old_id) = current.as_deref() { - self.telemetry.purge(old_id); + if is_new_id && let Some(old_id) = current.take() { + self.retire(DiscoverySource::Lemonade, &old_id); } self.upsert(&svc, !is_new_id); if is_new_id { @@ -1174,6 +1312,22 @@ impl Reconcile<'_> { } } +/// The `(id, port)` pairs the per-instance vLLM Prometheus scrape polls: every +/// instance with a port except the live Lemonade instance (its own collector +/// scrapes it) and the managed services whose engine is not vLLM. +fn vllm_scrape_targets( + instances: &HashMap, + lemonade_id: Option<&str>, + managed_non_vllm: &HashSet, +) -> Vec<(String, u16)> { + instances + .values() + .filter(|i| Some(i.container_id.as_str()) != lemonade_id) + .filter(|i| !managed_non_vllm.contains(&i.container_id)) + .filter_map(|i| i.port.map(|p| (i.container_id.clone(), p))) + .collect() +} + /// Set `vram_used_mb`/`vram_total_mb` on each instance from the per-process /// attribution map, falling back to device-summed VRAM over the instance's /// GPUs when its container has no per-process entry. Pure — the runner does the @@ -1962,6 +2116,276 @@ mod tests { assert!(pass.telemetry.prev_ttft.contains_key("a")); } + // ── Lemonade state matrix ─────────────────────────────────────────────── + // + // One Lemonade endpoint is probed per discovery pass. Its id encodes the host + // and port, so the states a probe can observe are: nothing ever answered; the + // first answer; the same endpoint again; the host or the port changed; the + // endpoint went quiet; it came back somewhere else; and the same answer + // arriving twice in one tick. Each row below checks the announcements, the + // instances a snapshot would carry, the telemetry kept per id, and the + // `(id, port)` pairs the vLLM scrape would poll, because a stale instance is + // visible on all four. + + fn lemonade_svc(host: &str, port: u16) -> DiscoveredService { + DiscoveredService { + container_id: format!("lemonade-{host}-{port}"), + container_name: "lemonade".into(), + model_name: "model".into(), + port: Some(port), + ..Default::default() + } + } + + impl Pass { + /// The ids a snapshot carries. The loop builds a snapshot's instances from + /// `state.instances`, so this is the dashboard's view of the pass. + fn instance_ids(&self) -> Vec { + let mut ids: Vec = self.state.instances.keys().cloned().collect(); + ids.sort(); + ids + } + + /// The ids the vLLM Prometheus scrape would poll while `current` is the + /// live Lemonade id. + fn scrape_targets(&self, current: Option<&str>) -> Vec { + let mut ids: Vec = + vllm_scrape_targets(&self.state.instances, current, &HashSet::new()) + .into_iter() + .map(|(id, _port)| id) + .collect(); + ids.sort(); + ids + } + + /// A live KV reading, latency baselines and a rate tracker for `id`. + fn give_live_telemetry(&mut self, id: &str) { + self.scrape(id); + self.telemetry + .gen_trackers + .insert(id.into(), make_tracker(2)); + } + + fn has_telemetry(&self, id: &str) -> bool { + self.telemetry.gen_trackers.contains_key(id) + || self.telemetry.prev_ttft.contains_key(id) + || self.telemetry.prev_tpot.contains_key(id) + } + } + + fn ids(ids: &[&str]) -> Vec { + ids.iter().map(|s| (*s).to_string()).collect() + } + + /// Discover `old`, give it live telemetry, then probe `new`. Returns the pass, + /// the loop's current Lemonade id and what the second probe announced. + fn lemonade_identity_change( + old: &DiscoveredService, + new: &DiscoveredService, + ) -> (Pass, Option, Vec) { + let mut pass = Pass::new(); + let mut current: Option = None; + pass.reconcile() + .reconcile_lemonade(Some(old.clone()), &mut current); + pass.give_live_telemetry(&old.container_id); + pass.announced(); + + pass.reconcile() + .reconcile_lemonade(Some(new.clone()), &mut current); + let announced = pass.announced(); + (pass, current, announced) + } + + #[test] + fn lemonade_that_never_answers_leaves_no_trace() { + let mut pass = Pass::new(); + let mut current: Option = None; + + pass.reconcile().reconcile_lemonade(None, &mut current); + pass.reconcile().reconcile_lemonade(None, &mut current); + + assert!(pass.announced().is_empty()); + assert_eq!(current, None); + assert!(pass.instance_ids().is_empty()); + assert!(pass.scrape_targets(current.as_deref()).is_empty()); + } + + #[test] + fn lemonade_first_answer_is_announced_once_and_left_to_its_own_collector() { + let mut pass = Pass::new(); + let mut current: Option = None; + + pass.reconcile() + .reconcile_lemonade(Some(lemonade_svc("127.0.0.1", 13305)), &mut current); + + assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13305"]); + assert_eq!(current.as_deref(), Some("lemonade-127.0.0.1-13305")); + assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13305"])); + assert!( + pass.scrape_targets(current.as_deref()).is_empty(), + "the live Lemonade instance is not a vLLM scrape target" + ); + } + + #[test] + fn lemonade_same_endpoint_again_keeps_its_instance_and_live_fields() { + let svc = lemonade_svc("127.0.0.1", 13305); + let id = svc.container_id.clone(); + let mut pass = Pass::new(); + let mut current: Option = None; + pass.reconcile() + .reconcile_lemonade(Some(svc.clone()), &mut current); + pass.give_live_telemetry(&id); + pass.announced(); + + pass.reconcile().reconcile_lemonade(Some(svc), &mut current); + + assert!(pass.announced().is_empty()); + assert_eq!(pass.instance_ids(), ids(&[&id])); + assert_eq!(pass.kv(&id), Some(25.0)); + assert!(pass.has_telemetry(&id)); + assert!(pass.scrape_targets(current.as_deref()).is_empty()); + } + + #[test] + fn lemonade_host_change_retires_the_old_instance_and_announces_the_new_one() { + let old = lemonade_svc("127.0.0.1", 13305); + let new = lemonade_svc("10.0.0.7", 13305); + let (pass, current, announced) = lemonade_identity_change(&old, &new); + + assert_eq!(current.as_deref(), Some(new.container_id.as_str())); + assert_eq!( + announced, + [ + "gone lemonade-127.0.0.1-13305", + "discovered lemonade-10.0.0.7-13305" + ] + ); + assert!(!pass.has_telemetry(&old.container_id)); + assert_eq!(pass.instance_ids(), ids(&["lemonade-10.0.0.7-13305"])); + assert!( + pass.scrape_targets(current.as_deref()).is_empty(), + "the old endpoint must not be polled as a vLLM target" + ); + } + + #[test] + fn lemonade_port_change_retires_the_old_instance_and_announces_the_new_one() { + let old = lemonade_svc("127.0.0.1", 13305); + let new = lemonade_svc("127.0.0.1", 13306); + let (pass, current, announced) = lemonade_identity_change(&old, &new); + + assert_eq!(current.as_deref(), Some(new.container_id.as_str())); + assert_eq!( + announced, + [ + "gone lemonade-127.0.0.1-13305", + "discovered lemonade-127.0.0.1-13306" + ] + ); + assert!(!pass.has_telemetry(&old.container_id)); + assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13306"])); + assert!( + pass.scrape_targets(current.as_deref()).is_empty(), + "the old endpoint must not be polled as a vLLM target" + ); + } + + #[test] + fn lemonade_endpoint_going_quiet_is_announced_gone_and_leaves_no_trace() { + let svc = lemonade_svc("127.0.0.1", 13305); + let id = svc.container_id.clone(); + let mut pass = Pass::new(); + let mut current: Option = None; + pass.reconcile().reconcile_lemonade(Some(svc), &mut current); + pass.give_live_telemetry(&id); + pass.announced(); + + pass.reconcile().reconcile_lemonade(None, &mut current); + + assert_eq!(pass.announced(), [format!("gone {id}")]); + assert_eq!(current, None); + assert!(pass.instance_ids().is_empty()); + assert!(!pass.has_telemetry(&id)); + assert!(pass.scrape_targets(current.as_deref()).is_empty()); + } + + #[test] + fn lemonade_coming_back_elsewhere_after_going_quiet_is_the_only_instance() { + let old = lemonade_svc("127.0.0.1", 13305); + let new = lemonade_svc("127.0.0.1", 13306); + let mut pass = Pass::new(); + let mut current: Option = None; + pass.reconcile() + .reconcile_lemonade(Some(old.clone()), &mut current); + pass.give_live_telemetry(&old.container_id); + pass.reconcile().reconcile_lemonade(None, &mut current); + assert_eq!( + pass.announced(), + [ + "discovered lemonade-127.0.0.1-13305", + "gone lemonade-127.0.0.1-13305" + ] + ); + + pass.reconcile() + .reconcile_lemonade(Some(new.clone()), &mut current); + + assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13306"]); + assert_eq!(current.as_deref(), Some(new.container_id.as_str())); + assert_eq!(pass.instance_ids(), ids(&[&new.container_id])); + assert!(!pass.has_telemetry(&old.container_id)); + assert!(pass.scrape_targets(current.as_deref()).is_empty()); + } + + #[test] + fn lemonade_answer_repeated_within_one_tick_is_announced_once() { + let svc = lemonade_svc("127.0.0.1", 13305); + let mut pass = Pass::new(); + let mut current: Option = None; + + pass.reconcile() + .reconcile_lemonade(Some(svc.clone()), &mut current); + pass.reconcile().reconcile_lemonade(Some(svc), &mut current); + + assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13305"]); + assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13305"])); + assert!(pass.scrape_targets(current.as_deref()).is_empty()); + } + + #[test] + fn lemonade_endpoint_moving_away_and_back_is_retired_each_time() { + let a = lemonade_svc("127.0.0.1", 13305); + let b = lemonade_svc("127.0.0.1", 13306); + let mut pass = Pass::new(); + let mut current: Option = None; + pass.reconcile() + .reconcile_lemonade(Some(a.clone()), &mut current); + pass.announced(); + + pass.reconcile().reconcile_lemonade(Some(b), &mut current); + assert_eq!( + pass.announced(), + [ + "gone lemonade-127.0.0.1-13305", + "discovered lemonade-127.0.0.1-13306" + ] + ); + pass.reconcile() + .reconcile_lemonade(Some(a.clone()), &mut current); + + assert_eq!( + pass.announced(), + [ + "gone lemonade-127.0.0.1-13306", + "discovered lemonade-127.0.0.1-13305" + ] + ); + assert_eq!(current.as_deref(), Some(a.container_id.as_str())); + assert_eq!(pass.instance_ids(), ids(&[&a.container_id])); + assert!(pass.scrape_targets(current.as_deref()).is_empty()); + } + #[test] fn reconcile_lemonade_follows_one_endpoint_through_its_lifecycle() { let mut pass = Pass::new(); @@ -1981,18 +2405,16 @@ mod tests { ); assert_eq!(pass.kv("l1"), Some(25.0), "live fields carry over"); - // A new id (the host or port changed) purges the old id's telemetry and - // announces the new one. + // A new id (the host or port changed) retires the old id, telemetry + // included, and announces the new one. pass.reconcile() .reconcile_lemonade(Some(discovered("l2", 13306)), &mut current); - assert_eq!(pass.announced(), ["discovered l2"]); + assert_eq!(pass.announced(), ["gone l1", "discovered l2"]); assert_eq!(current.as_deref(), Some("l2")); assert!(!pass.telemetry.prev_ttft.contains_key("l1")); - // Pins current behaviour, not a goal: the superseded instance stays in - // state (and so in every snapshot) because only its telemetry is purged. - // Retiring it later is a deliberate behaviour change, and this assertion - // is the one to update when that happens. - assert!(pass.state.instances.contains_key("l1")); + // The superseded instance leaves the state (and so every snapshot), not + // only its telemetry. + assert!(!pass.state.instances.contains_key("l1")); pass.reconcile().reconcile_lemonade(None, &mut current); assert_eq!(pass.announced(), ["gone l2"]); @@ -2002,4 +2424,176 @@ mod tests { pass.reconcile().reconcile_lemonade(None, &mut current); assert!(pass.announced().is_empty(), "nothing left to retire"); } + + // Registry read health: one test per row of the state matrix (first + // failure, repeated failure, recovery, empty, populated, malformed new, + // malformed unchanged, malformed cleared). + + fn registry_read( + malformed: &[&str], + ) -> Result { + Ok(crate::registry::RegistryRead { + records: Vec::new(), + malformed: malformed.iter().map(PathBuf::from).collect(), + }) + } + + fn registry_failure( + why: &str, + ) -> Result { + Err(crate::registry::RegistryReadError { + path: PathBuf::from("services"), + source: std::io::Error::other(why), + partial: crate::registry::RegistryRead::default(), + }) + } + + /// State matrix for what a partially read registry does to the discovery + /// state, one assertion group per row: a service it saw joins the known set + /// and its non-vLLM mark follows the record; one it did not see stays known + /// and keeps its mark; nothing leaves either set. + #[test] + fn partial_registry_read_adds_what_it_saw_and_forgets_nothing() { + let mut state = DiscoveryState { + known_services: id_set(&["old-vllm", "old-lemonade", "flips"]), + managed_non_vllm: id_set(&["old-lemonade", "flips"]), + ..DiscoveryState::default() + }; + + state.keep_through_partial_registry(crate::registry::ManagedDiscovery { + seen: id_set(&["new-lemonade", "new-vllm", "flips"]), + non_vllm: id_set(&["new-lemonade"]), + ..Default::default() + }); + + assert_eq!( + state.known_services, + id_set(&[ + "old-vllm", + "old-lemonade", + "flips", + "new-lemonade", + "new-vllm" + ]), + "the unseen services stay known and the new ones join them" + ); + assert_eq!( + state.managed_non_vllm, + id_set(&["old-lemonade", "new-lemonade"]), + "an unseen non-vLLM service keeps its mark, a seen one follows its record" + ); + } + + #[test] + fn partial_registry_read_that_saw_nothing_changes_nothing() { + let mut state = DiscoveryState { + known_services: id_set(&["a", "b"]), + managed_non_vllm: id_set(&["b"]), + ..DiscoveryState::default() + }; + + state.keep_through_partial_registry(crate::registry::ManagedDiscovery::default()); + + assert_eq!(state.known_services, id_set(&["a", "b"])); + assert_eq!(state.managed_non_vllm, id_set(&["b"])); + } + + #[test] + fn registry_health_reports_the_first_failure_of_a_streak_once() { + let mut health = RegistryHealth::default(); + + let first = health.observe(®istry_failure("disk gone")); + assert_eq!( + first.failed.as_deref(), + Some("cannot read services: disk gone"), + "the first failed pass names the error" + ); + assert!(!first.recovered); + + for pass in 2..=5 { + assert_eq!( + health.observe(®istry_failure("disk gone")), + RegistryChange::default(), + "failed pass {pass} of the same streak must stay quiet" + ); + } + } + + #[test] + fn registry_health_reports_recovery_once_and_a_new_streak_again() { + let mut health = RegistryHealth::default(); + health.observe(®istry_failure("one")); + health.observe(®istry_failure("one")); + + let recovered = health.observe(®istry_read(&[])); + assert!(recovered.recovered); + assert!(recovered.failed.is_none()); + assert_eq!( + health.observe(®istry_read(&[])), + RegistryChange::default(), + "only the first good pass after a streak is a recovery" + ); + + let again = health.observe(®istry_failure("two")); + assert_eq!(again.failed.as_deref(), Some("cannot read services: two")); + } + + #[test] + fn registry_health_is_quiet_for_an_empty_or_populated_registry() { + let mut health = RegistryHealth::default(); + // A healthy read, empty or not, says nothing: there is no failure to + // report and no failure to have recovered from. + for _ in 0..3 { + assert_eq!( + health.observe(®istry_read(&[])), + RegistryChange::default() + ); + } + } + + #[test] + fn registry_health_reports_a_malformed_set_when_it_changes() { + let mut health = RegistryHealth::default(); + + let first = health.observe(®istry_read(&["a.json"])); + assert_eq!(first.malformed, [PathBuf::from("a.json")]); + assert_eq!( + health.observe(®istry_read(&["a.json"])), + RegistryChange::default(), + "an unchanged malformed set is not reported again" + ); + + let grown = health.observe(®istry_read(&["a.json", "b.json"])); + assert_eq!( + grown.malformed, + [PathBuf::from("a.json"), PathBuf::from("b.json")], + "a newly malformed record reports the whole set" + ); + + let cleared = health.observe(®istry_read(&[])); + assert!( + cleared.malformed.is_empty(), + "clearing the set is not a warning" + ); + let back = health.observe(®istry_read(&["a.json"])); + assert_eq!( + back.malformed, + [PathBuf::from("a.json")], + "a record that goes bad again after being fixed is reported again" + ); + } + + #[test] + fn registry_health_keeps_the_malformed_set_across_a_read_failure() { + let mut health = RegistryHealth::default(); + health.observe(®istry_read(&["a.json"])); + health.observe(®istry_failure("blip")); + + let after = health.observe(®istry_read(&["a.json"])); + assert!(after.recovered); + assert!( + after.malformed.is_empty(), + "the same corrupt record is not news just because a read failed in between" + ); + } } diff --git a/crates/rocm-dash-daemon/tests/discovery_reconcile.rs b/crates/rocm-dash-daemon/tests/discovery_reconcile.rs index bd7c7b5200..570c0a656c 100644 --- a/crates/rocm-dash-daemon/tests/discovery_reconcile.rs +++ b/crates/rocm-dash-daemon/tests/discovery_reconcile.rs @@ -13,9 +13,10 @@ //! its endpoint changes. On Lemonade: the endpoint is announced once, scraped, //! and announced gone when it stops answering. //! -//! The registry's retirement of a vanished service and Lemonade's change of id -//! are not driven through the loop here; they are covered only by the unit tests -//! in `runner.rs`. The Docker source has no seam that works without a container +//! The registry's retirement of a vanished service is driven through the loop +//! here too, for an empty registry and for one whose directory cannot be read. +//! Lemonade's change of id is not driven through the loop here; it is covered only +//! by the unit tests in `runner.rs`. The Docker source has no seam that works without a container //! daemon, so its reconcile is covered only by the unit tests next to the shared //! helper in `runner.rs`. @@ -157,15 +158,19 @@ fn lemonade_opts(port: u16) -> RunnerOptions { /// record: a torn read would look like the service vanishing and coming back, /// which is exactly what the tests below assert does not happen. fn write_record(services_dir: &std::path::Path, port: u16) { + write_record_for(services_dir, SVC_ID, port); +} + +fn write_record_for(services_dir: &std::path::Path, id: &str, port: u16) { std::fs::create_dir_all(services_dir).unwrap(); let json = format!( - r#"{{"service_id":"{SVC_ID}","engine":"vllm","model_ref":"Reconcile/1B","canonical_model_id":"Reconcile/1B","host":"127.0.0.1","port":{port},"status":"ready","created_at_unix_ms":0}}"# + r#"{{"service_id":"{id}","engine":"vllm","model_ref":"Reconcile/1B","canonical_model_id":"Reconcile/1B","host":"127.0.0.1","port":{port},"status":"ready","created_at_unix_ms":0}}"# ); // The registry only reads `*.json`, so the `.tmp` suffix keeps the // half-written file invisible to it. - let tmp = services_dir.join(format!("{SVC_ID}.json.tmp")); + let tmp = services_dir.join(format!("{id}.json.tmp")); std::fs::write(&tmp, json).unwrap(); - std::fs::rename(&tmp, services_dir.join(format!("{SVC_ID}.json"))).unwrap(); + std::fs::rename(&tmp, services_dir.join(format!("{id}.json"))).unwrap(); } /// Read events until `pick` returns a value, or fail at the deadline. @@ -437,6 +442,514 @@ async fn managed_endpoint_change_starts_the_instance_clean() { ); } +// ── Managed-service registry: what the loop does when the registry changes ── + +/// A running loop with one managed service that has been scraped at least once, +/// so the tests below start from an instance that carries live fields. +struct LiveManaged { + rx: broadcast::Receiver, + services_dir: std::path::PathBuf, + server: MockServer, + // Held so the registry directory outlives the test body. + _tmp: tempfile::TempDir, + _runner: tokio::task::JoinHandle<()>, +} + +impl LiveManaged { + /// The port the service's record names, for writing the record back. + const fn last_port(&self) -> u16 { + self.server.port + } +} + +async fn start_live_managed() -> LiveManaged { + let tmp = tempfile::TempDir::new().unwrap(); + let services_dir = tmp.path().join("services"); + let server = spawn_server(|_| vllm_metrics()).await; + write_record(&services_dir, server.port); + let (mut rx, runner) = spawn_runner(managed_opts(services_dir.clone())); + snapshot_where(&mut rx, |snap| { + instance(snap, SVC_ID) + .filter(|i| approx(i.kv_cache_usage_pct, KV_PCT)) + .map(|_| ()) + }) + .await; + LiveManaged { + rx, + services_dir, + server, + _tmp: tmp, + _runner: runner, + } +} + +/// Wait until the service is announced gone and a snapshot no longer lists it: +/// the announcement and the snapshot are the two things a dashboard shows. +async fn assert_service_is_retired(rx: &mut broadcast::Receiver) { + next_event(rx, |ev| match ev { + Event::InstanceGone { container_id } if container_id == SVC_ID => Some(()), + _ => None, + }) + .await; + snapshot_where(rx, |snap| instance(snap, SVC_ID).is_none().then_some(())).await; +} + +/// Replace the registry directory with a regular file of the same name, so +/// listing it fails with something other than "not found". Portable: it needs no +/// permission change, so it behaves the same for a privileged user and on Windows. +/// +/// The directory is briefly absent between the two calls, which would read as a +/// confirmed-empty registry. The runner is a task on this test's own +/// current-thread runtime and reads the registry synchronously, so it cannot run +/// between them: nothing here awaits. A test that moves to a multi-thread runtime +/// must swap atomically instead. +fn replace_registry_dir_with_a_file(services_dir: &std::path::Path) { + std::fs::remove_dir_all(services_dir).unwrap(); + std::fs::write(services_dir, b"not a directory").unwrap(); +} + +/// A registry that is confirmed empty (the directory is there and holds no +/// record) retires the service it knew about. +#[tokio::test] +async fn managed_confirmed_empty_registry_retires_the_known_instance() { + let mut live = start_live_managed().await; + + std::fs::remove_file(live.services_dir.join(format!("{SVC_ID}.json"))).unwrap(); + + assert_service_is_retired(&mut live.rx).await; +} + +/// A registry directory that does not exist is also confirmed empty: it is +/// created the first time something is served, so its absence means no services. +#[tokio::test] +async fn managed_absent_registry_dir_retires_the_known_instance() { + let mut live = start_live_managed().await; + + std::fs::remove_dir_all(&live.services_dir).unwrap(); + + assert_service_is_retired(&mut live.rx).await; +} + +/// What the stream and the snapshots showed over a window of snapshots. +#[derive(Default)] +struct Window { + /// Every discovered/gone announcement for [`SVC_ID`], in order. + announcements: Vec<&'static str>, + snapshots: usize, + /// Snapshots in the window that did not list the service. + missing: usize, + /// The service as the last snapshot in the window listed it. + last: Option, + /// The warnings the last snapshot in the window carried. + warnings: Vec, +} + +/// Watch `count` snapshots, recording what was announced and what each snapshot +/// listed, under one deadline. +async fn watch(rx: &mut broadcast::Receiver, count: usize) -> Window { + let mut window = Window::default(); + observe( + rx, + "a window of snapshots", + &mut window, + |w, ev| { + match ev { + Event::InstanceDiscovered(inst) if inst.container_id == SVC_ID => { + w.announcements.push("discovered"); + } + Event::InstanceGone { container_id } if container_id == SVC_ID => { + w.announcements.push("gone"); + } + Event::Snapshot(snap) => { + w.snapshots += 1; + w.last = instance(snap, SVC_ID).cloned(); + w.missing += usize::from(w.last.is_none()); + w.warnings.clone_from(&snap.warnings); + } + _ => {} + } + w.snapshots >= count + }, + |w| { + format!( + "{} snapshots, {} without the service, announcements {:?}", + w.snapshots, w.missing, w.announcements + ) + }, + ) + .await; + window +} + +fn registry_warning(window: &Window) -> Option<&String> { + window + .warnings + .iter() + .find(|w| w.starts_with("managed registry")) +} + +/// Snapshots to watch across a read failure: several discovery ticks, so a loop +/// that retired the service would have done so well before the window ends. +const OUTAGE_SNAPSHOTS: usize = 8; + +/// The service as a still-scraped managed instance should look: present, with +/// the live fields its endpoint reports. +fn assert_still_live(inst: &Instance) { + assert!(inst.port.is_some(), "the instance lost its endpoint"); + assert!( + approx(inst.kv_cache_usage_pct, KV_PCT), + "the instance lost its kv cache reading: {:?}", + inst.kv_cache_usage_pct + ); + assert_eq!( + inst.running_reqs, + Some(1), + "the instance lost its request count" + ); +} + +/// A registry directory that cannot be listed says nothing about the services in +/// it, so the known service stays: not announced gone, still in every snapshot +/// with the data it holds, and the header says why the registry is not being +/// followed. +#[tokio::test] +async fn managed_unlistable_registry_dir_keeps_the_known_instance() { + let mut live = start_live_managed().await; + + replace_registry_dir_with_a_file(&live.services_dir); + let window = watch(&mut live.rx, OUTAGE_SNAPSHOTS).await; + + assert_eq!( + window.announcements, + Vec::<&str>::new(), + "a failed read must not announce the service gone or discovered" + ); + assert_eq!(window.missing, 0, "a snapshot dropped the service"); + assert_still_live(window.last.as_ref().expect("the service is listed")); + let warning = registry_warning(&window).expect("the header should say the registry is unread"); + assert!( + warning.contains("keeping the last known services"), + "{warning}" + ); +} + +/// Restores a directory's permissions on drop, so the temp dir can be cleaned +/// up even when the test fails part-way. +#[cfg(unix)] +struct RestorePermissions(std::path::PathBuf); + +#[cfg(unix)] +impl Drop for RestorePermissions { + fn drop(&mut self) { + use std::os::unix::fs::PermissionsExt; + let _ = std::fs::set_permissions(&self.0, std::fs::Permissions::from_mode(0o755)); + } +} + +/// The same through a permission error rather than a path error. Unix only, +/// because Windows has no mode bits to drop; skipped when the process can still +/// list the directory with its read bit removed (running as root). +#[cfg(unix)] +#[tokio::test] +async fn managed_unreadable_registry_dir_keeps_the_known_instance() { + use std::os::unix::fs::PermissionsExt; + let mut live = start_live_managed().await; + let _restore = RestorePermissions(live.services_dir.clone()); + + std::fs::set_permissions(&live.services_dir, std::fs::Permissions::from_mode(0o000)).unwrap(); + if std::fs::read_dir(&live.services_dir).is_ok() { + eprintln!("skipped: this process can list a mode-000 directory (running as root)"); + return; + } + let window = watch(&mut live.rx, OUTAGE_SNAPSHOTS).await; + + assert_eq!(window.announcements, Vec::<&str>::new()); + assert_eq!(window.missing, 0, "a snapshot dropped the service"); + assert_still_live(window.last.as_ref().expect("the service is listed")); + assert!(registry_warning(&window).is_some(), "{:?}", window.warnings); +} + +/// When the registry becomes readable again with the same service in it, the +/// service was never gone: nothing is announced, and the warning clears. +#[tokio::test] +async fn managed_registry_recovery_does_not_reannounce_the_known_instance() { + let mut live = start_live_managed().await; + let port = live.last_port(); + + replace_registry_dir_with_a_file(&live.services_dir); + let outage = watch(&mut live.rx, OUTAGE_SNAPSHOTS).await; + assert!(registry_warning(&outage).is_some()); + + std::fs::remove_file(&live.services_dir).unwrap(); + write_record(&live.services_dir, port); + let after = watch(&mut live.rx, OUTAGE_SNAPSHOTS).await; + + assert_eq!( + after.announcements, + Vec::<&str>::new(), + "recovery must not read as a discovery storm" + ); + assert_eq!(after.missing, 0); + assert_still_live(after.last.as_ref().expect("the service is listed")); + assert_eq!( + registry_warning(&after), + None, + "the warning must clear once the registry reads again" + ); +} + +/// A registry that comes back readable and empty is a confirmed-empty registry: +/// the service that was kept through the outage is retired now. +#[tokio::test] +async fn managed_registry_recovering_empty_retires_the_instance_kept_through_the_outage() { + let mut live = start_live_managed().await; + + replace_registry_dir_with_a_file(&live.services_dir); + let outage = watch(&mut live.rx, OUTAGE_SNAPSHOTS).await; + assert_eq!(outage.missing, 0); + + std::fs::remove_file(&live.services_dir).unwrap(); + std::fs::create_dir(&live.services_dir).unwrap(); + + assert_service_is_retired(&mut live.rx).await; +} + +/// One corrupt record does not hide the others: the well-formed service stays +/// and a new well-formed one beside the corrupt file is still discovered. +#[tokio::test] +async fn managed_malformed_record_does_not_hide_the_other_services() { + let mut live = start_live_managed().await; + let port = live.last_port(); + + std::fs::write(live.services_dir.join("corrupt.json"), "{ not valid").unwrap(); + write_record_for(&live.services_dir, "second-svc", port); + + let second_announced = std::cell::Cell::new(false); + let mut window = Window::default(); + observe( + &mut live.rx, + "the second service to be announced beside a corrupt record", + &mut window, + |w, ev| { + match ev { + Event::InstanceDiscovered(inst) if inst.container_id == "second-svc" => { + second_announced.set(true); + } + Event::InstanceGone { container_id } => { + panic!("{container_id} was announced gone beside a corrupt record") + } + Event::Snapshot(snap) => { + w.snapshots += 1; + w.missing += usize::from(instance(snap, SVC_ID).is_none()); + w.warnings.clone_from(&snap.warnings); + } + _ => {} + } + second_announced.get() && w.snapshots >= SETTLE_SNAPSHOTS + }, + |w| { + format!( + "{} snapshots, second announced: {}", + w.snapshots, + second_announced.get() + ) + }, + ) + .await; + + assert_eq!(window.missing, 0, "the well-formed service was dropped"); + assert_eq!( + registry_warning(&window), + None, + "a corrupt record is not an unreadable registry" + ); +} + +// ── Managed-service registry: one record that cannot be opened ────────────── +// +// State matrix for a registry where some record cannot be read, one test per +// row (the rows with no unreadable record, an empty registry, and a directory +// that cannot be listed are the tests above): +// +// | State | Announced | Snapshots list | Header warning | +// | -------------------------------------------------- | ---------------- | ----------------- | -------------- | +// | one record unreadable, a new valid record appears | the new one once | known + new | yes | +// | one record unreadable, a known record is removed | nothing | known stays | yes | +// | the same, then the unreadable record is fixed | gone, once | the removed one goes | clears | +// | the known service's own record becomes unreadable | nothing | known stays | yes | +// | ... and then reads again | nothing | known stays | clears | +// +// A record that cannot be opened is a symbolic link to itself, which fails with a +// loop error (not "not found", which would read as a record removed on purpose), +// needs no permission change and so also works when the tests run as root. + +/// Make `.json` in the registry something that cannot be opened. +#[cfg(unix)] +fn write_unreadable_record(services_dir: &std::path::Path, name: &str) { + let file = format!("{name}.json"); + // A relative target, so the link keeps pointing at itself when renamed. + std::os::unix::fs::symlink(&file, services_dir.join(&file)).unwrap(); +} + +/// What the stream and the snapshots showed, for every service, over a window of +/// snapshots. +#[cfg(unix)] +#[derive(Default)] +struct Roster { + /// Every discovered/gone announcement, in order, with the service id. + announcements: Vec<(String, &'static str)>, + snapshots: usize, + /// Snapshots in the window that did not list [`SVC_ID`]. + missing_known: usize, + /// The ids the last snapshot in the window listed, sorted. + ids: Vec, + /// [`SVC_ID`] as the last snapshot in the window listed it. + known: Option, + /// The warnings the last snapshot in the window carried. + warnings: Vec, +} + +#[cfg(unix)] +impl Roster { + fn registry_warning(&self) -> Option<&String> { + self.warnings + .iter() + .find(|w| w.starts_with("managed registry")) + } +} + +/// Watch `count` snapshots, recording every announcement and what the snapshots +/// listed, under one deadline. +#[cfg(unix)] +async fn watch_roster(rx: &mut broadcast::Receiver, count: usize) -> Roster { + let mut roster = Roster::default(); + observe( + rx, + "a window of snapshots", + &mut roster, + |r, ev| { + match ev { + Event::InstanceDiscovered(inst) => r + .announcements + .push((inst.container_id.clone(), "discovered")), + Event::InstanceGone { container_id } => { + r.announcements.push((container_id.clone(), "gone")); + } + Event::Snapshot(snap) => { + r.snapshots += 1; + r.known = instance(snap, SVC_ID).cloned(); + r.missing_known += usize::from(r.known.is_none()); + r.ids = snap + .instances + .iter() + .map(|i| i.container_id.clone()) + .collect(); + r.ids.sort(); + r.warnings.clone_from(&snap.warnings); + } + _ => {} + } + r.snapshots >= count + }, + |r| { + format!( + "{} snapshots, listing {:?}, announcements {:?}", + r.snapshots, r.ids, r.announcements + ) + }, + ) + .await; + roster +} + +/// A new service is added while one other record cannot be opened: the readable +/// records still add their services, announced once, and nothing is retired. +#[cfg(unix)] +#[tokio::test] +async fn managed_unreadable_record_does_not_hide_a_new_service() { + let mut live = start_live_managed().await; + let port = live.last_port(); + + write_unreadable_record(&live.services_dir, "stale"); + write_record_for(&live.services_dir, "second-svc", port); + let roster = watch_roster(&mut live.rx, OUTAGE_SNAPSHOTS).await; + + assert_eq!( + roster.announcements, + [("second-svc".to_string(), "discovered")], + "the new service is announced once, and nothing is announced gone" + ); + assert_eq!( + roster.missing_known, 0, + "a snapshot dropped the known service" + ); + assert_eq!(roster.ids, [SVC_ID.to_string(), "second-svc".to_string()]); + let warning = roster + .registry_warning() + .expect("the header should name the unreadable record"); + assert!(warning.contains("stale.json"), "{warning}"); +} + +/// While a record cannot be opened a service whose record is removed keeps its +/// row, because the unreadable record could be any known service's; once the +/// record reads again the registry is read in full and the removed service is +/// retired, with the warning cleared. +#[cfg(unix)] +#[tokio::test] +async fn managed_unreadable_record_defers_retirement_until_it_reads_again() { + let mut live = start_live_managed().await; + + write_unreadable_record(&live.services_dir, "stale"); + std::fs::remove_file(live.services_dir.join(format!("{SVC_ID}.json"))).unwrap(); + let during = watch_roster(&mut live.rx, OUTAGE_SNAPSHOTS).await; + + assert_eq!( + during.announcements, + Vec::<(String, &str)>::new(), + "a removed record must not retire anything while another cannot be read" + ); + assert_eq!(during.missing_known, 0, "a snapshot dropped the service"); + assert_eq!(during.ids, [SVC_ID.to_string()]); + assert!(during.registry_warning().is_some(), "{:?}", during.warnings); + + std::fs::remove_file(live.services_dir.join("stale.json")).unwrap(); + assert_service_is_retired(&mut live.rx).await; + let after = watch_roster(&mut live.rx, SETTLE_SNAPSHOTS).await; + assert_eq!(after.ids, Vec::::new()); + assert_eq!( + after.registry_warning(), + None, + "the warning must clear once the registry reads in full" + ); +} + +/// The known service's own record becomes unreadable (here it is replaced by a +/// self-referencing link, atomically): the service keeps its row and its live +/// fields, and is not announced again when the record reads again. +#[cfg(unix)] +#[tokio::test] +async fn managed_own_record_becoming_unreadable_keeps_the_instance_and_recovers_quietly() { + let mut live = start_live_managed().await; + let port = live.last_port(); + + let tmp_link = live.services_dir.join(format!("{SVC_ID}.json.tmp")); + std::os::unix::fs::symlink(format!("{SVC_ID}.json"), &tmp_link).unwrap(); + std::fs::rename(&tmp_link, live.services_dir.join(format!("{SVC_ID}.json"))).unwrap(); + let during = watch_roster(&mut live.rx, OUTAGE_SNAPSHOTS).await; + + assert_eq!(during.announcements, Vec::<(String, &str)>::new()); + assert_eq!(during.missing_known, 0, "a snapshot dropped the service"); + assert_still_live(during.known.as_ref().expect("the service is listed")); + assert!(during.registry_warning().is_some(), "{:?}", during.warnings); + + write_record(&live.services_dir, port); + let after = watch_roster(&mut live.rx, OUTAGE_SNAPSHOTS).await; + assert_eq!(after.announcements, Vec::<(String, &str)>::new()); + assert_eq!(after.missing_known, 0); + assert_eq!(after.ids, [SVC_ID.to_string()]); + assert_eq!(after.registry_warning(), None); +} + // ── Lemonade endpoint ─────────────────────────────────────────────────────── fn lemonade_responder(path: &str) -> String { diff --git a/crates/rocm-dash-daemon/tests/registry_read_logging.rs b/crates/rocm-dash-daemon/tests/registry_read_logging.rs new file mode 100644 index 0000000000..a40daa8db2 --- /dev/null +++ b/crates/rocm-dash-daemon/tests/registry_read_logging.rs @@ -0,0 +1,163 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! What the run loop logs about the managed-service registry's health. +//! +//! A read failure that lasts many discovery ticks is logged once, its recovery +//! once, and a second outage once more: one line per streak, not one per tick. +//! +//! This is its own test binary, with a single test, on purpose. `tracing` caches +//! whether a callsite is enabled the first time any thread reaches it; when a +//! test thread installs a per-thread subscriber after another thread has already +//! run the same `warn!` with none installed, the cached answer drops the event +//! and the capture comes up empty. In `discovery_reconcile.rs` the tests share a +//! process, so that is what happened there. Here nothing else logs first. + +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use rocm_dash_core::protocol::Event; +use tokio::sync::broadcast; +use tokio::time::timeout; + +use rocm_dash_daemon::bench_ring::BenchRing; +use rocm_dash_daemon::runner::{self, RunnerOptions}; +use rocm_dash_daemon::snapshot_ring::SnapshotRing; + +const TICK: Duration = Duration::from_millis(100); +const DEADLINE: Duration = Duration::from_secs(20); +const SVC_ID: &str = "logging-svc"; +/// Snapshots to watch across a streak: several discovery ticks. +const STREAK_SNAPSHOTS: usize = 8; +const FAILED: &str = "managed-service registry could not be read"; +const RECOVERED: &str = "managed-service registry is readable again"; + +/// A shared buffer a `tracing` subscriber writes to, so the test can count what +/// the loop logged. +#[derive(Clone, Default)] +struct LogCapture(Arc>>); + +impl std::io::Write for LogCapture { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.lock().unwrap().extend_from_slice(buf); + Ok(buf.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} + +impl LogCapture { + fn text(&self) -> String { + String::from_utf8_lossy(&self.0.lock().unwrap()).into_owned() + } + + fn lines_containing(&self, needle: &str) -> usize { + self.text() + .lines() + .filter(|line| line.contains(needle)) + .count() + } +} + +fn write_record(services_dir: &std::path::Path) { + std::fs::create_dir_all(services_dir).unwrap(); + // Nothing listens on the port, so the service is never scraped: this test is + // about discovery only. A rename keeps the tick from reading half a record. + let json = format!( + r#"{{"service_id":"{SVC_ID}","engine":"vllm","model_ref":"Logging/1B","canonical_model_id":"Logging/1B","host":"127.0.0.1","port":9,"status":"ready","created_at_unix_ms":0}}"# + ); + let tmp = services_dir.join(format!("{SVC_ID}.json.tmp")); + std::fs::write(&tmp, json).unwrap(); + std::fs::rename(&tmp, services_dir.join(format!("{SVC_ID}.json"))).unwrap(); +} + +/// Replace the registry directory with a regular file, so listing it fails. +fn break_registry(services_dir: &std::path::Path) { + std::fs::remove_dir_all(services_dir).unwrap(); + std::fs::write(services_dir, b"not a directory").unwrap(); +} + +/// Wait for `count` more snapshots, under one deadline. +async fn watch(rx: &mut broadcast::Receiver, count: usize) { + let deadline = tokio::time::Instant::now() + DEADLINE; + let mut seen = 0; + while seen < count { + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + match timeout(remaining, rx.recv()).await { + Ok(Ok(Event::Snapshot(_))) => seen += 1, + Ok(Ok(_) | Err(broadcast::error::RecvError::Lagged(_))) => {} + Ok(Err(broadcast::error::RecvError::Closed)) => panic!("broadcast channel closed"), + Err(elapsed) => { + panic!("only {seen} of {count} snapshots within {DEADLINE:?} ({elapsed})") + } + } + } +} + +/// The default subscriber is per thread and `#[tokio::test]` runs the loop on +/// the test's own thread, so the capture sees the loop's output. +#[tokio::test] +async fn registry_read_failure_is_logged_once_per_streak() { + let capture = LogCapture::default(); + let writer = capture.clone(); + let subscriber = tracing_subscriber::fmt() + .with_max_level(tracing::Level::INFO) + .with_ansi(false) + .with_writer(move || writer.clone()) + .finish(); + let _guard = tracing::subscriber::set_default(subscriber); + + let tmp = tempfile::TempDir::new().unwrap(); + let services_dir = tmp.path().join("services"); + write_record(&services_dir); + let (tx, mut rx) = broadcast::channel::(1024); + let _runner = tokio::spawn(runner::run_loop( + Some(TICK), + tx, + Arc::new(Mutex::new(SnapshotRing::new(16))), + Arc::new(Mutex::new(BenchRing::new(4))), + None, + RunnerOptions { + services_dir: Some(services_dir.clone()), + discovery_tick: TICK, + instance_tick: TICK * 2, + gpu_tick: TICK, + ..RunnerOptions::default() + }, + )); + watch(&mut rx, 3).await; + assert_eq!( + capture.lines_containing(FAILED), + 0, + "a readable registry logs nothing: {}", + capture.text() + ); + + break_registry(&services_dir); + watch(&mut rx, STREAK_SNAPSHOTS).await; + assert_eq!( + capture.lines_containing(FAILED), + 1, + "a streak of failed ticks must log one warning: {}", + capture.text() + ); + assert_eq!(capture.lines_containing(RECOVERED), 0); + + std::fs::remove_file(&services_dir).unwrap(); + write_record(&services_dir); + watch(&mut rx, STREAK_SNAPSHOTS).await; + assert_eq!(capture.lines_containing(RECOVERED), 1, "{}", capture.text()); + assert_eq!(capture.lines_containing(FAILED), 1); + + break_registry(&services_dir); + watch(&mut rx, STREAK_SNAPSHOTS).await; + assert_eq!( + capture.lines_containing(FAILED), + 2, + "a second streak is a new warning: {}", + capture.text() + ); + assert_eq!(capture.lines_containing(RECOVERED), 1); +} diff --git a/docs/architecture.md b/docs/architecture.md index bbd9d4b6f2..0e7a675d38 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -280,7 +280,7 @@ The other three dashboard crates are described at crate level here, not module b - [`rocm-dash-collectors`](../crates/rocm-dash-collectors/src/lib.rs) — the IO-heavy collector implementations behind `rocm-dash-core`'s traits (amd-smi, Docker, host metrics, Lemonade and vLLM scrapers, and on Windows the [`winperf.rs`](../crates/rocm-dash-collectors/src/winperf.rs) perf-counter/ADLX GPU collector) plus the benchmark load generator; used by `rocm-dash-daemon` and, on Windows, by `rocm-dash-tui` in-process. - [`rocm-dash-core`](../crates/rocm-dash-core/src/lib.rs) — pure types, traits, schemas and the reducer, with no rendering dependencies. -- [`rocm-dash-daemon`](../crates/rocm-dash-daemon/src/lib.rs) — the daemon library: [`runner.rs`](../crates/rocm-dash-daemon/src/runner.rs) drives the collectors on tick cadences and broadcasts snapshots, and [`server.rs`](../crates/rocm-dash-daemon/src/server.rs) streams them to TUI clients as NDJSON. `apps/rocm`'s [`dash.rs`](../apps/rocm/src/dash.rs) drives it. +- [`rocm-dash-daemon`](../crates/rocm-dash-daemon/src/lib.rs) — the daemon library: [`runner.rs`](../crates/rocm-dash-daemon/src/runner.rs) drives the collectors on tick cadences and broadcasts snapshots, and [`server.rs`](../crates/rocm-dash-daemon/src/server.rs) streams them to TUI clients as NDJSON. [`registry.rs`](../crates/rocm-dash-daemon/src/registry.rs) reads the managed-service records `rocm serve` writes and tells a registry that is confirmed empty (no directory, or no records) from one that could not be read: the run loop retires a managed service that an empty registry lacks, but when a pass cannot read the registry in full (a directory that cannot be listed, or a record file that cannot be opened) it retires nothing: the known services and their data stay, as when Docker discovery errors, the records that did read still add and refresh their services, and the header warns until a read succeeds. `apps/rocm`'s [`dash.rs`](../apps/rocm/src/dash.rs) drives it. ### `crates/rocm-engine-protocol` — engine IPC protocol diff --git a/tests/e2e-cucumber/features/dash.feature b/tests/e2e-cucumber/features/dash.feature index 5892ab6b04..227cbc16d6 100644 --- a/tests/e2e-cucumber/features/dash.feature +++ b/tests/e2e-cucumber/features/dash.feature @@ -740,3 +740,47 @@ Feature: Interactive dashboard And the instance detail lists recent throughput samples When the user quits the dashboard Then the dashboard exits successfully + + # The dashboard follows the managed-service registry on a timer. A pass that + # cannot read the registry says nothing about the services in it, so it must + # not retire the model that is on screen: the header warns that the registry + # is not being read, and the model stays. The registry is made unreadable by + # adding one record that cannot be opened (a link to itself), in a single step + # that leaves the model's own record in place, so no poll of the registry can + # see it empty or absent, and no permission setup of the runner (running as + # root) can make it readable. A registry that reads but is empty does retire + # a service; that contrast, the recovery path and the once-per-streak log + # line are asserted by the daemon's own tests in + # `crates/rocm-dash-daemon/tests/`, where the loop runs without a terminal. + @id:dash-managed-model-survives-unreadable-registry @requires-os:linux + Scenario: dash-43 - A managed model stays on screen while its registry cannot be read + Given a running managed model is available locally + When the user opens the dashboard + And the user opens the Observe view + Then the managed model is displayed + When the managed-service registry becomes unreadable + Then the header warns that the registry is not being read + And the managed model stays displayed + When the user quits the dashboard + Then the dashboard exits successfully + + # The same registry failure, from the other side: a service whose record can be + # read is still added while some other record cannot be opened. The dashboard + # reads what it can and retires nothing, so a model that is started beside a + # stale unreadable record appears on screen (and the first model stays). The + # daemon's own tests in `crates/rocm-dash-daemon/tests/` assert the rest of the + # matrix: a record removed during the failure is not retired until the registry + # reads in full, and a record that becomes unreadable is not announced again + # when it reads again. + @id:dash-new-model-appears-beside-unreadable-record @requires-os:linux + Scenario: dash-44 - A newly served model appears while another record in the registry cannot be read + Given a running managed model is available locally + When the user opens the dashboard + And the user opens the Observe view + Then the managed model is displayed + When the managed-service registry becomes unreadable + And another managed model is served + Then the other managed model is displayed + And the managed model stays displayed + When the user quits the dashboard + Then the dashboard exits successfully diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index d30920da4a..c19c9c139e 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -2610,6 +2610,165 @@ async fn metrics_endpoint_fails(world: &mut E2eWorld) { }); } +/// Create `path` as a symbolic link to itself, so opening it fails with a loop +/// error. The scenario that uses it is `@requires-os:linux`. +#[cfg(unix)] +fn make_unreadable_record(path: &std::path::Path) { + std::os::unix::fs::symlink(path, path).expect("failed to plant the unreadable record"); +} + +#[cfg(not(unix))] +fn make_unreadable_record(_path: &std::path::Path) { + panic!("the unreadable-registry scenario is Linux-only (@requires-os:linux)"); +} + +/// Make the managed-service registry unreadable and wait for the failure to +/// reach the screen. The registry is made unreadable by adding one record file +/// that cannot be opened: a symbolic link to itself, which fails with a loop +/// error rather than "not found" (a dangling link would read as a record that +/// was removed on purpose, and be skipped). That is a single atomic step that +/// leaves the existing record and the directory in place, and it works for a +/// privileged user, which a permission change does not. The dashboard process +/// polls the registry on its own schedule, so the step must never expose a +/// registry with the records gone: deleting the directory before putting +/// something in its place would let a poll in between read an absent registry, +/// which is a confirmed-empty one, and retire the model the scenario expects to +/// survive. The header warnings badge growing is the evidence that a frame the +/// daemon assembled after the failed pass is on display (the daemon adds a +/// warning for as long as the registry cannot be read); the count is read before +/// the change because other warnings, such as no amd-smi on a GPU-less runner, +/// are already present and constant. +#[when("the managed-service registry becomes unreadable")] +async fn managed_registry_becomes_unreadable(world: &mut E2eWorld) { + let before = warning_badge_count(&session(world).screen_text()); + let services_dir = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("data") + .join("services"); + let looping = services_dir.join("unreadable.json"); + make_unreadable_record(&looping); + + session(world) + .wait_for_screen_where( + &format!("the header warnings badge grows past {before}"), + |screen| warning_badge_count(screen) > before, + default_timeout(), + ) + .await + .unwrap_or_else(|e| { + panic!( + "the unreadable registry never reached the header warnings badge, so no \ + frame here is known to postdate the failed read: {e}" + ) + }); +} + +/// The badge growing is what the `When` step waited for; this reads it back so +/// the scenario states the expectation, and a failure here names the half that +/// broke: the registry failure went unreported, rather than the model dropped. +#[then("the header warns that the registry is not being read")] +async fn header_warns_registry_unread(world: &mut E2eWorld) { + assert!( + warning_badge_count(&session(world).screen_text()) > 0, + "the header shows no warning after the registry became unreadable" + ); +} + +/// The model must be on screen in the frame that carries the warning and stay +/// on every frame for a few seconds after it. A daemon that retired the model +/// on a failed read would drop it in the same frame as the warning appears, so +/// the first sample already catches it; the dwell covers a retirement that +/// trails by a discovery tick. +#[then("the managed model stays displayed")] +async fn managed_model_stays_displayed(world: &mut E2eWorld) { + let model = world + .model_name + .as_deref() + .expect("no model name set") + .to_string(); + let deadline = Instant::now() + Duration::from_secs(3); + loop { + let screen = session(world).screen_text(); + assert!( + screen.contains(&model), + "the managed model left the screen while its registry could not be read:\n{screen}" + ); + if Instant::now() >= deadline { + return; + } + tokio::time::sleep(Duration::from_millis(200)).await; + } +} + +/// The model `another managed model is served` adds beside the first one. +const OTHER_MODEL: &str = "TestModel/E2E-2B"; + +/// Serve a second model: write a record for it into the registry, as `rocm serve` +/// does, by copying the first model's record under another service id and name. +/// It is written to a temporary name and renamed into place, so a poll of the +/// registry never reads half a record (which would read as a malformed one, not +/// as the new service). +/// +/// The record points at a disposable `sleep` child, not at the harness, for the +/// reason `managed_model_record_points_at_disposable_process` gives: the World's +/// teardown runs `rocm services stop --yes` for every record but the first +/// model's, and that signals whatever process the record names. The child is kept +/// in `recorded_process`, which teardown reaps. +#[when("another managed model is served")] +async fn another_managed_model_is_served(world: &mut E2eWorld) { + let child = std::process::Command::new("sleep") + .arg("300") + .spawn() + .expect("failed to spawn the disposable process for the second model"); + let pid = child.id(); + let start_ticks = rocm_core::process_start_ticks(pid); + world.recorded_process = Some(child); + let services_dir = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("data") + .join("services"); + let first = std::fs::read(services_dir.join("e2e-mock.json")) + .expect("the first model's record should be in the registry"); + let mut record: serde_json::Value = + serde_json::from_slice(&first).expect("the first model's record should be JSON"); + record["service_id"] = "e2e-second".into(); + record["model_ref"] = OTHER_MODEL.into(); + record["canonical_model_id"] = OTHER_MODEL.into(); + record["supervisor_pid"] = pid.into(); + record["engine_pid"] = pid.into(); + record["supervisor_start_ticks"] = serde_json::to_value(start_ticks).unwrap(); + let staged = services_dir.join("e2e-second.json.tmp"); + std::fs::write( + &staged, + serde_json::to_vec_pretty(&record).expect("failed to serialize the second record"), + ) + .expect("failed to stage the second record"); + std::fs::rename(&staged, services_dir.join("e2e-second.json")) + .expect("failed to publish the second record"); +} + +/// The second model is on screen although the registry still holds a record that +/// cannot be opened. The step before it left the header warning in place, so the +/// registry is known to be failing while this waits. +#[then("the other managed model is displayed")] +async fn other_managed_model_displayed(world: &mut E2eWorld) { + session(world) + .wait_for_screen(OTHER_MODEL, default_timeout()) + .await + .unwrap_or_else(|e| { + panic!( + "the newly served model never appeared while one registry record could not \ + be read: {e}" + ) + }); +} + /// EAI-7960 principal regression assertion. /// /// The frame under assertion is provably post-failure (the header warnings