From a54889aafda4ebf06d61d475583390b38a3865a9 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 4 Nov 2026 09:12:24 +0000 Subject: [PATCH 1/8] ROCMAI-852: pin the runner's discovery reconcile before restructuring it Drive the real run_loop through the managed-service registry and the Lemonade endpoint and assert what the broadcast stream shows: a service is announced once however many ticks rediscover it, it keeps its scraped KV cache and request counts on ticks that do not scrape, it starts clean when the registry moves it to a new port, and a vanished Lemonade endpoint is announced gone and leaves the snapshots. Docker discovery needs a container daemon, so it is left to the unit tests that come with the shared helper. Signed-off-by: Jussi Elo --- .../tests/discovery_reconcile.rs | 387 ++++++++++++++++++ 1 file changed, 387 insertions(+) create mode 100644 crates/rocm-dash-daemon/tests/discovery_reconcile.rs diff --git a/crates/rocm-dash-daemon/tests/discovery_reconcile.rs b/crates/rocm-dash-daemon/tests/discovery_reconcile.rs new file mode 100644 index 0000000000..f2276f0886 --- /dev/null +++ b/crates/rocm-dash-daemon/tests/discovery_reconcile.rs @@ -0,0 +1,387 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Characterisation tests for the runner's service-discovery reconcile. +//! +//! `run_loop` reconciles three discovery sources against its reducer state: +//! Docker, the probe-based Lemonade endpoint, and the managed-service registry. +//! These tests drive the real loop through the sources that can run without a +//! container daemon (the registry and Lemonade) and pin what a user of the +//! broadcast stream can observe: a service is announced once, keeps its live +//! scrape fields across rediscovery, starts clean when its endpoint changes, and +//! is announced gone exactly when it disappears. +//! +//! The Docker source has no seam that works without a container daemon, so its +//! reconcile is covered only by the unit tests next to the shared helper in +//! `runner.rs`. + +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use rocm_dash_core::metrics::{Instance, InstanceStatus, Snapshot}; +use rocm_dash_core::protocol::Event; +use tokio::io::{AsyncReadExt, AsyncWriteExt}; +use tokio::net::TcpListener; +use tokio::sync::broadcast; +use tokio::time::timeout; + +use rocm_dash_daemon::bench_ring::BenchRing; +use rocm_dash_daemon::runner::{self, RunnerOptions}; +use rocm_dash_daemon::snapshot_ring::SnapshotRing; + +const TICK: Duration = Duration::from_millis(200); +const INSTANCE_TICK: Duration = Duration::from_millis(400); +const DEADLINE: Duration = Duration::from_secs(20); +const SVC_ID: &str = "reconcile-svc"; + +// ── Tiny scriptable HTTP server ───────────────────────────────────────────── + +/// A local HTTP server whose reply is chosen per request path. Setting `down` +/// makes it answer 503 to everything, which is how a test takes a service away +/// without closing the port. +struct MockServer { + port: u16, + down: Arc, +} + +async fn spawn_server(respond: impl Fn(&str) -> String + Send + Sync + 'static) -> MockServer { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let down = Arc::new(AtomicBool::new(false)); + let respond = Arc::new(respond); + let down_for_task = Arc::clone(&down); + tokio::spawn(async move { + loop { + let Ok((mut sock, _)) = listener.accept().await else { + return; + }; + let respond = Arc::clone(&respond); + let down = Arc::clone(&down_for_task); + tokio::spawn(async move { + let mut buf = vec![0u8; 4096]; + let n = sock.read(&mut buf).await.unwrap_or(0); + let request = String::from_utf8_lossy(&buf[..n]); + let path = request + .lines() + .next() + .and_then(|line| line.split_whitespace().nth(1)) + .unwrap_or("/") + .to_string(); + let reply = if down.load(Ordering::Relaxed) { + http(503, "text/plain", "") + } else { + respond(&path) + }; + let _ = sock.write_all(reply.as_bytes()).await; + let _ = sock.shutdown().await; + }); + } + }); + MockServer { port, down } +} + +fn http(status: u16, content_type: &str, body: &str) -> String { + let reason = if status == 200 { "OK" } else { "Unavailable" }; + format!( + "HTTP/1.1 {status} {reason}\r\nContent-Type: {content_type}\r\n\ + Content-Length: {}\r\nConnection: close\r\n\r\n{body}", + body.len() + ) +} + +/// A vLLM `/metrics` body with a growing token counter, one running request and +/// a 25% KV cache. +fn vllm_metrics() -> String { + http( + 200, + "text/plain; version=0.0.4", + "\ +# TYPE vllm:num_requests_running gauge +vllm:num_requests_running{model=\"mock\"} 1 +# TYPE vllm:num_requests_waiting gauge +vllm:num_requests_waiting{model=\"mock\"} 0 +# TYPE vllm:gpu_cache_usage_perc gauge +vllm:gpu_cache_usage_perc{model=\"mock\"} 0.25 +# TYPE vllm:generation_tokens_total counter +vllm:generation_tokens_total{model=\"mock\"} 100 +", + ) +} + +// ── Runner plumbing ───────────────────────────────────────────────────────── + +fn spawn_runner(opts: RunnerOptions) -> (broadcast::Receiver, tokio::task::JoinHandle<()>) { + let (tx, rx) = broadcast::channel::(1024); + let ring = Arc::new(Mutex::new(SnapshotRing::new(16))); + let bench_ring = Arc::new(Mutex::new(BenchRing::new(4))); + let handle = tokio::spawn(runner::run_loop( + Some(TICK), + tx, + ring, + bench_ring, + None, + opts, + )); + (rx, handle) +} + +fn managed_opts(services_dir: std::path::PathBuf) -> RunnerOptions { + RunnerOptions { + services_dir: Some(services_dir), + discovery_tick: TICK, + instance_tick: INSTANCE_TICK, + gpu_tick: TICK, + ..RunnerOptions::default() + } +} + +fn lemonade_opts(port: u16) -> RunnerOptions { + RunnerOptions { + enable_lemonade: true, + lemonade_host: "127.0.0.1".into(), + lemonade_port: port, + discovery_tick: TICK, + instance_tick: INSTANCE_TICK, + gpu_tick: TICK, + ..RunnerOptions::default() + } +} + +fn write_record(services_dir: &std::path::Path, port: u16) { + std::fs::create_dir_all(services_dir).unwrap(); + let json = format!( + r#"{{"service_id":"{SVC_ID}","engine":"vllm","model_ref":"Reconcile/1B","canonical_model_id":"Reconcile/1B","host":"127.0.0.1","port":{port},"status":"ready","created_at_unix_ms":0}}"# + ); + std::fs::write(services_dir.join(format!("{SVC_ID}.json")), json).unwrap(); +} + +/// Read events until `pick` returns a value, or fail at the deadline. +async fn next_event( + rx: &mut broadcast::Receiver, + pick: impl Fn(&Event) -> Option, +) -> T { + let deadline = tokio::time::Instant::now() + DEADLINE; + loop { + let remaining = deadline.saturating_duration_since(tokio::time::Instant::now()); + assert!( + !remaining.is_zero(), + "no matching event within {DEADLINE:?}" + ); + let Ok(received) = timeout(remaining, rx.recv()).await else { + panic!("no matching event within {DEADLINE:?}"); + }; + match received { + Ok(ev) => { + if let Some(v) = pick(&ev) { + return v; + } + } + Err(broadcast::error::RecvError::Lagged(_)) => {} + Err(broadcast::error::RecvError::Closed) => panic!("broadcast channel closed"), + } + } +} + +async fn next_snapshot(rx: &mut broadcast::Receiver) -> Snapshot { + next_event(rx, |ev| match ev { + Event::Snapshot(snap) => Some(snap.clone()), + _ => None, + }) + .await +} + +fn instance<'a>(snap: &'a Snapshot, id: &str) -> Option<&'a Instance> { + snap.instances.iter().find(|i| i.container_id == id) +} + +fn approx(value: Option, expected: f32) -> bool { + value.is_some_and(|v| (v - expected).abs() < 1e-3) +} + +/// The mock's `0.25` KV-cache fraction, which the collector reports as a percent. +const KV_PCT: f32 = 25.0; + +/// Read snapshots until `pick` returns a value. Bounded as a whole, so a +/// condition that never holds fails at [`DEADLINE`] instead of looping on fresh +/// snapshots forever. +async fn snapshot_where( + rx: &mut broadcast::Receiver, + pick: impl Fn(&Snapshot) -> Option, +) -> T { + let deadline = tokio::time::Instant::now() + DEADLINE; + loop { + assert!( + tokio::time::Instant::now() < deadline, + "no snapshot satisfied the condition within {DEADLINE:?}" + ); + let snap = next_snapshot(rx).await; + if let Some(v) = pick(&snap) { + return v; + } + } +} + +// ── Managed-service registry ──────────────────────────────────────────────── + +/// Rediscovery runs every tick but the scrape only every second one, so on the +/// ticks without a scrape the instance's KV cache and request counts survive +/// only because the discovery merge carries them over from the previous state. +#[tokio::test] +async fn managed_rediscovery_keeps_live_scrape_fields_between_scrapes() { + let tmp = tempfile::TempDir::new().unwrap(); + let services_dir = tmp.path().join("services"); + let server = spawn_server(|_| vllm_metrics()).await; + write_record(&services_dir, server.port); + let (mut rx, _runner) = spawn_runner(managed_opts(services_dir)); + + snapshot_where(&mut rx, |snap| { + instance(snap, SVC_ID) + .filter(|i| approx(i.kv_cache_usage_pct, KV_PCT)) + .map(|_| ()) + }) + .await; + + for n in 0..10 { + let snap = next_snapshot(&mut rx).await; + let inst = instance(&snap, SVC_ID) + .unwrap_or_else(|| panic!("snapshot {n} after the first scrape lost the instance")); + assert!( + approx(inst.kv_cache_usage_pct, KV_PCT), + "snapshot {n}: kv cache was dropped on a tick without a scrape: {:?}", + inst.kv_cache_usage_pct + ); + assert_eq!( + inst.running_reqs, + Some(1), + "snapshot {n}: running requests were dropped on a tick without a scrape" + ); + } +} + +/// A service is announced as discovered once, however many ticks rediscover it. +#[tokio::test] +async fn managed_service_is_announced_once_across_rediscovery() { + let tmp = tempfile::TempDir::new().unwrap(); + let services_dir = tmp.path().join("services"); + let server = spawn_server(|_| vllm_metrics()).await; + write_record(&services_dir, server.port); + let (mut rx, _runner) = spawn_runner(managed_opts(services_dir)); + + let mut announced = 0usize; + let mut snapshots_with_instance = 0usize; + while snapshots_with_instance < 8 { + match next_event(&mut rx, |ev| Some(ev.clone())).await { + Event::InstanceDiscovered(inst) if inst.container_id == SVC_ID => announced += 1, + Event::Snapshot(snap) if instance(&snap, SVC_ID).is_some() => { + snapshots_with_instance += 1; + } + _ => {} + } + } + assert_eq!( + announced, 1, + "the service must be announced exactly once, not once per discovery tick" + ); +} + +/// The registry's port is the authority: when it changes under the same service +/// id, the live fields scraped from the old port must not leak onto the new +/// endpoint, and the change must not be announced as a new service. +#[tokio::test] +async fn managed_endpoint_change_starts_the_instance_clean() { + let tmp = tempfile::TempDir::new().unwrap(); + let services_dir = tmp.path().join("services"); + let healthy = spawn_server(|_| vllm_metrics()).await; + // The new endpoint never answers a scrape, so any live field on the instance + // after the change was carried over from the old one. + let silent = spawn_server(|_| http(503, "text/plain", "")).await; + write_record(&services_dir, healthy.port); + let (mut rx, _runner) = spawn_runner(managed_opts(services_dir.clone())); + + snapshot_where(&mut rx, |snap| { + instance(snap, SVC_ID) + .filter(|i| approx(i.kv_cache_usage_pct, KV_PCT)) + .map(|_| ()) + }) + .await; + + write_record(&services_dir, silent.port); + let moved = snapshot_where(&mut rx, |snap| { + instance(snap, SVC_ID) + .filter(|i| i.port == Some(silent.port)) + .cloned() + }) + .await; + assert_eq!( + moved.kv_cache_usage_pct, None, + "the old endpoint's kv cache leaked onto the new one" + ); + assert_eq!( + moved.running_reqs, None, + "the old endpoint's request count leaked onto the new one" + ); +} + +// ── Lemonade endpoint ─────────────────────────────────────────────────────── + +fn lemonade_responder(path: &str) -> String { + match path { + "/api/v1/health" => http( + 200, + "application/json", + r#"{"status":"ok","model_loaded":"Mock-Lemonade-1B"}"#, + ), + "/api/v1/stats" => http(200, "application/json", r#"{"tokens_per_second":33.5}"#), + _ => http(404, "text/plain", ""), + } +} + +/// A reachable Lemonade endpoint is announced once with its health-reported +/// model, scraped for its rate, and announced gone when the endpoint stops +/// answering, after which it leaves the snapshots. +#[tokio::test] +async fn lemonade_endpoint_is_announced_scraped_and_announced_gone() { + let server = spawn_server(lemonade_responder).await; + let id = format!("lemonade-127.0.0.1-{}", server.port); + let (mut rx, _runner) = spawn_runner(lemonade_opts(server.port)); + + let announced = next_event(&mut rx, |ev| match ev { + Event::InstanceDiscovered(inst) if inst.container_id == id => Some(inst.clone()), + _ => None, + }) + .await; + assert_eq!(announced.model_name, "Mock-Lemonade-1B"); + assert_eq!(announced.status, InstanceStatus::Ready); + assert_eq!(announced.port, Some(server.port)); + + // Scraped: the stats endpoint's rate reaches the snapshot. + snapshot_where(&mut rx, |snap| { + instance(snap, &id) + .and_then(|i| i.gen_tps) + .filter(|tps| (tps - 33.5).abs() < 1e-6) + .map(|_| ()) + }) + .await; + + // Rediscovery does not announce it again. + let mut snapshots = 0usize; + while snapshots < 6 { + match next_event(&mut rx, |ev| Some(ev.clone())).await { + Event::InstanceDiscovered(inst) if inst.container_id == id => { + panic!("a rediscovered Lemonade endpoint was announced a second time") + } + Event::Snapshot(_) => snapshots += 1, + _ => {} + } + } + + server.down.store(true, Ordering::Relaxed); + next_event(&mut rx, |ev| match ev { + Event::InstanceGone { container_id } if *container_id == id => Some(()), + _ => None, + }) + .await; + snapshot_where(&mut rx, |snap| instance(snap, &id).is_none().then_some(())).await; +} From a681a9a6a69dc2101cde8c02711886a592b607ab Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 4 Nov 2026 09:34:21 +0000 Subject: [PATCH 2/8] ROCMAI-852: reconcile every discovery source through one helper and bundle run_loop's state run_loop carried three near-duplicate blocks that upserted rediscovered services, announced new ones and retired vanished ones for Docker, Lemonade and the managed-service registry, and passed the same three telemetry maps to each. The Docker and registry sources now go through one reconcile_set and Lemonade through reconcile_lemonade, both built on the same upsert, announce and retire steps, with the log wording chosen by a DiscoverySource value. The loop's telemetry maps and per-source known sets move into TelemetryState and DiscoveryState, which shortens run_loop by about 160 lines. Behaviour is unchanged: the same events are broadcast in the same order with the same log fields, an unchanged endpoint keeps its live fields, a moved port resets them, and the tick behaviour stays MissedTickBehavior::Skip. The test clock stays where it is because the E2E scenarios drive it through a runtime option on the real daemon, so it cannot be test-only code. Unit tests for the shared helper are added here; they also cover the Docker source, which needs a container daemon to run end to end. Signed-off-by: Jussi Elo --- crates/rocm-dash-daemon/src/runner.rs | 682 ++++++++++++++++++-------- 1 file changed, 474 insertions(+), 208 deletions(-) diff --git a/crates/rocm-dash-daemon/src/runner.rs b/crates/rocm-dash-daemon/src/runner.rs index aeb554b940..e0434df391 100644 --- a/crates/rocm-dash-daemon/src/runner.rs +++ b/crates/rocm-dash-daemon/src/runner.rs @@ -315,7 +315,7 @@ pub async fn run_loop( // Opt-in Lemonade discovery (probe-based; off unless a Lemonade endpoint is // configured). Distinct from Docker/vLLM discovery — a local server, not a - // container — so it is tracked separately from `known_instances`. + // container — so it is tracked separately from the Docker known set. let lemonade = if opts.enable_lemonade { info!( host = %opts.lemonade_host, @@ -330,9 +330,6 @@ pub async fn run_loop( } else { None }; - // The id of the currently-live Lemonade instance, if any. - let mut lemonade_id: Option = None; - let mut bench = opts.bench_csv.as_ref().map(|p| { info!(path = %p.display(), "tailing benchmark CSV"); CsvBenchTailer::new(p.clone()) @@ -349,20 +346,8 @@ pub async fn run_loop( } else { None }; - let mut known_instances: HashSet = HashSet::new(); - // Managed-service registry: ids surfaced from the rocm-cli - // `serve` registry, and the subset whose engine is NOT vLLM (excluded from - // the vLLM Prometheus scrape so they aren't mis-parsed). - let mut known_services: HashSet = HashSet::new(); - let mut managed_non_vllm: HashSet = HashSet::new(); - // Per-instance generation-throughput tracker. Keyed by container_id. - // Constructed lazily from opts.instance_tick; survives scrape failures so - // the last-observed rate is held for the validity window before clearing. - let mut gen_trackers: HashMap = HashMap::new(); - // Previous TTFT / TPOT histogram readings (sum_s, count, at) per instance, - // for the windowed avg-ms derivation (mirrors `prev_gen_tokens`). - let mut prev_ttft: HashMap)> = HashMap::new(); - let mut prev_tpot: HashMap)> = HashMap::new(); + let mut discovery = DiscoveryState::default(); + let mut telemetry = TelemetryState::default(); // Per-container VRAM (MB) from the last amd-smi `process` scrape. Refreshed // on the instance cadence and reused every tick so the attributed value is // stable between scrapes (mirrors how GPU power drives tokens_per_watt). @@ -513,76 +498,29 @@ pub async fn run_loop( last_sysinfo_refresh = tick_count; } - // Service discovery — every DISCOVERY_TICKS ticks, diff vs known set, - // emit Discovered/Gone events, and update reducer state. + // Service discovery — every DISCOVERY_TICKS ticks, diff each source vs + // its known set, emit Discovered/Gone events, and update reducer state. + let discovery_due = tick_count == 1 || tick_count.is_multiple_of(discovery_ticks); + let mut reconcile = Reconcile { + state: &mut runner.state, + telemetry: &mut telemetry, + tx: &tx, + persist: persist.as_ref(), + }; if let Some(d) = docker.as_ref() - && (tick_count == 1 || tick_count.is_multiple_of(discovery_ticks)) + && discovery_due { match d.discover_async().await { Ok(svcs) => { let seen: HashSet = svcs.iter().map(|s| s.container_id.clone()).collect(); - for svc in &svcs { - let existing = runner.state.instances.get(&svc.container_id).cloned(); - // Endpoint change: same container-id, new port → new logical service. - // Invalidate the rate baseline and latency windows so the new - // scrape address starts fresh. - let same_endpoint = existing.as_ref().is_none_or(|ex| ex.port == svc.port); - if !same_endpoint { - invalidate_on_endpoint_change( - &svc.container_id, - &mut gen_trackers, - &mut prev_ttft, - &mut prev_tpot, - ); - } - let inst = merge_discovery_refresh( - svc, - if same_endpoint { - existing.as_ref() - } else { - None - }, - ); - runner - .state - .apply(StateEvent::InstanceUpserted(inst.clone())); - if !known_instances.contains(&svc.container_id) { - info!( - id = %svc.container_id, - name = %svc.container_name, - model = %svc.model_name, - "instance discovered" - ); - // InstanceDiscovered is a discovery-metadata event; broadcast - // a clean instance (no prior collector-owned telemetry). - broadcast_and_persist( - &tx, - persist.as_ref(), - Event::InstanceDiscovered(instance_from_discovered(svc)), - ); - } - } - for gone in known_instances.difference(&seen) { - info!(id = %gone, "instance gone"); - purge_telemetry_state( - gone, - &mut gen_trackers, - &mut prev_ttft, - &mut prev_tpot, - ); - runner - .state - .apply(StateEvent::InstanceRemoved(gone.clone())); - broadcast_and_persist( - &tx, - persist.as_ref(), - Event::InstanceGone { - container_id: gone.clone(), - }, - ); - } - known_instances = seen; + reconcile.reconcile_set( + DiscoverySource::Docker, + &svcs, + &discovery.known_instances, + &seen, + ); + discovery.known_instances = seen; docker_warn = None; } Err(e) => docker_warn = Some(format!("docker discover: {e}")), @@ -592,70 +530,11 @@ pub async fn run_loop( // Lemonade discovery — probe the endpoint on the discovery cadence; add a // Lemonade Instance when reachable, emit Gone when it disappears, and stay // a clean no-op (no warning/panic) when no endpoint is configured. - // - // The Lemonade `container_id` encodes the host and port - // (`:`), so a port or host change produces a new id string - // and is treated as a full identity change (old gone, new discovered). if let Some(l) = lemonade.as_ref() - && (tick_count == 1 || tick_count.is_multiple_of(discovery_ticks)) + && discovery_due { - match l.discover().await { - Some(svc) => { - let is_new_id = lemonade_id.as_deref() != Some(svc.container_id.as_str()); - if is_new_id { - // New identity: purge all telemetry state for the old id so - // stale rate baseline, TTFT, and TPOT do not bleed across. - if let Some(old_id) = lemonade_id.as_deref() { - purge_telemetry_state( - old_id, - &mut gen_trackers, - &mut prev_ttft, - &mut prev_tpot, - ); - } - } - let existing = runner.state.instances.get(&svc.container_id).cloned(); - let inst = merge_discovery_refresh( - &svc, - if is_new_id { None } else { existing.as_ref() }, - ); - runner - .state - .apply(StateEvent::InstanceUpserted(inst.clone())); - if is_new_id { - info!( - id = %svc.container_id, - model = %svc.model_name, - "lemonade instance discovered" - ); - // InstanceDiscovered is a discovery-metadata event; broadcast - // a clean instance (no prior collector-owned telemetry). - broadcast_and_persist( - &tx, - persist.as_ref(), - Event::InstanceDiscovered(instance_from_discovered(&svc)), - ); - lemonade_id = Some(svc.container_id.clone()); - } - } - None => { - if let Some(id) = lemonade_id.take() { - info!(id = %id, "lemonade instance gone"); - purge_telemetry_state( - &id, - &mut gen_trackers, - &mut prev_ttft, - &mut prev_tpot, - ); - runner.state.apply(StateEvent::InstanceRemoved(id.clone())); - broadcast_and_persist( - &tx, - persist.as_ref(), - Event::InstanceGone { container_id: id }, - ); - } - } - } + let probed = l.discover().await; + reconcile.reconcile_lemonade(probed, &mut discovery.lemonade_id); } // Managed-service registry discovery — read the rocm-cli @@ -664,63 +543,18 @@ pub async fn run_loop( // scrape below skips them. A model served via `rocm serve` thus appears // in the dashboard and is scraped for `gen_tps` without Docker. if let Some(services_dir) = opts.services_dir.as_ref() - && (tick_count == 1 || tick_count.is_multiple_of(discovery_ticks)) + && discovery_due { let records = crate::registry::load_service_records(services_dir); let disc = crate::registry::discover_managed_services(&records); - managed_non_vllm = disc.non_vllm; - for svc in disc.svcs { - let id = svc.container_id.clone(); - let is_new = !known_services.contains(&id); - let existing = runner.state.instances.get(&id).cloned(); - // Endpoint change → new logical service; reset rate baseline and - // latency windows so the new port starts fresh. - let same_endpoint = existing.as_ref().is_none_or(|ex| ex.port == svc.port); - if !same_endpoint { - invalidate_on_endpoint_change( - &id, - &mut gen_trackers, - &mut prev_ttft, - &mut prev_tpot, - ); - } - let inst = merge_discovery_refresh( - &svc, - if same_endpoint { - existing.as_ref() - } else { - None - }, - ); - runner - .state - .apply(StateEvent::InstanceUpserted(inst.clone())); - if is_new { - info!(id = %id, "managed service discovered"); - // InstanceDiscovered is a discovery-metadata event; broadcast - // a clean instance (no prior collector-owned telemetry). - broadcast_and_persist( - &tx, - persist.as_ref(), - Event::InstanceDiscovered(instance_from_discovered(&svc)), - ); - } - } - for gone in known_services.difference(&disc.seen) { - info!(id = %gone, "managed service gone"); - purge_telemetry_state(gone, &mut gen_trackers, &mut prev_ttft, &mut prev_tpot); - runner - .state - .apply(StateEvent::InstanceRemoved(gone.clone())); - broadcast_and_persist( - &tx, - persist.as_ref(), - Event::InstanceGone { - container_id: gone.clone(), - }, - ); - } - known_services = disc.seen; + discovery.managed_non_vllm = disc.non_vllm; + reconcile.reconcile_set( + DiscoverySource::Managed, + &disc.svcs, + &discovery.known_services, + &disc.seen, + ); + discovery.known_services = disc.seen; } if tick_count.is_multiple_of(instance_ticks) { scrape_warns.clear(); @@ -738,8 +572,8 @@ pub async fn run_loop( .state .instances .values() - .filter(|i| Some(i.container_id.as_str()) != lemonade_id.as_deref()) - .filter(|i| !managed_non_vllm.contains(&i.container_id)) + .filter(|i| Some(i.container_id.as_str()) != discovery.lemonade_id.as_deref()) + .filter(|i| !discovery.managed_non_vllm.contains(&i.container_id)) .filter_map(|i| i.port.map(|p| (i.container_id.clone(), p))) .collect(); let prom = prom.clone(); @@ -764,7 +598,8 @@ pub async fn run_loop( // cumulative counter when present; omitted counter leaves // the tracker (and its held value) untouched. if let Some(cur) = sample.gen_tokens_total { - gen_trackers + telemetry + .gen_trackers .entry(id.clone()) .or_insert_with(|| { GenerationObservationTracker::new(opts.instance_tick) @@ -777,14 +612,14 @@ pub async fn run_loop( // Windowed avg latency (ms) from the cumulative TTFT/TPOT // histograms; cumulative-average fallback on first scrape. let ttft_ms = avg_ms_from_histogram( - &mut prev_ttft, + &mut telemetry.prev_ttft, &id, sample.ttft_sum_s, sample.ttft_count, cycle_at, ); let tpot_ms = avg_ms_from_histogram( - &mut prev_tpot, + &mut telemetry.prev_tpot, &id, sample.tpot_sum_s, sample.tpot_count, @@ -821,8 +656,8 @@ pub async fn run_loop( // Clear TTFT/TPOT baselines so recovery re-baselines from // the next successful scrape (latency clearing semantics // are unchanged; freshness generalization is out of scope). - prev_ttft.remove(&id); - prev_tpot.remove(&id); + telemetry.prev_ttft.remove(&id); + telemetry.prev_tpot.remove(&id); if let Some(mut inst) = runner.state.instances.get(&id).cloned() && (inst.ttft_ms.is_some() || inst.tpot_ms.is_some()) { @@ -844,7 +679,7 @@ pub async fn run_loop( // Lemonade per-instance scrape — reports an instantaneous rate directly // (`gen_tps`), so no counter-differencing. A scrape failure leaves the // last-known fields and warns; it never panics. - if let (Some(l), Some(id)) = (lemonade.as_ref(), lemonade_id.clone()) + if let (Some(l), Some(id)) = (lemonade.as_ref(), discovery.lemonade_id.clone()) && tick_count.is_multiple_of(instance_ticks) { match l.fetch_stats().await { @@ -854,7 +689,8 @@ pub async fn run_loop( // erase or refresh the tracker; the held value stays valid until // the validity window expires. if let Some(rate) = sample.gen_tps { - gen_trackers + telemetry + .gen_trackers .entry(id.clone()) .or_insert_with(|| { GenerationObservationTracker::new(opts.instance_tick) @@ -916,7 +752,8 @@ pub async fn run_loop( // metadata) before computing tokens_per_watt so the efficiency field // always derives from the current cycle's valid/held/expired rate. for inst in &mut instances { - let (gen_tps, meta) = gen_trackers + let (gen_tps, meta) = telemetry + .gen_trackers .get_mut(&inst.container_id) .map_or((None, None), |t| t.snapshot(cycle_at)); inst.gen_tps = gen_tps; @@ -1091,6 +928,203 @@ pub(crate) fn merge_discovery_refresh( inst } +/// Per-instance telemetry the loop carries between scrapes, keyed by container id. +#[derive(Default)] +struct TelemetryState { + /// Generation-throughput trackers. Constructed lazily from + /// `opts.instance_tick`; they survive scrape failures so the last-observed + /// rate is held for the validity window before clearing. + gen_trackers: HashMap, + /// Previous TTFT / TPOT histogram readings (sum_s, count, at) per instance, + /// for the windowed avg-ms derivation. + prev_ttft: HashMap)>, + prev_tpot: HashMap)>, +} + +impl TelemetryState { + fn purge(&mut self, id: &str) { + purge_telemetry_state( + id, + &mut self.gen_trackers, + &mut self.prev_ttft, + &mut self.prev_tpot, + ); + } + + fn invalidate_on_endpoint_change(&mut self, id: &str) { + invalidate_on_endpoint_change( + id, + &mut self.gen_trackers, + &mut self.prev_ttft, + &mut self.prev_tpot, + ); + } +} + +/// What each discovery source currently has live, so a pass can tell a new +/// service from a known one and a known one from one that has gone. +#[derive(Default)] +struct DiscoveryState { + /// Container ids the last Docker pass saw. + known_instances: HashSet, + /// Ids the last managed-service registry pass surfaced. + known_services: HashSet, + /// The subset of the registry ids whose engine is NOT vLLM, excluded from + /// the vLLM Prometheus scrape so they aren't mis-parsed. + managed_non_vllm: HashSet, + /// The id of the currently-live Lemonade instance, if any. + lemonade_id: Option, +} + +/// Which discovery source a service came from. It only selects the log wording; +/// the reconcile itself treats every source the same way. +#[derive(Clone, Copy)] +enum DiscoverySource { + Docker, + Lemonade, + Managed, +} + +impl DiscoverySource { + fn log_discovered(self, svc: &DiscoveredService) { + match self { + Self::Docker => info!( + id = %svc.container_id, + name = %svc.container_name, + model = %svc.model_name, + "instance discovered" + ), + Self::Lemonade => info!( + id = %svc.container_id, + model = %svc.model_name, + "lemonade instance discovered" + ), + Self::Managed => info!(id = %svc.container_id, "managed service discovered"), + } + } + + fn log_gone(self, id: &str) { + match self { + Self::Docker => info!(id = %id, "instance gone"), + Self::Lemonade => info!(id = %id, "lemonade instance gone"), + Self::Managed => info!(id = %id, "managed service gone"), + } + } +} + +/// The loop state a discovery pass changes, and the stream it announces on. +struct Reconcile<'a> { + state: &'a mut State, + telemetry: &'a mut TelemetryState, + tx: &'a broadcast::Sender, + persist: Option<&'a Arc>>, +} + +impl Reconcile<'_> { + /// Upsert a (re)discovered service. Collector-owned fields carry over from + /// the instance already in state unless `carry_live` is false, which is how + /// a changed identity starts from a clean baseline. + fn upsert(&mut self, svc: &DiscoveredService, carry_live: bool) { + let existing = self.state.instances.get(&svc.container_id).cloned(); + let inst = merge_discovery_refresh(svc, if carry_live { existing.as_ref() } else { None }); + self.state.apply(StateEvent::InstanceUpserted(inst)); + } + + /// Announce a newly discovered service. InstanceDiscovered is a + /// discovery-metadata event: it carries a clean instance, with no prior + /// collector-owned telemetry. + fn announce(&self, source: DiscoverySource, svc: &DiscoveredService) { + source.log_discovered(svc); + broadcast_and_persist( + self.tx, + self.persist, + Event::InstanceDiscovered(instance_from_discovered(svc)), + ); + } + + /// Drop a service that has gone: purge its telemetry, remove it from state + /// and announce it. + fn retire(&mut self, source: DiscoverySource, id: &str) { + source.log_gone(id); + self.telemetry.purge(id); + self.state + .apply(StateEvent::InstanceRemoved(id.to_string())); + broadcast_and_persist( + self.tx, + self.persist, + Event::InstanceGone { + container_id: id.to_string(), + }, + ); + } + + /// Reconcile a source that reports its whole set each pass (Docker, the + /// managed-service registry). `known` is what the previous pass saw and + /// `seen` what this one saw; the caller stores `seen` as the next `known`. + /// + /// A service whose port changed under the same id is a new logical service + /// at a new scrape address: its rate baseline and latency windows are reset + /// and no live fields carry over. + fn reconcile_set( + &mut self, + source: DiscoverySource, + svcs: &[DiscoveredService], + known: &HashSet, + seen: &HashSet, + ) { + for svc in svcs { + let same_endpoint = self + .state + .instances + .get(&svc.container_id) + .is_none_or(|ex| ex.port == svc.port); + if !same_endpoint { + self.telemetry + .invalidate_on_endpoint_change(&svc.container_id); + } + self.upsert(svc, same_endpoint); + if !known.contains(&svc.container_id) { + self.announce(source, svc); + } + } + for gone in known.difference(seen) { + self.retire(source, gone); + } + } + + /// Reconcile the probe-based Lemonade source, which has at most one service. + /// + /// Its `container_id` encodes the host and port, so a port or host change + /// yields a new id: the old id's telemetry is purged so a stale rate + /// baseline, TTFT and TPOT do not bleed across, and the new id is announced. + /// The old instance is not retired here. `None` means the endpoint did not + /// answer, which retires the current one. + fn reconcile_lemonade( + &mut self, + probed: Option, + current: &mut Option, + ) { + match probed { + Some(svc) => { + let is_new_id = current.as_deref() != Some(svc.container_id.as_str()); + if is_new_id && let Some(old_id) = current.as_deref() { + self.telemetry.purge(old_id); + } + self.upsert(&svc, !is_new_id); + if is_new_id { + self.announce(DiscoverySource::Lemonade, &svc); + *current = Some(svc.container_id.clone()); + } + } + None => { + if let Some(id) = current.take() { + self.retire(DiscoverySource::Lemonade, &id); + } + } + } + } +} + /// Set `vram_used_mb`/`vram_total_mb` on each instance from the per-process /// attribution map, falling back to device-summed VRAM over the instance's /// GPUs when its container has no per-process entry. Pure — the runner does the @@ -1555,4 +1589,236 @@ mod tests { "tpot baseline must be cleared on endpoint change" ); } + + // ── Discovery reconcile ───────────────────────────────────────────────── + // + // The Docker source cannot be driven end to end without a container + // daemon, and `reconcile_set` is the code both it and the managed-service + // registry run, so these tests exercise it directly. The registry and + // Lemonade sources are also covered end to end in + // `tests/discovery_reconcile.rs`. + + fn discovered(id: &str, port: u16) -> DiscoveredService { + DiscoveredService { + container_id: id.into(), + container_name: format!("name-{id}"), + model_name: "model".into(), + port: Some(port), + ..Default::default() + } + } + + fn id_set(ids: &[&str]) -> HashSet { + ids.iter().map(|s| (*s).to_string()).collect() + } + + /// The reducer state, telemetry and event stream a reconcile pass works on. + struct Pass { + state: State, + telemetry: TelemetryState, + tx: broadcast::Sender, + rx: broadcast::Receiver, + } + + impl Pass { + fn new() -> Self { + let (tx, rx) = broadcast::channel(64); + Self { + state: State::default(), + telemetry: TelemetryState::default(), + tx, + rx, + } + } + + fn reconcile(&mut self) -> Reconcile<'_> { + Reconcile { + state: &mut self.state, + telemetry: &mut self.telemetry, + tx: &self.tx, + persist: None, + } + } + + /// The announcements broadcast since the last call, as `discovered ` + /// and `gone ` strings. + fn announced(&mut self) -> Vec { + let mut out = Vec::new(); + while let Ok(ev) = self.rx.try_recv() { + match ev { + Event::InstanceDiscovered(i) => { + out.push(format!("discovered {}", i.container_id)); + } + Event::InstanceGone { container_id } => { + out.push(format!("gone {container_id}")); + } + _ => {} + } + } + out + } + + /// Give `id` a live KV reading and a latency baseline, as a scrape would. + fn scrape(&mut self, id: &str) { + let mut inst = self.state.instances.get(id).cloned().expect("instance"); + inst.kv_cache_usage_pct = Some(25.0); + self.state.apply(StateEvent::InstanceUpserted(inst)); + self.telemetry + .prev_ttft + .insert(id.into(), (1.0, 10.0, at(10))); + self.telemetry + .prev_tpot + .insert(id.into(), (0.5, 8.0, at(10))); + } + + fn kv(&self, id: &str) -> Option { + self.state + .instances + .get(id) + .and_then(|i| i.kv_cache_usage_pct) + } + } + + #[test] + fn reconcile_set_announces_a_new_service_once() { + let mut pass = Pass::new(); + let svcs = [discovered("a", 8000)]; + + pass.reconcile().reconcile_set( + DiscoverySource::Docker, + &svcs, + &id_set(&[]), + &id_set(&["a"]), + ); + assert_eq!(pass.announced(), ["discovered a"]); + assert!(pass.state.instances.contains_key("a")); + + pass.reconcile().reconcile_set( + DiscoverySource::Docker, + &svcs, + &id_set(&["a"]), + &id_set(&["a"]), + ); + assert!( + pass.announced().is_empty(), + "a known service must not be announced again" + ); + } + + #[test] + fn reconcile_set_keeps_live_fields_while_the_endpoint_is_unchanged() { + let mut pass = Pass::new(); + let svcs = [discovered("a", 8000)]; + pass.reconcile().reconcile_set( + DiscoverySource::Docker, + &svcs, + &id_set(&[]), + &id_set(&["a"]), + ); + pass.scrape("a"); + + pass.reconcile().reconcile_set( + DiscoverySource::Docker, + &svcs, + &id_set(&["a"]), + &id_set(&["a"]), + ); + + assert_eq!(pass.kv("a"), Some(25.0)); + assert!(pass.telemetry.prev_ttft.contains_key("a")); + assert!(pass.telemetry.prev_tpot.contains_key("a")); + } + + #[test] + fn reconcile_set_starts_clean_when_the_port_moves_under_the_same_id() { + let mut pass = Pass::new(); + pass.reconcile().reconcile_set( + DiscoverySource::Docker, + &[discovered("a", 8000)], + &id_set(&[]), + &id_set(&["a"]), + ); + pass.scrape("a"); + pass.announced(); + + pass.reconcile().reconcile_set( + DiscoverySource::Docker, + &[discovered("a", 9000)], + &id_set(&["a"]), + &id_set(&["a"]), + ); + + assert_eq!(pass.state.instances["a"].port, Some(9000)); + assert_eq!(pass.kv("a"), None, "the old endpoint's reading leaked"); + assert!(!pass.telemetry.prev_ttft.contains_key("a")); + assert!(!pass.telemetry.prev_tpot.contains_key("a")); + assert!( + pass.announced().is_empty(), + "a port change is not a new or a gone service" + ); + } + + #[test] + fn reconcile_set_retires_a_vanished_service_and_keeps_the_others() { + let mut pass = Pass::new(); + let svcs = [discovered("a", 8000), discovered("b", 8001)]; + pass.reconcile().reconcile_set( + DiscoverySource::Managed, + &svcs, + &id_set(&[]), + &id_set(&["a", "b"]), + ); + pass.scrape("a"); + pass.scrape("b"); + pass.announced(); + + pass.reconcile().reconcile_set( + DiscoverySource::Managed, + &svcs[..1], + &id_set(&["a", "b"]), + &id_set(&["a"]), + ); + + assert_eq!(pass.announced(), ["gone b"]); + assert!(!pass.state.instances.contains_key("b")); + assert!(!pass.telemetry.prev_ttft.contains_key("b")); + assert!(pass.state.instances.contains_key("a")); + assert!(pass.telemetry.prev_ttft.contains_key("a")); + } + + #[test] + fn reconcile_lemonade_follows_one_endpoint_through_its_lifecycle() { + let mut pass = Pass::new(); + let mut current: Option = None; + + pass.reconcile() + .reconcile_lemonade(Some(discovered("l1", 13305)), &mut current); + assert_eq!(pass.announced(), ["discovered l1"]); + assert_eq!(current.as_deref(), Some("l1")); + + pass.scrape("l1"); + pass.reconcile() + .reconcile_lemonade(Some(discovered("l1", 13305)), &mut current); + assert!( + pass.announced().is_empty(), + "same id must not be re-announced" + ); + assert_eq!(pass.kv("l1"), Some(25.0), "live fields carry over"); + + // A new id (the host or port changed) purges the old id's telemetry and + // announces the new one without carrying anything across. + pass.reconcile() + .reconcile_lemonade(Some(discovered("l2", 13306)), &mut current); + assert_eq!(pass.announced(), ["discovered l2"]); + assert_eq!(current.as_deref(), Some("l2")); + assert!(!pass.telemetry.prev_ttft.contains_key("l1")); + + pass.reconcile().reconcile_lemonade(None, &mut current); + assert_eq!(pass.announced(), ["gone l2"]); + assert_eq!(current, None); + assert!(!pass.state.instances.contains_key("l2")); + + pass.reconcile().reconcile_lemonade(None, &mut current); + assert!(pass.announced().is_empty(), "nothing left to retire"); + } } From e7addca0abefa1fa14662e5a6cabcd71061811aa Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 4 Nov 2026 09:36:22 +0000 Subject: [PATCH 3/8] ROCMAI-852: give ticks_per its own doc comment The line describing ticks_per sat on top of broadcast_and_persist's doc, so ticks_per had no doc and broadcast_and_persist's started with an unrelated sentence. Signed-off-by: Jussi Elo --- crates/rocm-dash-daemon/src/runner.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/rocm-dash-daemon/src/runner.rs b/crates/rocm-dash-daemon/src/runner.rs index e0434df391..62e317190e 100644 --- a/crates/rocm-dash-daemon/src/runner.rs +++ b/crates/rocm-dash-daemon/src/runner.rs @@ -1148,7 +1148,6 @@ fn enrich_instance_vram( } } -/// How many `tick`s fit into `period`, rounded to the nearest, minimum 1. /// Broadcast `ev` to subscribers and, if a session writer is wired, append /// the same event to disk for `--replay`. Persistence is best-effort — a /// write failure logs at warn level but does not interrupt the loop. @@ -1166,6 +1165,7 @@ fn broadcast_and_persist( let _ = tx.send(ev); } +/// How many `tick`s fit into `period`, rounded to the nearest, minimum 1. fn ticks_per(period: Duration, tick: Duration) -> u64 { let n = (period.as_secs_f64() / tick.as_secs_f64()).round() as i64; n.max(1) as u64 From 675d6e3d6a3e17c8a73a6e0b12a9603111d073ee Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 4 Nov 2026 10:55:33 +0000 Subject: [PATCH 4/8] ROCMAI-852: bound the announced-once test with one deadline for the whole observation managed_service_is_announced_once_across_rediscovery looped on next_event until it had counted eight snapshots containing the service, but next_event starts a fresh deadline on every call. If managed discovery stopped adding the service, the runner would keep broadcasting snapshots without it, every call would succeed, the count would stay at zero and the test would spin instead of failing. The observation now runs under a single deadline and, when it expires, says what it was waiting for and how many matching snapshots and announcements it had seen. Signed-off-by: Jussi Elo --- .../tests/discovery_reconcile.rs | 68 ++++++++++++++++--- 1 file changed, 57 insertions(+), 11 deletions(-) diff --git a/crates/rocm-dash-daemon/tests/discovery_reconcile.rs b/crates/rocm-dash-daemon/tests/discovery_reconcile.rs index f2276f0886..e4ef0eb363 100644 --- a/crates/rocm-dash-daemon/tests/discovery_reconcile.rs +++ b/crates/rocm-dash-daemon/tests/discovery_reconcile.rs @@ -25,7 +25,7 @@ use rocm_dash_core::protocol::Event; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; use tokio::sync::broadcast; -use tokio::time::timeout; +use tokio::time::{timeout, timeout_at}; use rocm_dash_daemon::bench_ring::BenchRing; use rocm_dash_daemon::runner::{self, RunnerOptions}; @@ -223,6 +223,40 @@ async fn snapshot_where( } } +/// Feed events to `step` until it returns true, under ONE deadline for the whole +/// observation. A loop that calls [`next_event`] repeatedly gets a fresh +/// deadline per call, so a runner that keeps broadcasting events the loop does +/// not count would spin forever; this fails at [`DEADLINE`] instead, naming what +/// was being waited for and, through `progress`, how far it got. +async fn observe( + rx: &mut broadcast::Receiver, + waiting_for: &str, + state: &mut S, + mut step: impl FnMut(&mut S, &Event) -> bool, + progress: impl Fn(&S) -> String, +) { + let deadline = tokio::time::Instant::now() + DEADLINE; + loop { + let Ok(received) = timeout_at(deadline, rx.recv()).await else { + panic!( + "timed out after {DEADLINE:?} waiting for {waiting_for}; seen so far: {}", + progress(state) + ); + }; + match received { + Ok(ev) => { + if step(state, &ev) { + return; + } + } + Err(broadcast::error::RecvError::Lagged(_)) => {} + Err(broadcast::error::RecvError::Closed) => { + panic!("broadcast channel closed while waiting for {waiting_for}") + } + } + } +} + // ── Managed-service registry ──────────────────────────────────────────────── /// Rediscovery runs every tick but the scrape only every second one, so on the @@ -269,17 +303,29 @@ async fn managed_service_is_announced_once_across_rediscovery() { write_record(&services_dir, server.port); let (mut rx, _runner) = spawn_runner(managed_opts(services_dir)); - let mut announced = 0usize; - let mut snapshots_with_instance = 0usize; - while snapshots_with_instance < 8 { - match next_event(&mut rx, |ev| Some(ev.clone())).await { - Event::InstanceDiscovered(inst) if inst.container_id == SVC_ID => announced += 1, - Event::Snapshot(snap) if instance(&snap, SVC_ID).is_some() => { - snapshots_with_instance += 1; + // (announcements of the service, snapshots that contain it) + let mut seen = (0usize, 0usize); + observe( + &mut rx, + "8 snapshots containing the managed service", + &mut seen, + |seen, ev| { + match ev { + Event::InstanceDiscovered(inst) if inst.container_id == SVC_ID => seen.0 += 1, + Event::Snapshot(snap) if instance(snap, SVC_ID).is_some() => seen.1 += 1, + _ => {} } - _ => {} - } - } + seen.1 >= 8 + }, + |seen| { + format!( + "{} snapshots containing it, {} announcements of it", + seen.1, seen.0 + ) + }, + ) + .await; + let (announced, _) = seen; assert_eq!( announced, 1, "the service must be announced exactly once, not once per discovery tick" From 93379e536386d534a647ec005553903ae7c67270 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Wed, 4 Nov 2026 18:56:00 +0000 Subject: [PATCH 5/8] test(dash-daemon): bound every wait and pin the clean-change and retire behaviour Review follow-ups on the discovery reconcile tests, with no change to the runner's behaviour. The Lemonade test's rediscovery watch looped on next_event, which gives every call a fresh 20 second deadline, so a runner that kept broadcasting snapshots without ever finishing the count could not fail it in bounded time. It now runs through observe, so one deadline covers the whole watch and the failure names how many snapshots were seen. managed_endpoint_change_starts_the_instance_clean documented that the change must not be announced as a new service but only checked that the scraped fields were cleared. It now watches the stream from the moment the record changes until the instance has been on its new port for a few snapshots and asserts no InstanceDiscovered or InstanceGone for the service appears. The registry record is written through a temp file and renamed into place so a discovery tick can never read half a record, which would look like the service vanishing and coming back. reconcile_lemonade_follows_one_endpoint_through_its_lifecycle pinned that the superseded id's telemetry is purged but not that the instance itself stays in state. It now asserts that, with a comment saying a later change to retire it is deliberate. Also corrects the vllm_metrics doc, which described a growing counter although the body is fixed. Signed-off-by: Jussi Elo --- crates/rocm-dash-daemon/src/runner.rs | 5 + .../tests/discovery_reconcile.rs | 106 +++++++++++++++--- 2 files changed, 94 insertions(+), 17 deletions(-) diff --git a/crates/rocm-dash-daemon/src/runner.rs b/crates/rocm-dash-daemon/src/runner.rs index 62e317190e..37cb21c102 100644 --- a/crates/rocm-dash-daemon/src/runner.rs +++ b/crates/rocm-dash-daemon/src/runner.rs @@ -1812,6 +1812,11 @@ mod tests { assert_eq!(pass.announced(), ["discovered l2"]); assert_eq!(current.as_deref(), Some("l2")); assert!(!pass.telemetry.prev_ttft.contains_key("l1")); + // Pins current behaviour, not a goal: the superseded instance stays in + // state (and so in every snapshot) because only its telemetry is purged. + // Retiring it later is a deliberate behaviour change, and this assertion + // is the one to update when that happens. + assert!(pass.state.instances.contains_key("l1")); pass.reconcile().reconcile_lemonade(None, &mut current); assert_eq!(pass.announced(), ["gone l2"]); diff --git a/crates/rocm-dash-daemon/tests/discovery_reconcile.rs b/crates/rocm-dash-daemon/tests/discovery_reconcile.rs index e4ef0eb363..5a1a81c0b8 100644 --- a/crates/rocm-dash-daemon/tests/discovery_reconcile.rs +++ b/crates/rocm-dash-daemon/tests/discovery_reconcile.rs @@ -91,8 +91,8 @@ fn http(status: u16, content_type: &str, body: &str) -> String { ) } -/// A vLLM `/metrics` body with a growing token counter, one running request and -/// a 25% KV cache. +/// A vLLM `/metrics` body with a fixed token counter, one running request and a +/// 25% KV cache. fn vllm_metrics() -> String { http( 200, @@ -149,12 +149,20 @@ fn lemonade_opts(port: u16) -> RunnerOptions { } } +/// Write the service's registry record. The record is written to a sibling temp +/// file and renamed into place, so the discovery tick can never read half a +/// record: a torn read would look like the service vanishing and coming back, +/// which is exactly what the tests below assert does not happen. fn write_record(services_dir: &std::path::Path, port: u16) { std::fs::create_dir_all(services_dir).unwrap(); let json = format!( r#"{{"service_id":"{SVC_ID}","engine":"vllm","model_ref":"Reconcile/1B","canonical_model_id":"Reconcile/1B","host":"127.0.0.1","port":{port},"status":"ready","created_at_unix_ms":0}}"# ); - std::fs::write(services_dir.join(format!("{SVC_ID}.json")), json).unwrap(); + // The registry only reads `*.json`, so the `.tmp` suffix keeps the + // half-written file invisible to it. + let tmp = services_dir.join(format!("{SVC_ID}.json.tmp")); + std::fs::write(&tmp, json).unwrap(); + std::fs::rename(&tmp, services_dir.join(format!("{SVC_ID}.json"))).unwrap(); } /// Read events until `pick` returns a value, or fail at the deadline. @@ -203,6 +211,10 @@ fn approx(value: Option, expected: f32) -> bool { /// The mock's `0.25` KV-cache fraction, which the collector reports as a percent. const KV_PCT: f32 = 25.0; +/// Snapshots to keep watching after an event of interest, so an announcement +/// that trails it by a tick or two is still seen. +const SETTLE_SNAPSHOTS: usize = 3; + /// Read snapshots until `pick` returns a value. Bounded as a whole, so a /// condition that never holds fails at [`DEADLINE`] instead of looping on fresh /// snapshots forever. @@ -332,6 +344,16 @@ async fn managed_service_is_announced_once_across_rediscovery() { ); } +/// What the stream showed from the moment a service's record changed port. +#[derive(Default)] +struct Change { + /// The instance as first seen on its new port. + moved: Option, + snapshots_after_move: usize, + /// Every discovered/gone announcement for the service in the window. + announcements: Vec<&'static str>, +} + /// The registry's port is the authority: when it changes under the same service /// id, the live fields scraped from the old port must not leak onto the new /// endpoint, and the change must not be announced as a new service. @@ -354,12 +376,52 @@ async fn managed_endpoint_change_starts_the_instance_clean() { .await; write_record(&services_dir, silent.port); - let moved = snapshot_where(&mut rx, |snap| { - instance(snap, SVC_ID) - .filter(|i| i.port == Some(silent.port)) - .cloned() - }) + + // Watch the stream from the moment the record changes until the instance has + // been seen on its new port and has stayed there for a few snapshots, under + // one deadline. Any announcement or retirement of the service in that window + // means the change was treated as a different service rather than the same + // one moving. + let mut change = Change::default(); + observe( + &mut rx, + "the managed service to appear on its new port", + &mut change, + |change, ev| { + match ev { + Event::InstanceDiscovered(inst) if inst.container_id == SVC_ID => { + change.announcements.push("discovered"); + } + Event::InstanceGone { container_id } if container_id == SVC_ID => { + change.announcements.push("gone"); + } + Event::Snapshot(snap) if change.moved.is_none() => { + change.moved = instance(snap, SVC_ID) + .filter(|i| i.port == Some(silent.port)) + .cloned(); + } + Event::Snapshot(_) => change.snapshots_after_move += 1, + _ => {} + } + change.snapshots_after_move >= SETTLE_SNAPSHOTS + }, + |change| { + format!( + "moved={}, {} snapshots after the move, announcements {:?}", + change.moved.is_some(), + change.snapshots_after_move, + change.announcements + ) + }, + ) .await; + + assert!( + change.announcements.is_empty(), + "an endpoint change must not announce the service as discovered or gone: {:?}", + change.announcements + ); + let moved = change.moved.expect("observe returned before the move"); assert_eq!( moved.kv_cache_usage_pct, None, "the old endpoint's kv cache leaked onto the new one" @@ -411,17 +473,27 @@ async fn lemonade_endpoint_is_announced_scraped_and_announced_gone() { }) .await; - // Rediscovery does not announce it again. + // Rediscovery does not announce it again. One deadline covers the whole + // watch, so a runner that stops broadcasting snapshots fails here instead of + // each wait starting its own. let mut snapshots = 0usize; - while snapshots < 6 { - match next_event(&mut rx, |ev| Some(ev.clone())).await { - Event::InstanceDiscovered(inst) if inst.container_id == id => { - panic!("a rediscovered Lemonade endpoint was announced a second time") + observe( + &mut rx, + "6 snapshots with no second announcement of the Lemonade endpoint", + &mut snapshots, + |snapshots, ev| { + match ev { + Event::InstanceDiscovered(inst) if inst.container_id == id => { + panic!("a rediscovered Lemonade endpoint was announced a second time") + } + Event::Snapshot(_) => *snapshots += 1, + _ => {} } - Event::Snapshot(_) => snapshots += 1, - _ => {} - } - } + *snapshots >= 6 + }, + |snapshots| format!("{snapshots} snapshots"), + ) + .await; server.down.store(true, Ordering::Relaxed); next_event(&mut rx, |ev| match ev { From a1a71d4710d33d688a322b8e98485ac1432d6467 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 07:06:53 +0000 Subject: [PATCH 6/8] docs(dash-daemon): make the reconcile comments say what the code and tests do Comment-only. The module doc of the discovery_reconcile tests listed behaviour the file does not drive through the loop (registry retirement, Lemonade id change) and now says which source covers what. The snapshot_where doc claimed a single bound that the nested per-snapshot wait loosens. The endpoint-change test comment said the instance "stayed" on its new port, which the test does not check. In runner.rs, the DiscoverySource doc said every source is reconciled the same way, although Lemonade has its own pass; the reconcile_set doc called a moved port a new service while the code and tests do not announce it; a loop comment named a constant that does not exist; and one test comment claimed a carry-over the test cannot observe. Signed-off-by: Jussi Elo --- crates/rocm-dash-daemon/src/runner.rs | 20 ++++++++----- .../tests/discovery_reconcile.rs | 29 +++++++++++-------- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/crates/rocm-dash-daemon/src/runner.rs b/crates/rocm-dash-daemon/src/runner.rs index 37cb21c102..168a138bde 100644 --- a/crates/rocm-dash-daemon/src/runner.rs +++ b/crates/rocm-dash-daemon/src/runner.rs @@ -498,8 +498,9 @@ pub async fn run_loop( last_sysinfo_refresh = tick_count; } - // Service discovery — every DISCOVERY_TICKS ticks, diff each source vs - // its known set, emit Discovered/Gone events, and update reducer state. + // Service discovery — every `discovery_ticks` ticks, reconcile each + // source against what it had live before, emit Discovered/Gone events, + // and update reducer state. let discovery_due = tick_count == 1 || tick_count.is_multiple_of(discovery_ticks); let mut reconcile = Reconcile { state: &mut runner.state, @@ -976,8 +977,11 @@ struct DiscoveryState { lemonade_id: Option, } -/// Which discovery source a service came from. It only selects the log wording; -/// the reconcile itself treats every source the same way. +/// Which discovery source a service came from. It only selects the log wording +/// in [`Reconcile::announce`] and [`Reconcile::retire`]; those two steps behave +/// the same for every source. How a pass decides what is new, known or gone +/// differs: Docker and the registry go through `reconcile_set`, Lemonade through +/// `reconcile_lemonade`. #[derive(Clone, Copy)] enum DiscoverySource { Docker, @@ -1062,9 +1066,9 @@ impl Reconcile<'_> { /// managed-service registry). `known` is what the previous pass saw and /// `seen` what this one saw; the caller stores `seen` as the next `known`. /// - /// A service whose port changed under the same id is a new logical service - /// at a new scrape address: its rate baseline and latency windows are reset - /// and no live fields carry over. + /// A service whose port changed under the same id has a new scrape address: + /// its rate baseline and latency windows are reset and no live fields carry + /// over. It is still the same service, so it is not announced again. fn reconcile_set( &mut self, source: DiscoverySource, @@ -1806,7 +1810,7 @@ mod tests { assert_eq!(pass.kv("l1"), Some(25.0), "live fields carry over"); // A new id (the host or port changed) purges the old id's telemetry and - // announces the new one without carrying anything across. + // announces the new one. pass.reconcile() .reconcile_lemonade(Some(discovered("l2", 13306)), &mut current); assert_eq!(pass.announced(), ["discovered l2"]); diff --git a/crates/rocm-dash-daemon/tests/discovery_reconcile.rs b/crates/rocm-dash-daemon/tests/discovery_reconcile.rs index 5a1a81c0b8..bd7c7b5200 100644 --- a/crates/rocm-dash-daemon/tests/discovery_reconcile.rs +++ b/crates/rocm-dash-daemon/tests/discovery_reconcile.rs @@ -8,13 +8,16 @@ //! Docker, the probe-based Lemonade endpoint, and the managed-service registry. //! These tests drive the real loop through the sources that can run without a //! container daemon (the registry and Lemonade) and pin what a user of the -//! broadcast stream can observe: a service is announced once, keeps its live -//! scrape fields across rediscovery, starts clean when its endpoint changes, and -//! is announced gone exactly when it disappears. +//! broadcast stream can observe. On the registry source: a service is announced +//! once, keeps its live scrape fields across rediscovery, and starts clean when +//! its endpoint changes. On Lemonade: the endpoint is announced once, scraped, +//! and announced gone when it stops answering. //! -//! The Docker source has no seam that works without a container daemon, so its -//! reconcile is covered only by the unit tests next to the shared helper in -//! `runner.rs`. +//! The registry's retirement of a vanished service and Lemonade's change of id +//! are not driven through the loop here; they are covered only by the unit tests +//! in `runner.rs`. The Docker source has no seam that works without a container +//! daemon, so its reconcile is covered only by the unit tests next to the shared +//! helper in `runner.rs`. use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; @@ -215,9 +218,11 @@ const KV_PCT: f32 = 25.0; /// that trails it by a tick or two is still seen. const SETTLE_SNAPSHOTS: usize = 3; -/// Read snapshots until `pick` returns a value. Bounded as a whole, so a -/// condition that never holds fails at [`DEADLINE`] instead of looping on fresh -/// snapshots forever. +/// Read snapshots until `pick` returns a value. The deadline is checked before +/// each snapshot, so a condition that never holds fails instead of looping on +/// fresh snapshots forever. Each wait for the next snapshot still has its own +/// [`DEADLINE`], so if the runner stops broadcasting, the failure can come up to +/// one further [`DEADLINE`] after the first. async fn snapshot_where( rx: &mut broadcast::Receiver, pick: impl Fn(&Snapshot) -> Option, @@ -363,7 +368,7 @@ async fn managed_endpoint_change_starts_the_instance_clean() { let services_dir = tmp.path().join("services"); let healthy = spawn_server(|_| vllm_metrics()).await; // The new endpoint never answers a scrape, so any live field on the instance - // after the change was carried over from the old one. + // when it first appears on the new port was carried over from the old one. let silent = spawn_server(|_| http(503, "text/plain", "")).await; write_record(&services_dir, healthy.port); let (mut rx, _runner) = spawn_runner(managed_opts(services_dir.clone())); @@ -378,8 +383,8 @@ async fn managed_endpoint_change_starts_the_instance_clean() { write_record(&services_dir, silent.port); // Watch the stream from the moment the record changes until the instance has - // been seen on its new port and has stayed there for a few snapshots, under - // one deadline. Any announcement or retirement of the service in that window + // been seen on its new port and a few more snapshots have followed, under one + // deadline. Any announcement or retirement of the service in that window // means the change was treated as a different service rather than the same // one moving. let mut change = Change::default(); From 1afbda1d2eea33bf5af281d14e85553e90d272b3 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 07:16:54 +0000 Subject: [PATCH 7/8] docs(dash-daemon): say which reconcile behaviours only the unit tests cover Comment-only. The heading of the runner's reconcile unit-test section said the registry and Lemonade sources are covered end to end in tests/discovery_reconcile.rs without qualification. The registry's retirement of a vanished service and Lemonade's change of id are driven only by the unit tests next to the helper, as the integration file's module doc already says. Signed-off-by: Jussi Elo --- crates/rocm-dash-daemon/src/runner.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/crates/rocm-dash-daemon/src/runner.rs b/crates/rocm-dash-daemon/src/runner.rs index 168a138bde..781fdd5303 100644 --- a/crates/rocm-dash-daemon/src/runner.rs +++ b/crates/rocm-dash-daemon/src/runner.rs @@ -1599,8 +1599,10 @@ mod tests { // The Docker source cannot be driven end to end without a container // daemon, and `reconcile_set` is the code both it and the managed-service // registry run, so these tests exercise it directly. The registry and - // Lemonade sources are also covered end to end in - // `tests/discovery_reconcile.rs`. + // Lemonade sources are also driven end to end in + // `tests/discovery_reconcile.rs`, except the registry's retirement of a + // vanished service and Lemonade's change of id, which only these tests + // cover. fn discovered(id: &str, port: u16) -> DiscoveredService { DiscoveredService { From f088812dae210165c7a28d04d32f77e8cec4e1d2 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 14:28:56 +0000 Subject: [PATCH 8/8] refactor(dash-daemon): make TelemetryState purge and endpoint invalidation the implementation TelemetryState::purge and TelemetryState::invalidate_on_endpoint_change only forwarded to same-named free functions (purge_telemetry_state and invalidate_on_endpoint_change) that nothing but unit tests called. Move the bodies into the methods and delete the free functions so each operation has a single name. The two unit tests now drive the methods on a TelemetryState instead of the free functions. The purge test is renamed from purge_telemetry_state_removes_all_three_maps_for_id to telemetry_state_purge_removes_all_three_maps_for_id because its old name referred to the removed function; the invalidate test keeps its name, which already matches the method. No behaviour change. Signed-off-by: Jussi Elo --- crates/rocm-dash-daemon/src/runner.rs | 129 +++++++++++--------------- 1 file changed, 55 insertions(+), 74 deletions(-) diff --git a/crates/rocm-dash-daemon/src/runner.rs b/crates/rocm-dash-daemon/src/runner.rs index 87b590f199..7d526b9577 100644 --- a/crates/rocm-dash-daemon/src/runner.rs +++ b/crates/rocm-dash-daemon/src/runner.rs @@ -925,37 +925,6 @@ fn count_vram_join_misses( .count() } -/// Remove all per-service telemetry state for `id`: tracker, TTFT, and TPOT -/// baselines. Call on explicit removal (Docker Gone, Lemonade Gone, managed -/// Gone) and whenever an identity change requires a clean slate. -pub(crate) fn purge_telemetry_state( - id: &str, - gen_trackers: &mut HashMap, - prev_ttft: &mut LatencyWindows, - prev_tpot: &mut LatencyWindows, -) { - gen_trackers.remove(id); - prev_ttft.remove(id); - prev_tpot.remove(id); -} - -/// Invalidate the generation tracker for `id` (reset baseline without -/// removing the map entry) and clear the TTFT/TPOT latency windows. -/// Called when an endpoint changes within the same container/service id so -/// the new scrape address starts with a fresh rate baseline. -pub(crate) fn invalidate_on_endpoint_change( - id: &str, - gen_trackers: &mut HashMap, - prev_ttft: &mut LatencyWindows, - prev_tpot: &mut LatencyWindows, -) { - if let Some(t) = gen_trackers.get_mut(id) { - t.invalidate(); - } - prev_ttft.remove(id); - prev_tpot.remove(id); -} - /// Build an `Instance` from a `DiscoveredService` with no live KV/req sample yet. /// /// Status is taken from `svc.status` — the caller sets it from whatever real @@ -1016,22 +985,25 @@ struct TelemetryState { } impl TelemetryState { + /// Remove all per-service telemetry state for `id`: tracker, TTFT, and + /// TPOT baselines. Call on explicit removal (Docker Gone, Lemonade Gone, + /// managed Gone) and whenever an identity change requires a clean slate. fn purge(&mut self, id: &str) { - purge_telemetry_state( - id, - &mut self.gen_trackers, - &mut self.prev_ttft, - &mut self.prev_tpot, - ); + self.gen_trackers.remove(id); + self.prev_ttft.remove(id); + self.prev_tpot.remove(id); } + /// Invalidate the generation tracker for `id` (reset baseline without + /// removing the map entry) and clear the TTFT/TPOT latency windows. + /// Called when an endpoint changes within the same container/service id so + /// the new scrape address starts with a fresh rate baseline. fn invalidate_on_endpoint_change(&mut self, id: &str) { - invalidate_on_endpoint_change( - id, - &mut self.gen_trackers, - &mut self.prev_ttft, - &mut self.prev_tpot, - ); + if let Some(t) = self.gen_trackers.get_mut(id) { + t.invalidate(); + } + self.prev_ttft.remove(id); + self.prev_tpot.remove(id); } } @@ -1719,66 +1691,75 @@ mod tests { GenerationObservationTracker::new(Duration::from_secs(tick_secs)) } - /// `purge_telemetry_state` removes the tracker entry, TTFT baseline, and + /// `TelemetryState::purge` removes the tracker entry, TTFT baseline, and /// TPOT baseline for the target id and leaves other ids untouched. #[test] - fn purge_telemetry_state_removes_all_three_maps_for_id() { - let mut trackers: HashMap = HashMap::new(); - let mut ttft: LatencyWindows = HashMap::new(); - let mut tpot: LatencyWindows = HashMap::new(); + fn telemetry_state_purge_removes_all_three_maps_for_id() { + let mut state = TelemetryState::default(); // Populate two ids. - trackers.insert("a".into(), make_tracker(2)); - trackers.insert("b".into(), make_tracker(2)); - ttft.insert("a".into(), baseline(1.0, 1.0)); - ttft.insert("b".into(), baseline(2.0, 2.0)); - tpot.insert("a".into(), baseline(3.0, 3.0)); - tpot.insert("b".into(), baseline(4.0, 4.0)); + state.gen_trackers.insert("a".into(), make_tracker(2)); + state.gen_trackers.insert("b".into(), make_tracker(2)); + state.prev_ttft.insert("a".into(), baseline(1.0, 1.0)); + state.prev_ttft.insert("b".into(), baseline(2.0, 2.0)); + state.prev_tpot.insert("a".into(), baseline(3.0, 3.0)); + state.prev_tpot.insert("b".into(), baseline(4.0, 4.0)); - purge_telemetry_state("a", &mut trackers, &mut ttft, &mut tpot); + state.purge("a"); // "a" must be gone from all maps. - assert!(!trackers.contains_key("a"), "tracker for a must be removed"); - assert!(!ttft.contains_key("a"), "ttft for a must be removed"); - assert!(!tpot.contains_key("a"), "tpot for a must be removed"); + assert!( + !state.gen_trackers.contains_key("a"), + "tracker for a must be removed" + ); + assert!( + !state.prev_ttft.contains_key("a"), + "ttft for a must be removed" + ); + assert!( + !state.prev_tpot.contains_key("a"), + "tpot for a must be removed" + ); // "b" must be unaffected. - assert!(trackers.contains_key("b"), "tracker for b must survive"); - assert!(ttft.contains_key("b"), "ttft for b must survive"); - assert!(tpot.contains_key("b"), "tpot for b must survive"); + assert!( + state.gen_trackers.contains_key("b"), + "tracker for b must survive" + ); + assert!(state.prev_ttft.contains_key("b"), "ttft for b must survive"); + assert!(state.prev_tpot.contains_key("b"), "tpot for b must survive"); } - /// `invalidate_on_endpoint_change` invalidates the tracker (resets its - /// window so subsequent snapshots return None) and removes the latency - /// baselines, without removing the tracker entry itself. + /// `TelemetryState::invalidate_on_endpoint_change` invalidates the tracker + /// (resets its window so subsequent snapshots return None) and removes the + /// latency baselines, without removing the tracker entry itself. #[test] fn invalidate_on_endpoint_change_resets_tracker_and_clears_latency() { - let mut trackers: HashMap = HashMap::new(); - let mut ttft: LatencyWindows = HashMap::new(); - let mut tpot: LatencyWindows = HashMap::new(); + let mut state = TelemetryState::default(); let mut tracker = make_tracker(2); // Give the tracker a reading so it has a non-None baseline. tracker.observe_counter(100.0, at(10)); tracker.observe_counter(200.0, at(12)); - trackers.insert("svc".into(), tracker); - ttft.insert("svc".into(), baseline(1.0, 10.0)); - tpot.insert("svc".into(), baseline(0.5, 8.0)); + state.gen_trackers.insert("svc".into(), tracker); + state.prev_ttft.insert("svc".into(), baseline(1.0, 10.0)); + state.prev_tpot.insert("svc".into(), baseline(0.5, 8.0)); - invalidate_on_endpoint_change("svc", &mut trackers, &mut ttft, &mut tpot); + state.invalidate_on_endpoint_change("svc"); // Tracker still exists but its window is reset: snapshot returns None. - let (rate, _meta) = trackers + let (rate, _meta) = state + .gen_trackers .get_mut("svc") .expect("tracker entry must remain") .snapshot(at(12)); assert!(rate.is_none(), "invalidated tracker must yield None rate"); // Latency baselines must be cleared. assert!( - !ttft.contains_key("svc"), + !state.prev_ttft.contains_key("svc"), "ttft baseline must be cleared on endpoint change" ); assert!( - !tpot.contains_key("svc"), + !state.prev_tpot.contains_key("svc"), "tpot baseline must be cleared on endpoint change" ); }