From 5ac34a40a0364f05e5ffb14178e23715635371e2 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 21:31:42 +0000 Subject: [PATCH 1/2] test(dash-daemon): cover every state a Lemonade probe can observe Add one test per lifecycle row of the Lemonade source: never answered, first answer, the same endpoint again, a host change, a port change, the endpoint going quiet, coming back elsewhere, the same answer twice in one tick, and moving away and back. Each row asserts the announcements, the instances a snapshot would carry, the per-id telemetry and the (id, port) pairs the vLLM scrape would poll. The vLLM scrape target selection moves out of run_loop into vllm_scrape_targets, unchanged, so the tests can ask which endpoints would be polled. The host-change, port-change and move-away-and-back rows pin what the code does today: the superseded instance is neither announced gone nor removed and is still a scrape target. A following change flips them. Signed-off-by: Jussi Elo --- crates/rocm-dash-daemon/src/runner.rs | 295 +++++++++++++++++++++++++- 1 file changed, 287 insertions(+), 8 deletions(-) diff --git a/crates/rocm-dash-daemon/src/runner.rs b/crates/rocm-dash-daemon/src/runner.rs index 7d526b9577..af2f117d51 100644 --- a/crates/rocm-dash-daemon/src/runner.rs +++ b/crates/rocm-dash-daemon/src/runner.rs @@ -614,14 +614,11 @@ pub async fn run_loop( && !runner.state.instances.is_empty() && tick_count.is_multiple_of(instance_ticks) { - let targets: Vec<(String, u16)> = runner - .state - .instances - .values() - .filter(|i| Some(i.container_id.as_str()) != discovery.lemonade_id.as_deref()) - .filter(|i| !discovery.managed_non_vllm.contains(&i.container_id)) - .filter_map(|i| i.port.map(|p| (i.container_id.clone(), p))) - .collect(); + let targets = vllm_scrape_targets( + &runner.state.instances, + discovery.lemonade_id.as_deref(), + &discovery.managed_non_vllm, + ); let prom = prom.clone(); let results = parallel_scrape(targets, move |(id, port)| { let prom = prom.clone(); @@ -1174,6 +1171,22 @@ impl Reconcile<'_> { } } +/// The `(id, port)` pairs the per-instance vLLM Prometheus scrape polls: every +/// instance with a port except the live Lemonade instance (its own collector +/// scrapes it) and the managed services whose engine is not vLLM. +fn vllm_scrape_targets( + instances: &HashMap, + lemonade_id: Option<&str>, + managed_non_vllm: &HashSet, +) -> Vec<(String, u16)> { + instances + .values() + .filter(|i| Some(i.container_id.as_str()) != lemonade_id) + .filter(|i| !managed_non_vllm.contains(&i.container_id)) + .filter_map(|i| i.port.map(|p| (i.container_id.clone(), p))) + .collect() +} + /// Set `vram_used_mb`/`vram_total_mb` on each instance from the per-process /// attribution map, falling back to device-summed VRAM over the instance's /// GPUs when its container has no per-process entry. Pure — the runner does the @@ -1962,6 +1975,272 @@ mod tests { assert!(pass.telemetry.prev_ttft.contains_key("a")); } + // ── Lemonade state matrix ─────────────────────────────────────────────── + // + // One Lemonade endpoint is probed per discovery pass. Its id encodes the host + // and port, so the states a probe can observe are: nothing ever answered; the + // first answer; the same endpoint again; the host or the port changed; the + // endpoint went quiet; it came back somewhere else; and the same answer + // arriving twice in one tick. Each row below checks the announcements, the + // instances a snapshot would carry, the telemetry kept per id, and the + // `(id, port)` pairs the vLLM scrape would poll, because a stale instance is + // visible on all four. + + fn lemonade_svc(host: &str, port: u16) -> DiscoveredService { + DiscoveredService { + container_id: format!("lemonade-{host}-{port}"), + container_name: "lemonade".into(), + model_name: "model".into(), + port: Some(port), + ..Default::default() + } + } + + impl Pass { + /// The ids a snapshot carries. The loop builds a snapshot's instances from + /// `state.instances`, so this is the dashboard's view of the pass. + fn instance_ids(&self) -> Vec { + let mut ids: Vec = self.state.instances.keys().cloned().collect(); + ids.sort(); + ids + } + + /// The ids the vLLM Prometheus scrape would poll while `current` is the + /// live Lemonade id. + fn scrape_targets(&self, current: Option<&str>) -> Vec { + let mut ids: Vec = + vllm_scrape_targets(&self.state.instances, current, &HashSet::new()) + .into_iter() + .map(|(id, _port)| id) + .collect(); + ids.sort(); + ids + } + + /// A live KV reading, latency baselines and a rate tracker for `id`. + fn give_live_telemetry(&mut self, id: &str) { + self.scrape(id); + self.telemetry + .gen_trackers + .insert(id.into(), make_tracker(2)); + } + + fn has_telemetry(&self, id: &str) -> bool { + self.telemetry.gen_trackers.contains_key(id) + || self.telemetry.prev_ttft.contains_key(id) + || self.telemetry.prev_tpot.contains_key(id) + } + } + + fn ids(ids: &[&str]) -> Vec { + ids.iter().map(|s| (*s).to_string()).collect() + } + + /// Discover `old`, give it live telemetry, then probe `new`. Returns the pass, + /// the loop's current Lemonade id and what the second probe announced. + fn lemonade_identity_change( + old: &DiscoveredService, + new: &DiscoveredService, + ) -> (Pass, Option, Vec) { + let mut pass = Pass::new(); + let mut current: Option = None; + pass.reconcile() + .reconcile_lemonade(Some(old.clone()), &mut current); + pass.give_live_telemetry(&old.container_id); + pass.announced(); + + pass.reconcile() + .reconcile_lemonade(Some(new.clone()), &mut current); + let announced = pass.announced(); + (pass, current, announced) + } + + #[test] + fn lemonade_that_never_answers_leaves_no_trace() { + let mut pass = Pass::new(); + let mut current: Option = None; + + pass.reconcile().reconcile_lemonade(None, &mut current); + pass.reconcile().reconcile_lemonade(None, &mut current); + + assert!(pass.announced().is_empty()); + assert_eq!(current, None); + assert!(pass.instance_ids().is_empty()); + assert!(pass.scrape_targets(current.as_deref()).is_empty()); + } + + #[test] + fn lemonade_first_answer_is_announced_once_and_left_to_its_own_collector() { + let mut pass = Pass::new(); + let mut current: Option = None; + + pass.reconcile() + .reconcile_lemonade(Some(lemonade_svc("127.0.0.1", 13305)), &mut current); + + assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13305"]); + assert_eq!(current.as_deref(), Some("lemonade-127.0.0.1-13305")); + assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13305"])); + assert!( + pass.scrape_targets(current.as_deref()).is_empty(), + "the live Lemonade instance is not a vLLM scrape target" + ); + } + + #[test] + fn lemonade_same_endpoint_again_keeps_its_instance_and_live_fields() { + let svc = lemonade_svc("127.0.0.1", 13305); + let id = svc.container_id.clone(); + let mut pass = Pass::new(); + let mut current: Option = None; + pass.reconcile() + .reconcile_lemonade(Some(svc.clone()), &mut current); + pass.give_live_telemetry(&id); + pass.announced(); + + pass.reconcile().reconcile_lemonade(Some(svc), &mut current); + + assert!(pass.announced().is_empty()); + assert_eq!(pass.instance_ids(), ids(&[&id])); + assert_eq!(pass.kv(&id), Some(25.0)); + assert!(pass.has_telemetry(&id)); + assert!(pass.scrape_targets(current.as_deref()).is_empty()); + } + + #[test] + fn lemonade_host_change_pins_the_stale_old_instance() { + let old = lemonade_svc("127.0.0.1", 13305); + let new = lemonade_svc("10.0.0.7", 13305); + let (pass, current, announced) = lemonade_identity_change(&old, &new); + + assert_eq!(current.as_deref(), Some(new.container_id.as_str())); + assert_eq!(announced, ["discovered lemonade-10.0.0.7-13305"]); + assert!(!pass.has_telemetry(&old.container_id)); + // Pins current behaviour, not a goal: the old instance is neither + // announced gone nor removed, stays in every snapshot, and is polled as + // a vLLM target because the scrape filter only skips the current id. + assert_eq!( + pass.instance_ids(), + ids(&["lemonade-10.0.0.7-13305", "lemonade-127.0.0.1-13305"]) + ); + assert_eq!( + pass.scrape_targets(current.as_deref()), + ids(&[&old.container_id]) + ); + } + + #[test] + fn lemonade_port_change_pins_the_stale_old_instance() { + let old = lemonade_svc("127.0.0.1", 13305); + let new = lemonade_svc("127.0.0.1", 13306); + let (pass, current, announced) = lemonade_identity_change(&old, &new); + + assert_eq!(current.as_deref(), Some(new.container_id.as_str())); + assert_eq!(announced, ["discovered lemonade-127.0.0.1-13306"]); + assert!(!pass.has_telemetry(&old.container_id)); + // Pins current behaviour, not a goal: see the host-change row. + assert_eq!( + pass.instance_ids(), + ids(&["lemonade-127.0.0.1-13305", "lemonade-127.0.0.1-13306"]) + ); + assert_eq!( + pass.scrape_targets(current.as_deref()), + ids(&[&old.container_id]) + ); + } + + #[test] + fn lemonade_endpoint_going_quiet_is_announced_gone_and_leaves_no_trace() { + let svc = lemonade_svc("127.0.0.1", 13305); + let id = svc.container_id.clone(); + let mut pass = Pass::new(); + let mut current: Option = None; + pass.reconcile().reconcile_lemonade(Some(svc), &mut current); + pass.give_live_telemetry(&id); + pass.announced(); + + pass.reconcile().reconcile_lemonade(None, &mut current); + + assert_eq!(pass.announced(), [format!("gone {id}")]); + assert_eq!(current, None); + assert!(pass.instance_ids().is_empty()); + assert!(!pass.has_telemetry(&id)); + assert!(pass.scrape_targets(current.as_deref()).is_empty()); + } + + #[test] + fn lemonade_coming_back_elsewhere_after_going_quiet_is_the_only_instance() { + let old = lemonade_svc("127.0.0.1", 13305); + let new = lemonade_svc("127.0.0.1", 13306); + let mut pass = Pass::new(); + let mut current: Option = None; + pass.reconcile() + .reconcile_lemonade(Some(old.clone()), &mut current); + pass.give_live_telemetry(&old.container_id); + pass.reconcile().reconcile_lemonade(None, &mut current); + assert_eq!( + pass.announced(), + [ + "discovered lemonade-127.0.0.1-13305", + "gone lemonade-127.0.0.1-13305" + ] + ); + + pass.reconcile() + .reconcile_lemonade(Some(new.clone()), &mut current); + + assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13306"]); + assert_eq!(current.as_deref(), Some(new.container_id.as_str())); + assert_eq!(pass.instance_ids(), ids(&[&new.container_id])); + assert!(!pass.has_telemetry(&old.container_id)); + assert!(pass.scrape_targets(current.as_deref()).is_empty()); + } + + #[test] + fn lemonade_answer_repeated_within_one_tick_is_announced_once() { + let svc = lemonade_svc("127.0.0.1", 13305); + let mut pass = Pass::new(); + let mut current: Option = None; + + pass.reconcile() + .reconcile_lemonade(Some(svc.clone()), &mut current); + pass.reconcile().reconcile_lemonade(Some(svc), &mut current); + + assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13305"]); + assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13305"])); + assert!(pass.scrape_targets(current.as_deref()).is_empty()); + } + + #[test] + fn lemonade_endpoint_moving_away_and_back_pins_the_stale_instances() { + let a = lemonade_svc("127.0.0.1", 13305); + let b = lemonade_svc("127.0.0.1", 13306); + let mut pass = Pass::new(); + let mut current: Option = None; + pass.reconcile() + .reconcile_lemonade(Some(a.clone()), &mut current); + pass.announced(); + + pass.reconcile() + .reconcile_lemonade(Some(b.clone()), &mut current); + assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13306"]); + pass.reconcile() + .reconcile_lemonade(Some(a.clone()), &mut current); + + // Pins current behaviour, not a goal: the first endpoint is announced + // as discovered for a second time without ever being announced gone, and + // the endpoint that was left stays behind as a vLLM scrape target. + assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13305"]); + assert_eq!(current.as_deref(), Some(a.container_id.as_str())); + assert_eq!( + pass.instance_ids(), + ids(&[&a.container_id, &b.container_id]) + ); + assert_eq!( + pass.scrape_targets(current.as_deref()), + ids(&[&b.container_id]) + ); + } + #[test] fn reconcile_lemonade_follows_one_endpoint_through_its_lifecycle() { let mut pass = Pass::new(); From a8aceb6a265f63958b7646119081110305db5e87 Mon Sep 17 00:00:00 2001 From: Jussi Elo Date: Thu, 5 Nov 2026 21:41:55 +0000 Subject: [PATCH 2/2] fix(dash-daemon): retire the old Lemonade instance when its host or port changes The Lemonade id encodes host and port, so moving the endpoint yields a new id. The old id's telemetry was purged and the new id announced, but the old instance was never retired: it stayed in the state and in every snapshot, no InstanceGone was sent for it, and the vLLM scrape kept polling its port, because the scrape filter only skips the current Lemonade id. On an id change the old id is now retired first (InstanceGone, removed from the state, telemetry purged) and then the new id is upserted and announced, so a snapshot carries exactly one Lemonade instance and nothing polls the endpoint it left. The rows of the Lemonade state matrix that pinned the stale instance, and the pin in the lifecycle test, now assert the retired behaviour. Signed-off-by: Jussi Elo --- crates/rocm-dash-daemon/src/runner.rs | 102 +++++++++++++------------- 1 file changed, 53 insertions(+), 49 deletions(-) diff --git a/crates/rocm-dash-daemon/src/runner.rs b/crates/rocm-dash-daemon/src/runner.rs index af2f117d51..5b03a64642 100644 --- a/crates/rocm-dash-daemon/src/runner.rs +++ b/crates/rocm-dash-daemon/src/runner.rs @@ -1141,10 +1141,12 @@ impl Reconcile<'_> { /// Reconcile the probe-based Lemonade source, which has at most one service. /// /// Its `container_id` encodes the host and port, so a port or host change - /// yields a new id: the old id's telemetry is purged so a stale rate - /// baseline, TTFT and TPOT do not bleed across, and the new id is announced. - /// The old instance is not retired here. `None` means the endpoint did not - /// answer, which retires the current one. + /// yields a new id. The old id is retired first (its telemetry is purged, so + /// a stale rate baseline, TTFT and TPOT do not bleed across; it leaves the + /// state and is announced gone), then the new id is announced, so the state + /// and every snapshot carry one Lemonade instance and the vLLM scrape never + /// polls the old endpoint. `None` means the endpoint did not answer, which + /// retires the current one. fn reconcile_lemonade( &mut self, probed: Option, @@ -1153,8 +1155,8 @@ impl Reconcile<'_> { match probed { Some(svc) => { let is_new_id = current.as_deref() != Some(svc.container_id.as_str()); - if is_new_id && let Some(old_id) = current.as_deref() { - self.telemetry.purge(old_id); + if is_new_id && let Some(old_id) = current.take() { + self.retire(DiscoverySource::Lemonade, &old_id); } self.upsert(&svc, !is_new_id); if is_new_id { @@ -2107,44 +2109,46 @@ mod tests { } #[test] - fn lemonade_host_change_pins_the_stale_old_instance() { + fn lemonade_host_change_retires_the_old_instance_and_announces_the_new_one() { let old = lemonade_svc("127.0.0.1", 13305); let new = lemonade_svc("10.0.0.7", 13305); let (pass, current, announced) = lemonade_identity_change(&old, &new); assert_eq!(current.as_deref(), Some(new.container_id.as_str())); - assert_eq!(announced, ["discovered lemonade-10.0.0.7-13305"]); - assert!(!pass.has_telemetry(&old.container_id)); - // Pins current behaviour, not a goal: the old instance is neither - // announced gone nor removed, stays in every snapshot, and is polled as - // a vLLM target because the scrape filter only skips the current id. assert_eq!( - pass.instance_ids(), - ids(&["lemonade-10.0.0.7-13305", "lemonade-127.0.0.1-13305"]) + announced, + [ + "gone lemonade-127.0.0.1-13305", + "discovered lemonade-10.0.0.7-13305" + ] ); - assert_eq!( - pass.scrape_targets(current.as_deref()), - ids(&[&old.container_id]) + assert!(!pass.has_telemetry(&old.container_id)); + assert_eq!(pass.instance_ids(), ids(&["lemonade-10.0.0.7-13305"])); + assert!( + pass.scrape_targets(current.as_deref()).is_empty(), + "the old endpoint must not be polled as a vLLM target" ); } #[test] - fn lemonade_port_change_pins_the_stale_old_instance() { + fn lemonade_port_change_retires_the_old_instance_and_announces_the_new_one() { let old = lemonade_svc("127.0.0.1", 13305); let new = lemonade_svc("127.0.0.1", 13306); let (pass, current, announced) = lemonade_identity_change(&old, &new); assert_eq!(current.as_deref(), Some(new.container_id.as_str())); - assert_eq!(announced, ["discovered lemonade-127.0.0.1-13306"]); - assert!(!pass.has_telemetry(&old.container_id)); - // Pins current behaviour, not a goal: see the host-change row. assert_eq!( - pass.instance_ids(), - ids(&["lemonade-127.0.0.1-13305", "lemonade-127.0.0.1-13306"]) + announced, + [ + "gone lemonade-127.0.0.1-13305", + "discovered lemonade-127.0.0.1-13306" + ] ); - assert_eq!( - pass.scrape_targets(current.as_deref()), - ids(&[&old.container_id]) + assert!(!pass.has_telemetry(&old.container_id)); + assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13306"])); + assert!( + pass.scrape_targets(current.as_deref()).is_empty(), + "the old endpoint must not be polled as a vLLM target" ); } @@ -2211,7 +2215,7 @@ mod tests { } #[test] - fn lemonade_endpoint_moving_away_and_back_pins_the_stale_instances() { + fn lemonade_endpoint_moving_away_and_back_is_retired_each_time() { let a = lemonade_svc("127.0.0.1", 13305); let b = lemonade_svc("127.0.0.1", 13306); let mut pass = Pass::new(); @@ -2220,25 +2224,27 @@ mod tests { .reconcile_lemonade(Some(a.clone()), &mut current); pass.announced(); - pass.reconcile() - .reconcile_lemonade(Some(b.clone()), &mut current); - assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13306"]); + pass.reconcile().reconcile_lemonade(Some(b), &mut current); + assert_eq!( + pass.announced(), + [ + "gone lemonade-127.0.0.1-13305", + "discovered lemonade-127.0.0.1-13306" + ] + ); pass.reconcile() .reconcile_lemonade(Some(a.clone()), &mut current); - // Pins current behaviour, not a goal: the first endpoint is announced - // as discovered for a second time without ever being announced gone, and - // the endpoint that was left stays behind as a vLLM scrape target. - assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13305"]); - assert_eq!(current.as_deref(), Some(a.container_id.as_str())); assert_eq!( - pass.instance_ids(), - ids(&[&a.container_id, &b.container_id]) - ); - assert_eq!( - pass.scrape_targets(current.as_deref()), - ids(&[&b.container_id]) + pass.announced(), + [ + "gone lemonade-127.0.0.1-13306", + "discovered lemonade-127.0.0.1-13305" + ] ); + assert_eq!(current.as_deref(), Some(a.container_id.as_str())); + assert_eq!(pass.instance_ids(), ids(&[&a.container_id])); + assert!(pass.scrape_targets(current.as_deref()).is_empty()); } #[test] @@ -2260,18 +2266,16 @@ mod tests { ); assert_eq!(pass.kv("l1"), Some(25.0), "live fields carry over"); - // A new id (the host or port changed) purges the old id's telemetry and - // announces the new one. + // A new id (the host or port changed) retires the old id, telemetry + // included, and announces the new one. pass.reconcile() .reconcile_lemonade(Some(discovered("l2", 13306)), &mut current); - assert_eq!(pass.announced(), ["discovered l2"]); + assert_eq!(pass.announced(), ["gone l1", "discovered l2"]); assert_eq!(current.as_deref(), Some("l2")); assert!(!pass.telemetry.prev_ttft.contains_key("l1")); - // Pins current behaviour, not a goal: the superseded instance stays in - // state (and so in every snapshot) because only its telemetry is purged. - // Retiring it later is a deliberate behaviour change, and this assertion - // is the one to update when that happens. - assert!(pass.state.instances.contains_key("l1")); + // The superseded instance leaves the state (and so every snapshot), not + // only its telemetry. + assert!(!pass.state.instances.contains_key("l1")); pass.reconcile().reconcile_lemonade(None, &mut current); assert_eq!(pass.announced(), ["gone l2"]);