diff --git a/crates/rocm-dash-daemon/src/runner.rs b/crates/rocm-dash-daemon/src/runner.rs index 7d526b9577..5b03a64642 100644 --- a/crates/rocm-dash-daemon/src/runner.rs +++ b/crates/rocm-dash-daemon/src/runner.rs @@ -614,14 +614,11 @@ pub async fn run_loop( && !runner.state.instances.is_empty() && tick_count.is_multiple_of(instance_ticks) { - let targets: Vec<(String, u16)> = runner - .state - .instances - .values() - .filter(|i| Some(i.container_id.as_str()) != discovery.lemonade_id.as_deref()) - .filter(|i| !discovery.managed_non_vllm.contains(&i.container_id)) - .filter_map(|i| i.port.map(|p| (i.container_id.clone(), p))) - .collect(); + let targets = vllm_scrape_targets( + &runner.state.instances, + discovery.lemonade_id.as_deref(), + &discovery.managed_non_vllm, + ); let prom = prom.clone(); let results = parallel_scrape(targets, move |(id, port)| { let prom = prom.clone(); @@ -1144,10 +1141,12 @@ impl Reconcile<'_> { /// Reconcile the probe-based Lemonade source, which has at most one service. /// /// Its `container_id` encodes the host and port, so a port or host change - /// yields a new id: the old id's telemetry is purged so a stale rate - /// baseline, TTFT and TPOT do not bleed across, and the new id is announced. - /// The old instance is not retired here. `None` means the endpoint did not - /// answer, which retires the current one. + /// yields a new id. The old id is retired first (its telemetry is purged, so + /// a stale rate baseline, TTFT and TPOT do not bleed across; it leaves the + /// state and is announced gone), then the new id is announced, so the state + /// and every snapshot carry one Lemonade instance and the vLLM scrape never + /// polls the old endpoint. `None` means the endpoint did not answer, which + /// retires the current one. fn reconcile_lemonade( &mut self, probed: Option, @@ -1156,8 +1155,8 @@ impl Reconcile<'_> { match probed { Some(svc) => { let is_new_id = current.as_deref() != Some(svc.container_id.as_str()); - if is_new_id && let Some(old_id) = current.as_deref() { - self.telemetry.purge(old_id); + if is_new_id && let Some(old_id) = current.take() { + self.retire(DiscoverySource::Lemonade, &old_id); } self.upsert(&svc, !is_new_id); if is_new_id { @@ -1174,6 +1173,22 @@ impl Reconcile<'_> { } } +/// The `(id, port)` pairs the per-instance vLLM Prometheus scrape polls: every +/// instance with a port except the live Lemonade instance (its own collector +/// scrapes it) and the managed services whose engine is not vLLM. +fn vllm_scrape_targets( + instances: &HashMap, + lemonade_id: Option<&str>, + managed_non_vllm: &HashSet, +) -> Vec<(String, u16)> { + instances + .values() + .filter(|i| Some(i.container_id.as_str()) != lemonade_id) + .filter(|i| !managed_non_vllm.contains(&i.container_id)) + .filter_map(|i| i.port.map(|p| (i.container_id.clone(), p))) + .collect() +} + /// Set `vram_used_mb`/`vram_total_mb` on each instance from the per-process /// attribution map, falling back to device-summed VRAM over the instance's /// GPUs when its container has no per-process entry. Pure — the runner does the @@ -1962,6 +1977,276 @@ mod tests { assert!(pass.telemetry.prev_ttft.contains_key("a")); } + // ── Lemonade state matrix ─────────────────────────────────────────────── + // + // One Lemonade endpoint is probed per discovery pass. Its id encodes the host + // and port, so the states a probe can observe are: nothing ever answered; the + // first answer; the same endpoint again; the host or the port changed; the + // endpoint went quiet; it came back somewhere else; and the same answer + // arriving twice in one tick. Each row below checks the announcements, the + // instances a snapshot would carry, the telemetry kept per id, and the + // `(id, port)` pairs the vLLM scrape would poll, because a stale instance is + // visible on all four. + + fn lemonade_svc(host: &str, port: u16) -> DiscoveredService { + DiscoveredService { + container_id: format!("lemonade-{host}-{port}"), + container_name: "lemonade".into(), + model_name: "model".into(), + port: Some(port), + ..Default::default() + } + } + + impl Pass { + /// The ids a snapshot carries. The loop builds a snapshot's instances from + /// `state.instances`, so this is the dashboard's view of the pass. + fn instance_ids(&self) -> Vec { + let mut ids: Vec = self.state.instances.keys().cloned().collect(); + ids.sort(); + ids + } + + /// The ids the vLLM Prometheus scrape would poll while `current` is the + /// live Lemonade id. + fn scrape_targets(&self, current: Option<&str>) -> Vec { + let mut ids: Vec = + vllm_scrape_targets(&self.state.instances, current, &HashSet::new()) + .into_iter() + .map(|(id, _port)| id) + .collect(); + ids.sort(); + ids + } + + /// A live KV reading, latency baselines and a rate tracker for `id`. + fn give_live_telemetry(&mut self, id: &str) { + self.scrape(id); + self.telemetry + .gen_trackers + .insert(id.into(), make_tracker(2)); + } + + fn has_telemetry(&self, id: &str) -> bool { + self.telemetry.gen_trackers.contains_key(id) + || self.telemetry.prev_ttft.contains_key(id) + || self.telemetry.prev_tpot.contains_key(id) + } + } + + fn ids(ids: &[&str]) -> Vec { + ids.iter().map(|s| (*s).to_string()).collect() + } + + /// Discover `old`, give it live telemetry, then probe `new`. Returns the pass, + /// the loop's current Lemonade id and what the second probe announced. + fn lemonade_identity_change( + old: &DiscoveredService, + new: &DiscoveredService, + ) -> (Pass, Option, Vec) { + let mut pass = Pass::new(); + let mut current: Option = None; + pass.reconcile() + .reconcile_lemonade(Some(old.clone()), &mut current); + pass.give_live_telemetry(&old.container_id); + pass.announced(); + + pass.reconcile() + .reconcile_lemonade(Some(new.clone()), &mut current); + let announced = pass.announced(); + (pass, current, announced) + } + + #[test] + fn lemonade_that_never_answers_leaves_no_trace() { + let mut pass = Pass::new(); + let mut current: Option = None; + + pass.reconcile().reconcile_lemonade(None, &mut current); + pass.reconcile().reconcile_lemonade(None, &mut current); + + assert!(pass.announced().is_empty()); + assert_eq!(current, None); + assert!(pass.instance_ids().is_empty()); + assert!(pass.scrape_targets(current.as_deref()).is_empty()); + } + + #[test] + fn lemonade_first_answer_is_announced_once_and_left_to_its_own_collector() { + let mut pass = Pass::new(); + let mut current: Option = None; + + pass.reconcile() + .reconcile_lemonade(Some(lemonade_svc("127.0.0.1", 13305)), &mut current); + + assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13305"]); + assert_eq!(current.as_deref(), Some("lemonade-127.0.0.1-13305")); + assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13305"])); + assert!( + pass.scrape_targets(current.as_deref()).is_empty(), + "the live Lemonade instance is not a vLLM scrape target" + ); + } + + #[test] + fn lemonade_same_endpoint_again_keeps_its_instance_and_live_fields() { + let svc = lemonade_svc("127.0.0.1", 13305); + let id = svc.container_id.clone(); + let mut pass = Pass::new(); + let mut current: Option = None; + pass.reconcile() + .reconcile_lemonade(Some(svc.clone()), &mut current); + pass.give_live_telemetry(&id); + pass.announced(); + + pass.reconcile().reconcile_lemonade(Some(svc), &mut current); + + assert!(pass.announced().is_empty()); + assert_eq!(pass.instance_ids(), ids(&[&id])); + assert_eq!(pass.kv(&id), Some(25.0)); + assert!(pass.has_telemetry(&id)); + assert!(pass.scrape_targets(current.as_deref()).is_empty()); + } + + #[test] + fn lemonade_host_change_retires_the_old_instance_and_announces_the_new_one() { + let old = lemonade_svc("127.0.0.1", 13305); + let new = lemonade_svc("10.0.0.7", 13305); + let (pass, current, announced) = lemonade_identity_change(&old, &new); + + assert_eq!(current.as_deref(), Some(new.container_id.as_str())); + assert_eq!( + announced, + [ + "gone lemonade-127.0.0.1-13305", + "discovered lemonade-10.0.0.7-13305" + ] + ); + assert!(!pass.has_telemetry(&old.container_id)); + assert_eq!(pass.instance_ids(), ids(&["lemonade-10.0.0.7-13305"])); + assert!( + pass.scrape_targets(current.as_deref()).is_empty(), + "the old endpoint must not be polled as a vLLM target" + ); + } + + #[test] + fn lemonade_port_change_retires_the_old_instance_and_announces_the_new_one() { + let old = lemonade_svc("127.0.0.1", 13305); + let new = lemonade_svc("127.0.0.1", 13306); + let (pass, current, announced) = lemonade_identity_change(&old, &new); + + assert_eq!(current.as_deref(), Some(new.container_id.as_str())); + assert_eq!( + announced, + [ + "gone lemonade-127.0.0.1-13305", + "discovered lemonade-127.0.0.1-13306" + ] + ); + assert!(!pass.has_telemetry(&old.container_id)); + assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13306"])); + assert!( + pass.scrape_targets(current.as_deref()).is_empty(), + "the old endpoint must not be polled as a vLLM target" + ); + } + + #[test] + fn lemonade_endpoint_going_quiet_is_announced_gone_and_leaves_no_trace() { + let svc = lemonade_svc("127.0.0.1", 13305); + let id = svc.container_id.clone(); + let mut pass = Pass::new(); + let mut current: Option = None; + pass.reconcile().reconcile_lemonade(Some(svc), &mut current); + pass.give_live_telemetry(&id); + pass.announced(); + + pass.reconcile().reconcile_lemonade(None, &mut current); + + assert_eq!(pass.announced(), [format!("gone {id}")]); + assert_eq!(current, None); + assert!(pass.instance_ids().is_empty()); + assert!(!pass.has_telemetry(&id)); + assert!(pass.scrape_targets(current.as_deref()).is_empty()); + } + + #[test] + fn lemonade_coming_back_elsewhere_after_going_quiet_is_the_only_instance() { + let old = lemonade_svc("127.0.0.1", 13305); + let new = lemonade_svc("127.0.0.1", 13306); + let mut pass = Pass::new(); + let mut current: Option = None; + pass.reconcile() + .reconcile_lemonade(Some(old.clone()), &mut current); + pass.give_live_telemetry(&old.container_id); + pass.reconcile().reconcile_lemonade(None, &mut current); + assert_eq!( + pass.announced(), + [ + "discovered lemonade-127.0.0.1-13305", + "gone lemonade-127.0.0.1-13305" + ] + ); + + pass.reconcile() + .reconcile_lemonade(Some(new.clone()), &mut current); + + assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13306"]); + assert_eq!(current.as_deref(), Some(new.container_id.as_str())); + assert_eq!(pass.instance_ids(), ids(&[&new.container_id])); + assert!(!pass.has_telemetry(&old.container_id)); + assert!(pass.scrape_targets(current.as_deref()).is_empty()); + } + + #[test] + fn lemonade_answer_repeated_within_one_tick_is_announced_once() { + let svc = lemonade_svc("127.0.0.1", 13305); + let mut pass = Pass::new(); + let mut current: Option = None; + + pass.reconcile() + .reconcile_lemonade(Some(svc.clone()), &mut current); + pass.reconcile().reconcile_lemonade(Some(svc), &mut current); + + assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13305"]); + assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13305"])); + assert!(pass.scrape_targets(current.as_deref()).is_empty()); + } + + #[test] + fn lemonade_endpoint_moving_away_and_back_is_retired_each_time() { + let a = lemonade_svc("127.0.0.1", 13305); + let b = lemonade_svc("127.0.0.1", 13306); + let mut pass = Pass::new(); + let mut current: Option = None; + pass.reconcile() + .reconcile_lemonade(Some(a.clone()), &mut current); + pass.announced(); + + pass.reconcile().reconcile_lemonade(Some(b), &mut current); + assert_eq!( + pass.announced(), + [ + "gone lemonade-127.0.0.1-13305", + "discovered lemonade-127.0.0.1-13306" + ] + ); + pass.reconcile() + .reconcile_lemonade(Some(a.clone()), &mut current); + + assert_eq!( + pass.announced(), + [ + "gone lemonade-127.0.0.1-13306", + "discovered lemonade-127.0.0.1-13305" + ] + ); + assert_eq!(current.as_deref(), Some(a.container_id.as_str())); + assert_eq!(pass.instance_ids(), ids(&[&a.container_id])); + assert!(pass.scrape_targets(current.as_deref()).is_empty()); + } + #[test] fn reconcile_lemonade_follows_one_endpoint_through_its_lifecycle() { let mut pass = Pass::new(); @@ -1981,18 +2266,16 @@ mod tests { ); assert_eq!(pass.kv("l1"), Some(25.0), "live fields carry over"); - // A new id (the host or port changed) purges the old id's telemetry and - // announces the new one. + // A new id (the host or port changed) retires the old id, telemetry + // included, and announces the new one. pass.reconcile() .reconcile_lemonade(Some(discovered("l2", 13306)), &mut current); - assert_eq!(pass.announced(), ["discovered l2"]); + assert_eq!(pass.announced(), ["gone l1", "discovered l2"]); assert_eq!(current.as_deref(), Some("l2")); assert!(!pass.telemetry.prev_ttft.contains_key("l1")); - // Pins current behaviour, not a goal: the superseded instance stays in - // state (and so in every snapshot) because only its telemetry is purged. - // Retiring it later is a deliberate behaviour change, and this assertion - // is the one to update when that happens. - assert!(pass.state.instances.contains_key("l1")); + // The superseded instance leaves the state (and so every snapshot), not + // only its telemetry. + assert!(!pass.state.instances.contains_key("l1")); pass.reconcile().reconcile_lemonade(None, &mut current); assert_eq!(pass.announced(), ["gone l2"]);