Skip to content
Closed
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
327 changes: 305 additions & 22 deletions crates/rocm-dash-daemon/src/runner.rs
Original file line number Diff line number Diff line change
Expand Up @@ -614,14 +614,11 @@ pub async fn run_loop(
&& !runner.state.instances.is_empty()
&& tick_count.is_multiple_of(instance_ticks)
{
let targets: Vec<(String, u16)> = runner
.state
.instances
.values()
.filter(|i| Some(i.container_id.as_str()) != discovery.lemonade_id.as_deref())
.filter(|i| !discovery.managed_non_vllm.contains(&i.container_id))
.filter_map(|i| i.port.map(|p| (i.container_id.clone(), p)))
.collect();
let targets = vllm_scrape_targets(
&runner.state.instances,
discovery.lemonade_id.as_deref(),
&discovery.managed_non_vllm,
);
let prom = prom.clone();
let results = parallel_scrape(targets, move |(id, port)| {
let prom = prom.clone();
Expand Down Expand Up @@ -1144,10 +1141,12 @@ impl Reconcile<'_> {
/// Reconcile the probe-based Lemonade source, which has at most one service.
///
/// Its `container_id` encodes the host and port, so a port or host change
/// yields a new id: the old id's telemetry is purged so a stale rate
/// baseline, TTFT and TPOT do not bleed across, and the new id is announced.
/// The old instance is not retired here. `None` means the endpoint did not
/// answer, which retires the current one.
/// yields a new id. The old id is retired first (its telemetry is purged, so
/// a stale rate baseline, TTFT and TPOT do not bleed across; it leaves the
/// state and is announced gone), then the new id is announced, so the state
/// and every snapshot carry one Lemonade instance and the vLLM scrape never
/// polls the old endpoint. `None` means the endpoint did not answer, which
/// retires the current one.
fn reconcile_lemonade(
&mut self,
probed: Option<DiscoveredService>,
Expand All @@ -1156,8 +1155,8 @@ impl Reconcile<'_> {
match probed {
Some(svc) => {
let is_new_id = current.as_deref() != Some(svc.container_id.as_str());
if is_new_id && let Some(old_id) = current.as_deref() {
self.telemetry.purge(old_id);
if is_new_id && let Some(old_id) = current.take() {
self.retire(DiscoverySource::Lemonade, &old_id);
}
self.upsert(&svc, !is_new_id);
if is_new_id {
Expand All @@ -1174,6 +1173,22 @@ impl Reconcile<'_> {
}
}

/// The `(id, port)` pairs the per-instance vLLM Prometheus scrape polls: every
/// instance with a port except the live Lemonade instance (its own collector
/// scrapes it) and the managed services whose engine is not vLLM.
fn vllm_scrape_targets(
instances: &HashMap<String, Instance>,
lemonade_id: Option<&str>,
managed_non_vllm: &HashSet<String>,
) -> Vec<(String, u16)> {
instances
.values()
.filter(|i| Some(i.container_id.as_str()) != lemonade_id)
.filter(|i| !managed_non_vllm.contains(&i.container_id))
.filter_map(|i| i.port.map(|p| (i.container_id.clone(), p)))
.collect()
}

/// Set `vram_used_mb`/`vram_total_mb` on each instance from the per-process
/// attribution map, falling back to device-summed VRAM over the instance's
/// GPUs when its container has no per-process entry. Pure — the runner does the
Expand Down Expand Up @@ -1962,6 +1977,276 @@ mod tests {
assert!(pass.telemetry.prev_ttft.contains_key("a"));
}

// ── Lemonade state matrix ───────────────────────────────────────────────
//
// One Lemonade endpoint is probed per discovery pass. Its id encodes the host
// and port, so the states a probe can observe are: nothing ever answered; the
// first answer; the same endpoint again; the host or the port changed; the
// endpoint went quiet; it came back somewhere else; and the same answer
// arriving twice in one tick. Each row below checks the announcements, the
// instances a snapshot would carry, the telemetry kept per id, and the
// `(id, port)` pairs the vLLM scrape would poll, because a stale instance is
// visible on all four.

fn lemonade_svc(host: &str, port: u16) -> DiscoveredService {
DiscoveredService {
container_id: format!("lemonade-{host}-{port}"),
container_name: "lemonade".into(),
model_name: "model".into(),
port: Some(port),
..Default::default()
}
}

impl Pass {
/// The ids a snapshot carries. The loop builds a snapshot's instances from
/// `state.instances`, so this is the dashboard's view of the pass.
fn instance_ids(&self) -> Vec<String> {
let mut ids: Vec<String> = self.state.instances.keys().cloned().collect();
ids.sort();
ids
}

/// The ids the vLLM Prometheus scrape would poll while `current` is the
/// live Lemonade id.
fn scrape_targets(&self, current: Option<&str>) -> Vec<String> {
let mut ids: Vec<String> =
vllm_scrape_targets(&self.state.instances, current, &HashSet::new())
.into_iter()
.map(|(id, _port)| id)
.collect();
ids.sort();
ids
}

/// A live KV reading, latency baselines and a rate tracker for `id`.
fn give_live_telemetry(&mut self, id: &str) {
self.scrape(id);
self.telemetry
.gen_trackers
.insert(id.into(), make_tracker(2));
}

fn has_telemetry(&self, id: &str) -> bool {
self.telemetry.gen_trackers.contains_key(id)
|| self.telemetry.prev_ttft.contains_key(id)
|| self.telemetry.prev_tpot.contains_key(id)
}
}

fn ids(ids: &[&str]) -> Vec<String> {
ids.iter().map(|s| (*s).to_string()).collect()
}

/// Discover `old`, give it live telemetry, then probe `new`. Returns the pass,
/// the loop's current Lemonade id and what the second probe announced.
fn lemonade_identity_change(
old: &DiscoveredService,
new: &DiscoveredService,
) -> (Pass, Option<String>, Vec<String>) {
let mut pass = Pass::new();
let mut current: Option<String> = None;
pass.reconcile()
.reconcile_lemonade(Some(old.clone()), &mut current);
pass.give_live_telemetry(&old.container_id);
pass.announced();

pass.reconcile()
.reconcile_lemonade(Some(new.clone()), &mut current);
let announced = pass.announced();
(pass, current, announced)
}

#[test]
fn lemonade_that_never_answers_leaves_no_trace() {
let mut pass = Pass::new();
let mut current: Option<String> = None;

pass.reconcile().reconcile_lemonade(None, &mut current);
pass.reconcile().reconcile_lemonade(None, &mut current);

assert!(pass.announced().is_empty());
assert_eq!(current, None);
assert!(pass.instance_ids().is_empty());
assert!(pass.scrape_targets(current.as_deref()).is_empty());
}

#[test]
fn lemonade_first_answer_is_announced_once_and_left_to_its_own_collector() {
let mut pass = Pass::new();
let mut current: Option<String> = None;

pass.reconcile()
.reconcile_lemonade(Some(lemonade_svc("127.0.0.1", 13305)), &mut current);

assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13305"]);
assert_eq!(current.as_deref(), Some("lemonade-127.0.0.1-13305"));
assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13305"]));
assert!(
pass.scrape_targets(current.as_deref()).is_empty(),
"the live Lemonade instance is not a vLLM scrape target"
);
}

#[test]
fn lemonade_same_endpoint_again_keeps_its_instance_and_live_fields() {
let svc = lemonade_svc("127.0.0.1", 13305);
let id = svc.container_id.clone();
let mut pass = Pass::new();
let mut current: Option<String> = None;
pass.reconcile()
.reconcile_lemonade(Some(svc.clone()), &mut current);
pass.give_live_telemetry(&id);
pass.announced();

pass.reconcile().reconcile_lemonade(Some(svc), &mut current);

assert!(pass.announced().is_empty());
assert_eq!(pass.instance_ids(), ids(&[&id]));
assert_eq!(pass.kv(&id), Some(25.0));
assert!(pass.has_telemetry(&id));
assert!(pass.scrape_targets(current.as_deref()).is_empty());
}

#[test]
fn lemonade_host_change_retires_the_old_instance_and_announces_the_new_one() {
let old = lemonade_svc("127.0.0.1", 13305);
let new = lemonade_svc("10.0.0.7", 13305);
let (pass, current, announced) = lemonade_identity_change(&old, &new);

assert_eq!(current.as_deref(), Some(new.container_id.as_str()));
assert_eq!(
announced,
[
"gone lemonade-127.0.0.1-13305",
"discovered lemonade-10.0.0.7-13305"
]
);
assert!(!pass.has_telemetry(&old.container_id));
assert_eq!(pass.instance_ids(), ids(&["lemonade-10.0.0.7-13305"]));
assert!(
pass.scrape_targets(current.as_deref()).is_empty(),
"the old endpoint must not be polled as a vLLM target"
);
}

#[test]
fn lemonade_port_change_retires_the_old_instance_and_announces_the_new_one() {
let old = lemonade_svc("127.0.0.1", 13305);
let new = lemonade_svc("127.0.0.1", 13306);
let (pass, current, announced) = lemonade_identity_change(&old, &new);

assert_eq!(current.as_deref(), Some(new.container_id.as_str()));
assert_eq!(
announced,
[
"gone lemonade-127.0.0.1-13305",
"discovered lemonade-127.0.0.1-13306"
]
);
assert!(!pass.has_telemetry(&old.container_id));
assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13306"]));
assert!(
pass.scrape_targets(current.as_deref()).is_empty(),
"the old endpoint must not be polled as a vLLM target"
);
}

#[test]
fn lemonade_endpoint_going_quiet_is_announced_gone_and_leaves_no_trace() {
let svc = lemonade_svc("127.0.0.1", 13305);
let id = svc.container_id.clone();
let mut pass = Pass::new();
let mut current: Option<String> = None;
pass.reconcile().reconcile_lemonade(Some(svc), &mut current);
pass.give_live_telemetry(&id);
pass.announced();

pass.reconcile().reconcile_lemonade(None, &mut current);

assert_eq!(pass.announced(), [format!("gone {id}")]);
assert_eq!(current, None);
assert!(pass.instance_ids().is_empty());
assert!(!pass.has_telemetry(&id));
assert!(pass.scrape_targets(current.as_deref()).is_empty());
}

#[test]
fn lemonade_coming_back_elsewhere_after_going_quiet_is_the_only_instance() {
let old = lemonade_svc("127.0.0.1", 13305);
let new = lemonade_svc("127.0.0.1", 13306);
let mut pass = Pass::new();
let mut current: Option<String> = None;
pass.reconcile()
.reconcile_lemonade(Some(old.clone()), &mut current);
pass.give_live_telemetry(&old.container_id);
pass.reconcile().reconcile_lemonade(None, &mut current);
assert_eq!(
pass.announced(),
[
"discovered lemonade-127.0.0.1-13305",
"gone lemonade-127.0.0.1-13305"
]
);

pass.reconcile()
.reconcile_lemonade(Some(new.clone()), &mut current);

assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13306"]);
assert_eq!(current.as_deref(), Some(new.container_id.as_str()));
assert_eq!(pass.instance_ids(), ids(&[&new.container_id]));
assert!(!pass.has_telemetry(&old.container_id));
assert!(pass.scrape_targets(current.as_deref()).is_empty());
}

#[test]
fn lemonade_answer_repeated_within_one_tick_is_announced_once() {
let svc = lemonade_svc("127.0.0.1", 13305);
let mut pass = Pass::new();
let mut current: Option<String> = None;

pass.reconcile()
.reconcile_lemonade(Some(svc.clone()), &mut current);
pass.reconcile().reconcile_lemonade(Some(svc), &mut current);

assert_eq!(pass.announced(), ["discovered lemonade-127.0.0.1-13305"]);
assert_eq!(pass.instance_ids(), ids(&["lemonade-127.0.0.1-13305"]));
assert!(pass.scrape_targets(current.as_deref()).is_empty());
}

#[test]
fn lemonade_endpoint_moving_away_and_back_is_retired_each_time() {
let a = lemonade_svc("127.0.0.1", 13305);
let b = lemonade_svc("127.0.0.1", 13306);
let mut pass = Pass::new();
let mut current: Option<String> = None;
pass.reconcile()
.reconcile_lemonade(Some(a.clone()), &mut current);
pass.announced();

pass.reconcile().reconcile_lemonade(Some(b), &mut current);
assert_eq!(
pass.announced(),
[
"gone lemonade-127.0.0.1-13305",
"discovered lemonade-127.0.0.1-13306"
]
);
pass.reconcile()
.reconcile_lemonade(Some(a.clone()), &mut current);

assert_eq!(
pass.announced(),
[
"gone lemonade-127.0.0.1-13306",
"discovered lemonade-127.0.0.1-13305"
]
);
assert_eq!(current.as_deref(), Some(a.container_id.as_str()));
assert_eq!(pass.instance_ids(), ids(&[&a.container_id]));
assert!(pass.scrape_targets(current.as_deref()).is_empty());
}

#[test]
fn reconcile_lemonade_follows_one_endpoint_through_its_lifecycle() {
let mut pass = Pass::new();
Expand All @@ -1981,18 +2266,16 @@ mod tests {
);
assert_eq!(pass.kv("l1"), Some(25.0), "live fields carry over");

// A new id (the host or port changed) purges the old id's telemetry and
// announces the new one.
// A new id (the host or port changed) retires the old id, telemetry
// included, and announces the new one.
pass.reconcile()
.reconcile_lemonade(Some(discovered("l2", 13306)), &mut current);
assert_eq!(pass.announced(), ["discovered l2"]);
assert_eq!(pass.announced(), ["gone l1", "discovered l2"]);
assert_eq!(current.as_deref(), Some("l2"));
assert!(!pass.telemetry.prev_ttft.contains_key("l1"));
// Pins current behaviour, not a goal: the superseded instance stays in
// state (and so in every snapshot) because only its telemetry is purged.
// Retiring it later is a deliberate behaviour change, and this assertion
// is the one to update when that happens.
assert!(pass.state.instances.contains_key("l1"));
// The superseded instance leaves the state (and so every snapshot), not
// only its telemetry.
assert!(!pass.state.instances.contains_key("l1"));

pass.reconcile().reconcile_lemonade(None, &mut current);
assert_eq!(pass.announced(), ["gone l2"]);
Expand Down
Loading