From 13bc462395ed7f1572209a36eb75837376c06ade Mon Sep 17 00:00:00 2001 From: Eugene Volen Date: Fri, 30 Oct 2026 11:00:29 +0200 Subject: [PATCH 1/4] feat(dash): hold then expire TTFT/TPOT instead of showing lifetime averages (ROCMAI-62) TTFT and TPOT fell back to the histogram's cumulative average whenever a scrape window saw no new requests, so an idle server kept showing its all-time latency indefinitely, with no freshness signal. Only windows with new requests now produce a value. The existing GenerationObservationTracker holds it (marked Held) for the observation validity window and then clears it. Instance gains ttft_observation and tpot_observation, serde-default and omitted when unknown, so legacy NDJSON replays still load and are never labelled Fresh. The instances table marks held latency with the shared `*` marker and legend. Signed-off-by: Eugene Volen --- crates/rocm-dash-core/src/metrics.rs | 33 +++ crates/rocm-dash-core/src/traits.rs | 2 + crates/rocm-dash-daemon/src/demo.rs | 2 + crates/rocm-dash-daemon/src/runner.rs | 256 ++++++++++++------ .../tests/telemetry_contract.rs | 83 ++++++ crates/rocm-dash-tui/src/ui/format.rs | 29 ++ crates/rocm-dash-tui/src/ui/tabs/instances.rs | 56 +++- tests/e2e-cucumber/tests/e2e/dash_steps.rs | 74 ++--- 8 files changed, 401 insertions(+), 134 deletions(-) diff --git a/crates/rocm-dash-core/src/metrics.rs b/crates/rocm-dash-core/src/metrics.rs index b81ef252c4..8686c4b5df 100644 --- a/crates/rocm-dash-core/src/metrics.rs +++ b/crates/rocm-dash-core/src/metrics.rs @@ -224,10 +224,20 @@ pub struct Instance { /// Observe shows `—`. Serde-default for NDJSON replay back-compat. #[serde(default)] pub ttft_ms: Option, + /// Freshness provenance for [`Instance::ttft_ms`]. A value with no new + /// requests in the latest scrape window is `Held` until the observation + /// validity window lapses, then cleared. `None` when unknown (legacy + /// snapshots). Omitted from JSON when `None`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ttft_observation: Option, /// Live average time-per-output-token (ms), windowed from the vLLM TPOT /// histogram. `None` until two scrapes (or absent); Observe shows `—`. #[serde(default)] pub tpot_ms: Option, + /// Freshness provenance for [`Instance::tpot_ms`]; same semantics as + /// [`Instance::ttft_observation`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tpot_observation: Option, pub launch_args: Vec, pub env_vars: std::collections::BTreeMap, pub log_file: Option, @@ -272,6 +282,29 @@ mod tests { assert_eq!(back.tpot_ms, Some(22.0)); } + #[test] + fn instance_latency_observation_is_omitted_when_unknown_and_round_trips() { + // Legacy payloads carry no freshness: it stays `None`, never `Fresh`, + // and is not written back out. + let json = serde_json::to_string(&Instance::default()).expect("serialize"); + assert!(!json.contains("ttft_observation") && !json.contains("tpot_observation")); + let inst = Instance { + ttft_ms: Some(100.0), + ttft_observation: Some(ObservationMetadata { + observed_at: Utc::now(), + freshness: ObservationFreshness::Held, + }), + ..Default::default() + }; + let back: Instance = + serde_json::from_str(&serde_json::to_string(&inst).expect("serialize")).expect("parse"); + assert_eq!( + back.ttft_observation.map(|m| m.freshness), + Some(ObservationFreshness::Held) + ); + assert!(back.tpot_observation.is_none()); + } + #[test] fn instance_status_ready_round_trips_as_lowercase_json() { // The new `Ready` variant must serialize/deserialize the same way the diff --git a/crates/rocm-dash-core/src/traits.rs b/crates/rocm-dash-core/src/traits.rs index 59105f5f03..b7dd893693 100644 --- a/crates/rocm-dash-core/src/traits.rs +++ b/crates/rocm-dash-core/src/traits.rs @@ -162,7 +162,9 @@ pub fn merge_instance( // Derived at the runner by windowing the sample's cumulative histogram // readings (mirrors gen_tps); left None at merge time. ttft_ms: None, + ttft_observation: None, tpot_ms: None, + tpot_observation: None, launch_args: svc.launch_args.clone(), env_vars: svc.env_vars.clone(), log_file: svc.log_file.clone(), diff --git a/crates/rocm-dash-daemon/src/demo.rs b/crates/rocm-dash-daemon/src/demo.rs index 2690aa4ef9..c9bbc3357d 100644 --- a/crates/rocm-dash-daemon/src/demo.rs +++ b/crates/rocm-dash-daemon/src/demo.rs @@ -629,7 +629,9 @@ fn build_instances(t_s: f64, seed: &mut u64) -> Vec { gen_tps_observation: None, tokens_per_watt: None, ttft_ms, + ttft_observation: None, tpot_ms, + tpot_observation: None, launch_args: c.launch_args.iter().map(|s| (*s).to_string()).collect(), env_vars, log_file: Some(format!("/var/log/vllm/{}.log", c.container_name)), diff --git a/crates/rocm-dash-daemon/src/runner.rs b/crates/rocm-dash-daemon/src/runner.rs index aeb554b940..12d1b1c830 100644 --- a/crates/rocm-dash-daemon/src/runner.rs +++ b/crates/rocm-dash-daemon/src/runner.rs @@ -25,7 +25,9 @@ use rocm_dash_collectors::host::HostCollector; use rocm_dash_collectors::lemonade::LemonadeCollector; use rocm_dash_collectors::parallel::parallel_scrape; use rocm_dash_collectors::vllm_prom::VllmPrometheusCollector; -use rocm_dash_core::metrics::{GpuMetrics, GpuSystemInfo, Instance, InstanceStatus, Snapshot}; +use rocm_dash_core::metrics::{ + GpuMetrics, GpuSystemInfo, Instance, InstanceStatus, ObservationMetadata, Snapshot, +}; use rocm_dash_core::observation::GenerationObservationTracker; use rocm_dash_core::protocol::Event; use rocm_dash_core::state::{State, StateEvent}; @@ -359,10 +361,10 @@ pub async fn run_loop( // Constructed lazily from opts.instance_tick; survives scrape failures so // the last-observed rate is held for the validity window before clearing. let mut gen_trackers: HashMap = HashMap::new(); - // Previous TTFT / TPOT histogram readings (sum_s, count, at) per instance, - // for the windowed avg-ms derivation (mirrors `prev_gen_tokens`). - let mut prev_ttft: HashMap)> = HashMap::new(); - let mut prev_tpot: HashMap)> = HashMap::new(); + // Previous TTFT / TPOT histogram readings plus a freshness tracker per + // instance, for the windowed avg-ms derivation (mirrors `gen_trackers`). + let mut prev_ttft: LatencyWindows = HashMap::new(); + let mut prev_tpot: LatencyWindows = HashMap::new(); // Per-container VRAM (MB) from the last amd-smi `process` scrape. Refreshed // on the instance cadence and reused every tick so the attributed value is // stable between scrapes (mirrors how GPU power drives tokens_per_watt). @@ -775,17 +777,21 @@ pub async fn run_loop( // assembly below (tracker.snapshot per cycle_at). // // Windowed avg latency (ms) from the cumulative TTFT/TPOT - // histograms; cumulative-average fallback on first scrape. - let ttft_ms = avg_ms_from_histogram( + // histograms. Only a window with new requests produces a + // value; the tracker holds it, then expires it, in the + // snapshot assembly below. + observe_latency( &mut prev_ttft, &id, + opts.instance_tick, sample.ttft_sum_s, sample.ttft_count, cycle_at, ); - let tpot_ms = avg_ms_from_histogram( + observe_latency( &mut prev_tpot, &id, + opts.instance_tick, sample.tpot_sum_s, sample.tpot_count, cycle_at, @@ -794,8 +800,6 @@ pub async fn run_loop( inst.kv_cache_usage_pct = sample.kv_cache_usage_pct; inst.running_reqs = sample.running_reqs; inst.waiting_reqs = sample.waiting_reqs; - inst.ttft_ms = ttft_ms; - inst.tpot_ms = tpot_ms; // Docker-discovered instances have no // `ManagedServiceRecord` to promote their status, so // they start life as `Starting`. A successful vLLM @@ -818,18 +822,11 @@ pub async fn run_loop( // window (clamp(3×instance_tick, 6s, 30s)) and only clears // it after expiry — satisfying the "held" contract. // - // Clear TTFT/TPOT baselines so recovery re-baselines from - // the next successful scrape (latency clearing semantics - // are unchanged; freshness generalization is out of scope). - prev_ttft.remove(&id); - prev_tpot.remove(&id); - if let Some(mut inst) = runner.state.instances.get(&id).cloned() - && (inst.ttft_ms.is_some() || inst.tpot_ms.is_some()) - { - inst.ttft_ms = None; - inst.tpot_ms = None; - runner.state.apply(StateEvent::InstanceUpserted(inst)); - } + // Drop the TTFT/TPOT baselines so recovery re-baselines + // from the next successful scrape; the trackers keep the + // last windowed value Held until its validity lapses. + rebaseline_latency(&mut prev_ttft, &id); + rebaseline_latency(&mut prev_tpot, &id); } } } @@ -921,6 +918,10 @@ pub async fn run_loop( .map_or((None, None), |t| t.snapshot(cycle_at)); inst.gen_tps = gen_tps; inst.gen_tps_observation = meta; + (inst.ttft_ms, inst.ttft_observation) = + snapshot_latency(&mut prev_ttft, &inst.container_id, cycle_at); + (inst.tpot_ms, inst.tpot_observation) = + snapshot_latency(&mut prev_tpot, &inst.container_id, cycle_at); } // Derive per-instance efficiency now that this tick's GPU power is known // and gen_tps reflects the tracker state for this cycle. @@ -952,6 +953,10 @@ pub async fn run_loop( if let Some(entry) = runner.state.instances.get_mut(&inst.container_id) { entry.gen_tps = inst.gen_tps; entry.gen_tps_observation = inst.gen_tps_observation.clone(); + entry.ttft_ms = inst.ttft_ms; + entry.ttft_observation = inst.ttft_observation.clone(); + entry.tpot_ms = inst.tpot_ms; + entry.tpot_observation = inst.tpot_observation.clone(); entry.tokens_per_watt = inst.tokens_per_watt; } } @@ -1023,8 +1028,8 @@ pub async fn run_loop( pub(crate) fn purge_telemetry_state( id: &str, gen_trackers: &mut HashMap, - prev_ttft: &mut HashMap)>, - prev_tpot: &mut HashMap)>, + prev_ttft: &mut LatencyWindows, + prev_tpot: &mut LatencyWindows, ) { gen_trackers.remove(id); prev_ttft.remove(id); @@ -1038,8 +1043,8 @@ pub(crate) fn purge_telemetry_state( pub(crate) fn invalidate_on_endpoint_change( id: &str, gen_trackers: &mut HashMap, - prev_ttft: &mut HashMap)>, - prev_tpot: &mut HashMap)>, + prev_ttft: &mut LatencyWindows, + prev_tpot: &mut LatencyWindows, ) { if let Some(t) = gen_trackers.get_mut(id) { t.invalidate(); @@ -1086,7 +1091,9 @@ pub(crate) fn merge_discovery_refresh( inst.gen_tps_observation = ex.gen_tps_observation.clone(); inst.tokens_per_watt = ex.tokens_per_watt; inst.ttft_ms = ex.ttft_ms; + inst.ttft_observation = ex.ttft_observation.clone(); inst.tpot_ms = ex.tpot_ms; + inst.tpot_observation = ex.tpot_observation.clone(); } inst } @@ -1137,46 +1144,78 @@ fn ticks_per(period: Duration, tick: Duration) -> u64 { n.max(1) as u64 } -/// Upper bound on the scrape interval accepted by `avg_ms_from_histogram`. +/// Upper bound on the scrape interval accepted by [`observe_latency`]. /// Applies to TTFT/TPOT latency histograms only; generation-counter staleness /// is managed by `GenerationObservationTracker` (see `rocm-dash-core`). -/// A gap longer than this makes the windowed delta stale (outage or forward -/// wall-clock jump / NTP / VM resume), so the histogram falls back to the -/// cumulative average for that sample. +/// A gap longer than this (outage, forward wall-clock jump / NTP / VM resume) +/// makes the windowed delta meaningless, so that sample yields no value and +/// only re-baselines. const MAX_RATE_INTERVAL_S: f64 = 60.0; -/// Average latency (ms) from a cumulative histogram's `_sum` (seconds) and -/// `_count` series, windowed over successive scrapes: `(Δsum / Δcount) × 1000`. -/// Updates the per-instance baseline in `prev`. On the FIRST reading (or after a -/// counter reset / stale interval / no new requests) it falls back to the -/// cumulative average `sum/count × 1000`, so a single scrape already yields a -/// value. Returns `None` when the histogram is absent (`sum`/`count` `None`) or -/// `count == 0` — Observe then shows `—` (never a fabricated number). -fn avg_ms_from_histogram( - prev: &mut HashMap)>, +/// Per-instance latency state for one histogram: the previous cumulative +/// reading and the tracker that holds and then expires the derived average. +pub(crate) struct LatencyWindow { + /// Previous `(sum_s, count, at)` reading; `None` until baselined. + prev: Option<(f64, f64, DateTime)>, + /// Holds the last windowed average for the observation validity window. + tracker: GenerationObservationTracker, +} + +pub(crate) type LatencyWindows = HashMap; + +/// Feed one scrape of a cumulative latency histogram (`_sum` in seconds, +/// `_count`) into `id`'s window. A value is recorded only when the window saw +/// new requests: `(Δsum / Δcount) × 1000` ms. The first reading, a counter +/// reset, a stale interval, or a window with no new requests only moves the +/// baseline; the lifetime average is never reported, so an idle server's old +/// latency ages out instead of persisting. An absent histogram (`None`) leaves +/// the window untouched. +fn observe_latency( + windows: &mut LatencyWindows, id: &str, + instance_tick: Duration, sum_s: Option, count: Option, now: DateTime, -) -> Option { - let (sum_s, count) = (sum_s?, count?); - let last = prev.insert(id.to_string(), (sum_s, count, now)); - if let Some((psum, pcount, pat)) = last { +) { + let (Some(sum_s), Some(count)) = (sum_s, count) else { + return; + }; + let w = windows + .entry(id.to_string()) + .or_insert_with(|| LatencyWindow { + prev: None, + tracker: GenerationObservationTracker::new(instance_tick), + }); + if let Some((psum, pcount, pat)) = w.prev.replace((sum_s, count, now)) { let dcount = count - pcount; let dsum = sum_s - psum; let dt = (now - pat).num_milliseconds() as f64 / 1000.0; if dcount > 0.0 && dsum >= 0.0 && dt > 0.0 && dt <= MAX_RATE_INTERVAL_S { - return Some(dsum / dcount * 1000.0); + w.tracker.observe_direct(dsum / dcount * 1000.0, now); } } - // Cumulative-average fallback: first scrape, no new requests, or a reset. - if count > 0.0 { - Some(sum_s / count * 1000.0) - } else { - None +} + +/// Forget the previous reading after a failed scrape so recovery re-baselines, +/// while the tracker keeps the last value held until it expires. +fn rebaseline_latency(windows: &mut LatencyWindows, id: &str) { + if let Some(w) = windows.get_mut(id) { + w.prev = None; } } +/// Current held/fresh/expired latency for `id` at `now`. +fn snapshot_latency( + windows: &mut LatencyWindows, + id: &str, + now: DateTime, +) -> (Option, Option) { + windows + .get_mut(id) + .map_or((None, None), |w| w.tracker.snapshot(now)) +} + #[cfg(test)] mod tests { use super::*; @@ -1340,36 +1379,95 @@ mod tests { Utc.timestamp_opt(secs, 0).unwrap() } + use rocm_dash_core::metrics::ObservationFreshness; + + const TICK: Duration = Duration::from_secs(2); + + fn baseline(sum_s: f64, count: f64) -> LatencyWindow { + LatencyWindow { + prev: Some((sum_s, count, at(10))), + tracker: GenerationObservationTracker::new(TICK), + } + } + + fn snap_at( + w: &mut LatencyWindows, + id: &str, + secs: i64, + ) -> (Option, Option) { + let (v, m) = snapshot_latency(w, id, at(secs)); + (v, m.map(|m| m.freshness)) + } + #[test] - fn avg_ms_first_scrape_uses_cumulative_then_windows() { - let mut prev = HashMap::new(); - // First scrape: cumulative average = 2.0s / 4 × 1000 = 500 ms. + fn latency_first_scrape_reports_nothing_then_windows() { + let mut w = LatencyWindows::new(); + // First scrape only baselines: the lifetime average is not reported. + observe_latency(&mut w, "a", TICK, Some(2.0), Some(4.0), at(10)); + assert_eq!(snap_at(&mut w, "a", 10), (None, None)); + // Second scrape: Δsum=1.0s over Δcount=10 → 100 ms, fresh. + observe_latency(&mut w, "a", TICK, Some(3.0), Some(14.0), at(12)); assert_eq!( - avg_ms_from_histogram(&mut prev, "a", Some(2.0), Some(4.0), at(10)), - Some(500.0) + snap_at(&mut w, "a", 12), + (Some(100.0), Some(ObservationFreshness::Fresh)) ); - // Second scrape: Δsum=1.0s over Δcount=10 → 100 ms (windowed). + } + + #[test] + fn latency_is_held_without_new_requests_then_expires() { + let mut w = LatencyWindows::new(); + observe_latency(&mut w, "a", TICK, Some(2.0), Some(4.0), at(10)); + observe_latency(&mut w, "a", TICK, Some(3.0), Some(14.0), at(12)); + // Idle scrapes: counters unchanged → no new value, previous is held. + observe_latency(&mut w, "a", TICK, Some(3.0), Some(14.0), at(14)); assert_eq!( - avg_ms_from_histogram(&mut prev, "a", Some(3.0), Some(14.0), at(12)), - Some(100.0) + snap_at(&mut w, "a", 14), + (Some(100.0), Some(ObservationFreshness::Held)) ); - // Absent histogram → None (Observe shows `—`). + // Validity for a 2 s tick is the 6 s floor: still held at +6 s... assert_eq!( - avg_ms_from_histogram(&mut prev, "b", None, None, at(12)), - None + snap_at(&mut w, "a", 18), + (Some(100.0), Some(ObservationFreshness::Held)) ); - // count == 0 (no requests yet) → None, never a divide-by-zero number. + // ...and gone just past it. The stale lifetime average never returns. + observe_latency(&mut w, "a", TICK, Some(3.0), Some(14.0), at(19)); + assert_eq!(snap_at(&mut w, "a", 19), (None, None)); + } + + #[test] + fn latency_absent_empty_reset_and_stale_interval_report_nothing() { + let mut w = LatencyWindows::new(); + // Absent histogram / zero count never fabricate a number. + observe_latency(&mut w, "b", TICK, None, None, at(12)); + assert_eq!(snap_at(&mut w, "b", 12), (None, None)); + observe_latency(&mut w, "c", TICK, Some(0.0), Some(0.0), at(12)); + observe_latency(&mut w, "c", TICK, Some(0.0), Some(0.0), at(14)); + assert_eq!(snap_at(&mut w, "c", 14), (None, None)); + // A counter reset (sum drops) only re-baselines. + observe_latency(&mut w, "d", TICK, Some(10.0), Some(100.0), at(10)); + observe_latency(&mut w, "d", TICK, Some(0.2), Some(2.0), at(12)); + assert_eq!(snap_at(&mut w, "d", 12), (None, None)); + // A gap beyond MAX_RATE_INTERVAL_S is not a usable window. + observe_latency(&mut w, "e", TICK, Some(1.0), Some(10.0), at(0)); + observe_latency(&mut w, "e", TICK, Some(2.0), Some(20.0), at(100)); + assert_eq!(snap_at(&mut w, "e", 100), (None, None)); + } + + #[test] + fn failed_scrape_rebaselines_but_keeps_value_held() { + let mut w = LatencyWindows::new(); + observe_latency(&mut w, "a", TICK, Some(2.0), Some(4.0), at(10)); + observe_latency(&mut w, "a", TICK, Some(3.0), Some(14.0), at(12)); + rebaseline_latency(&mut w, "a"); assert_eq!( - avg_ms_from_histogram(&mut prev, "c", Some(0.0), Some(0.0), at(12)), - None + snap_at(&mut w, "a", 14), + (Some(100.0), Some(ObservationFreshness::Held)) ); - // A counter reset (sum drops) re-bases to the cumulative average. - let mut p2 = HashMap::new(); - avg_ms_from_histogram(&mut p2, "d", Some(10.0), Some(100.0), at(10)); + // Recovery re-baselines: the first post-failure scrape yields nothing new. + observe_latency(&mut w, "a", TICK, Some(9.0), Some(50.0), at(15)); assert_eq!( - avg_ms_from_histogram(&mut p2, "d", Some(0.2), Some(2.0), at(12)), - Some(100.0), - "reset falls back to cumulative 0.2/2×1000" + snap_at(&mut w, "a", 15), + (Some(100.0), Some(ObservationFreshness::Held)) ); } @@ -1497,16 +1595,16 @@ mod tests { #[test] fn purge_telemetry_state_removes_all_three_maps_for_id() { let mut trackers: HashMap = HashMap::new(); - let mut ttft: HashMap)> = HashMap::new(); - let mut tpot: HashMap)> = HashMap::new(); + let mut ttft: LatencyWindows = HashMap::new(); + let mut tpot: LatencyWindows = HashMap::new(); // Populate two ids. trackers.insert("a".into(), make_tracker(2)); trackers.insert("b".into(), make_tracker(2)); - ttft.insert("a".into(), (1.0, 1.0, at(10))); - ttft.insert("b".into(), (2.0, 2.0, at(10))); - tpot.insert("a".into(), (3.0, 3.0, at(10))); - tpot.insert("b".into(), (4.0, 4.0, at(10))); + ttft.insert("a".into(), baseline(1.0, 1.0)); + ttft.insert("b".into(), baseline(2.0, 2.0)); + tpot.insert("a".into(), baseline(3.0, 3.0)); + tpot.insert("b".into(), baseline(4.0, 4.0)); purge_telemetry_state("a", &mut trackers, &mut ttft, &mut tpot); @@ -1526,16 +1624,16 @@ mod tests { #[test] fn invalidate_on_endpoint_change_resets_tracker_and_clears_latency() { let mut trackers: HashMap = HashMap::new(); - let mut ttft: HashMap)> = HashMap::new(); - let mut tpot: HashMap)> = HashMap::new(); + let mut ttft: LatencyWindows = HashMap::new(); + let mut tpot: LatencyWindows = HashMap::new(); let mut tracker = make_tracker(2); // Give the tracker a reading so it has a non-None baseline. tracker.observe_counter(100.0, at(10)); tracker.observe_counter(200.0, at(12)); trackers.insert("svc".into(), tracker); - ttft.insert("svc".into(), (1.0, 10.0, at(10))); - tpot.insert("svc".into(), (0.5, 8.0, at(10))); + ttft.insert("svc".into(), baseline(1.0, 10.0)); + tpot.insert("svc".into(), baseline(0.5, 8.0)); invalidate_on_endpoint_change("svc", &mut trackers, &mut ttft, &mut tpot); diff --git a/crates/rocm-dash-daemon/tests/telemetry_contract.rs b/crates/rocm-dash-daemon/tests/telemetry_contract.rs index 577ab6f8b5..2a9716c2a7 100644 --- a/crates/rocm-dash-daemon/tests/telemetry_contract.rs +++ b/crates/rocm-dash-daemon/tests/telemetry_contract.rs @@ -649,6 +649,89 @@ async fn gen_tps_held_for_validity_window_after_single_failure() { ); } +// ── TTFT freshness (ROCMAI-62) ────────────────────────────────────────────── + +/// Extract the `contract-svc` instance's TTFT and its freshness. +/// Outer `None` = instance absent. +#[allow(clippy::type_complexity)] +fn svc_ttft( + snap: &rocm_dash_core::metrics::Snapshot, +) -> Option<( + Option, + Option, +)> { + snap.instances + .iter() + .find(|i| i.container_id == SVC_ID) + .map(|i| (i.ttft_ms, i.ttft_observation.as_ref().map(|m| m.freshness))) +} + +/// Contract: once new requests stop arriving, the last windowed TTFT is held +/// (`Held`) for the observation validity window and then cleared. It is never +/// replaced by the histogram's lifetime average, which the mock's constant +/// `Frozen` body would otherwise report as a steady 50 ms indefinitely. +/// +/// Exercises the real runner loop: scrape, per-instance tracker, snapshot. +#[tokio::test] +async fn ttft_is_held_then_cleared_when_requests_stop() { + use rocm_dash_core::metrics::ObservationFreshness::{Fresh, Held}; + + let tmp = tempfile::TempDir::new().unwrap(); + let services_dir = tmp.path().join("services"); + + let mode = Arc::new(Mutex::new(MockMode::Growing)); + let ticks = Arc::new(AtomicU64::new(0)); + let stop = Arc::new(AtomicBool::new(false)); + let failure_count = Arc::new(AtomicU64::new(0)); + let port = spawn_mock_server( + Arc::clone(&mode), + Arc::clone(&ticks), + Arc::clone(&failure_count), + Arc::clone(&stop), + ) + .await; + + write_service_record(&services_dir, port); + let (tx, mut rx) = broadcast::channel::(64); + let _runner = spawn_runner(tx, fast_opts(services_dir)).await; + + // Step 1: traffic is flowing, so TTFT is a live (Fresh) windowed value. + let live = wait_for_snapshot(&mut rx, |snap| { + svc_ttft(snap).and_then(|(v, f)| (f == Some(Fresh)).then_some(v).flatten()) + }) + .await + .unwrap_or_else(|e| panic!("fresh TTFT never established: {e}")); + assert!(live > 0.0, "windowed TTFT should be positive, got {live}"); + + // Step 2: the engine goes idle: histograms stop advancing. + drain_snapshots(&mut rx); + *mode + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = MockMode::Frozen(1000); + + // Step 3: the value is held, marked as such, rather than dropped or + // silently presented as live. + let held = wait_for_snapshot(&mut rx, |snap| { + svc_ttft(snap).and_then(|(v, f)| (f == Some(Held)).then_some(v).flatten()) + }) + .await + .unwrap_or_else(|e| panic!("TTFT was never reported as Held once idle: {e}")); + assert!(held > 0.0); + + // Step 4: after the validity window it clears, and stays clear. The + // pre-fix behaviour would keep reporting the 50 ms lifetime average here. + let cleared = wait_for_snapshot(&mut rx, |snap| { + svc_ttft(snap).and_then(|(v, _)| v.is_none().then_some(())) + }) + .await; + stop.store(true, Ordering::Relaxed); + assert!( + cleared.is_ok(), + "idle TTFT was never cleared: stale latency persisted past the validity window ({:?})", + observation_validity_window() + ); +} + // ── Scenario 5: Omitted counter → None (not zero, not panic) ─────────────── /// Contract: when the `/metrics` endpoint returns HTTP 200 but the body omits diff --git a/crates/rocm-dash-tui/src/ui/format.rs b/crates/rocm-dash-tui/src/ui/format.rs index cd6dbaa62a..6eb0fcc1c5 100644 --- a/crates/rocm-dash-tui/src/ui/format.rs +++ b/crates/rocm-dash-tui/src/ui/format.rs @@ -245,6 +245,25 @@ pub fn gen_tps_cell(tps: Option, obs: Option<&ObservationMetadata>) -> Stri } } +/// Latency (TTFT/TPOT) for narrow table cells. +/// +/// - `None` / non-finite → `"—"` +/// - `Some(v)`, Held → `"{v:.0}ms*"` +/// - `Some(v)`, Fresh or unknown metadata → `"{v:.0}ms"` +pub fn latency_cell(ms: Option, obs: Option<&ObservationMetadata>) -> String { + match ms.filter(|v| v.is_finite()) { + None => "—".to_string(), + Some(v) => { + let base = format!("{v:.0}ms"); + if obs.is_some_and(|m| m.freshness == ObservationFreshness::Held) { + format!("{base}{HELD_MARKER}") + } else { + base + } + } + } +} + /// Full-unit gen_tps for pane rows and services views. /// Mirrors [`tps_opt`] format and appends [`HELD_MARKER`] when metadata is Held. /// @@ -609,6 +628,16 @@ mod tests { } } + #[test] + fn latency_cell_marks_held_and_never_prints_non_finite() { + assert_eq!(latency_cell(None, None), "—"); + assert_eq!(latency_cell(Some(f64::NAN), Some(&held_obs())), "—"); + assert_eq!(latency_cell(Some(150.4), Some(&held_obs())), "150ms*"); + assert_eq!(latency_cell(Some(150.4), Some(&fresh_obs())), "150ms"); + // Unknown freshness (legacy snapshot) is never shown as held. + assert_eq!(latency_cell(Some(150.4), None), "150ms"); + } + fn held_obs() -> ObservationMetadata { ObservationMetadata { observed_at: DateTime::from_timestamp(1_700_000_000, 0).unwrap(), diff --git a/crates/rocm-dash-tui/src/ui/tabs/instances.rs b/crates/rocm-dash-tui/src/ui/tabs/instances.rs index d549767119..86be1c6a5e 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/instances.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/instances.rs @@ -12,7 +12,9 @@ use ratatui::style::{Modifier, Style}; use ratatui::text::{Line, Span}; use ratatui::widgets::{Cell, Paragraph, Row, Table}; -use rocm_dash_core::metrics::{Instance, InstanceStatus, ObservationFreshness}; +use rocm_dash_core::metrics::{ + Instance, InstanceStatus, ObservationFreshness, ObservationMetadata, +}; use crate::app::{AppState, ConnState}; use crate::ui::format; @@ -80,8 +82,15 @@ pub fn draw_table(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { .gen_tps_observation .as_ref() .is_some_and(|m| m.freshness == ObservationFreshness::Held); - held && (inst.gen_tps.is_some_and(f64::is_finite) - || inst.tokens_per_watt.is_some_and(f64::is_finite)) + let is_held = |m: &Option| { + m.as_ref() + .is_some_and(|m| m.freshness == ObservationFreshness::Held) + }; + (held + && (inst.gen_tps.is_some_and(f64::is_finite) + || inst.tokens_per_watt.is_some_and(f64::is_finite))) + || (is_held(&inst.ttft_observation) && inst.ttft_ms.is_some_and(f64::is_finite)) + || (is_held(&inst.tpot_observation) && inst.tpot_ms.is_some_and(f64::is_finite)) }); let (table_area, legend_area) = if any_held { let split = Layout::default() @@ -108,14 +117,8 @@ pub fn draw_table(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { let tps = format::gen_tps_cell(inst.gen_tps, inst.gen_tps_observation.as_ref()); let tpw = format::tokens_per_watt_cell(inst.tokens_per_watt, inst.gen_tps_observation.as_ref()); - let ttft = inst - .ttft_ms - .filter(|v| v.is_finite()) - .map_or_else(|| dash.to_string(), |v| format!("{v:.0}ms")); - let tpot = inst - .tpot_ms - .filter(|v| v.is_finite()) - .map_or_else(|| dash.to_string(), |v| format!("{v:.0}ms")); + let ttft = format::latency_cell(inst.ttft_ms, inst.ttft_observation.as_ref()); + let tpot = format::latency_cell(inst.tpot_ms, inst.tpot_observation.as_ref()); let power = rocm_dash_core::efficiency::instance_power_w(&inst.gpu_ids, gpus) .map_or_else(|| dash.to_string(), |w| format!("{w:.0}W")); let queue = match (inst.running_reqs, inst.waiting_reqs) { @@ -243,7 +246,9 @@ pub(in crate::ui) mod tests { gen_tps_observation: None, tokens_per_watt: None, ttft_ms: None, + ttft_observation: None, tpot_ms: None, + tpot_observation: None, launch_args: vec![], env_vars: BTreeMap::new(), log_file: None, @@ -692,6 +697,35 @@ pub(in crate::ui) mod tests { ); } + #[test] + fn table_held_ttft_shows_marker_and_legend_even_with_fresh_gen_tps() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + + let mut inst = mk_inst_obs( + "idle", + Some(456.0), + Some(obs(ObservationFreshness::Fresh, 5)), + ); + inst.ttft_ms = Some(150.0); + inst.ttft_observation = Some(obs(ObservationFreshness::Held, 30)); + let state = state_with_snap(inst); + let mut term = Terminal::new(TestBackend::new(160, 20)).unwrap(); + term.draw(|f| draw_table(f, f.area(), &state, &state.theme)) + .unwrap(); + let out = buffer_text(&term); + assert!( + out.contains("150ms*"), + "held TTFT must be marked; got: +{out}" + ); + assert!( + out.contains(format::HELD_LEGEND), + "legend must explain a held TTFT marker; got: +{out}" + ); + } + #[test] fn table_held_legend_absent_when_all_fresh() { use ratatui::Terminal; diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index 759e68b102..f906992243 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -23,27 +23,6 @@ const MANAGED_MODEL_PROMPT: &str = "hello from the terminal"; /// it proves both that the overlay opened and - as an absence - that it closed. const SERVICES_OVERLAY_TITLE: &str = "Services — managed inference servers"; -/// The Observe instances table's TTFT cell while the scripted mock is serving: -/// its histogram pins time-to-first-token at exactly 50 ms -/// (`ttft_sum_s = ticks × 0.050` over `ttft_count = ticks`), and the cell is -/// rendered `"{v:.0}ms"`. A *failed* scrape clears `ttft_ms`/`tpot_ms` -/// (`runner.rs`), so this cell changing is the screen's own proof that the -/// frame on display was assembled after the failure — the only frame the -/// held-throughput assertion is about. -const SCRIPTED_TTFT_CELL: &str = "50ms"; - -/// Zero-based index of the TTFT cell within an Observe instances row, counting -/// from the model id: `MODEL TOK/S TOK/W TTFT TPOT POWER QUEUE KV%` -/// (`instances.rs`). -/// -/// The cell is read by position on the scripted instance's own row rather than -/// matched as a substring of the whole screen. A bare substring cannot tell a -/// cleared cell from a surviving one: any future ms-suffixed value that merely -/// *contains* the scripted one (`150ms`, `250ms`), or a second row whose TTFT -/// is also 50 ms, would keep the marker on screen and time this step out for a -/// reason that has nothing to do with the scrape it synchronises on. -const TTFT_COLUMN: usize = 3; - /// Borrow the scenario's active TUI session, or fail clearly if none was opened. const fn session(world: &mut E2eWorld) -> &mut TuiSession { world @@ -2409,19 +2388,27 @@ fn write_dash_clock(path: &std::path::Path, directive: &str) { std::fs::rename(&tmp, path).expect("failed to publish the dashboard test clock"); } -/// The TTFT cell `model`'s row is currently rendering, or `None` while that row -/// is not on screen at all. +/// The count on the header's `⚠ N` warnings badge, or 0 while no badge is drawn. /// -/// Cells are whitespace-separated and a model id carries no spaces, so counting -/// fields from the id yields one field per column — including the `—` -/// placeholder a cleared cell renders, which keeps the columns aligned. -fn scripted_ttft_cell<'a>(screen: &'a str, model: &str) -> Option<&'a str> { +/// The badge counts the daemon's current warnings, and a failed `/metrics` +/// scrape adds a `vllm scrape: N failed` warning to the snapshot it publishes. +/// A TTFT or throughput cell cannot serve as proof that a failure reached the +/// screen: those values are deliberately *held* through a failed scrape (and, +/// with the clock held, stay marked fresh), so the badge growing is the +/// visible evidence that a frame assembled after the failure is on display. +fn warning_badge_count(screen: &str) -> usize { screen - .lines() - .find(|line| line.contains(model))? - .split_whitespace() - .skip_while(|field| *field != model) - .nth(TTFT_COLUMN) + .split('⚠') + .nth(1) + .map(|after| { + after + .trim_start() + .chars() + .take_while(char::is_ascii_digit) + .collect::() + }) + .and_then(|digits| digits.parse().ok()) + .unwrap_or(0) } /// The Observe tab's node-throughput hero shows the "tok/s" unit whenever @@ -2465,12 +2452,17 @@ async fn dashboard_observation_time_is_held(world: &mut E2eWorld) { /// Switch the scripted mock to Failure mode and wait for the failure to reach /// the screen: first the mock's own counter proves the daemon was served a 503, -/// then the cleared TTFT cell proves the frame on display is one the daemon -/// assembled after that scrape. Both waits are synchronizations on observed -/// events, not fixed sleeps, and — the clock being held — neither can consume -/// the validity window they precede. +/// then the header warnings badge growing proves the frame on display is one +/// the daemon assembled after that scrape. Both waits are synchronizations on +/// observed events, not fixed sleeps, and — the clock being held — neither can +/// consume the validity window they precede. #[when("the metrics endpoint fails transiently")] async fn metrics_endpoint_fails(world: &mut E2eWorld) { + // Read the badge before the failure is injected: the count that matters is + // the one this failure adds, and other warnings (e.g. no amd-smi on a + // GPU-less runner) are already present and constant. + let before = warning_badge_count(&session(world).screen_text()); + let mock = world.mock.as_ref().expect("no mock server running"); mock.set_metrics_mode(MetricsMode::Failure); @@ -2490,16 +2482,10 @@ async fn metrics_endpoint_fails(world: &mut E2eWorld) { tokio::time::sleep(Duration::from_millis(100)).await; } - let model = world - .model_name - .clone() - .expect("the scripted-metrics Given records the model this row belongs to"); session(world) .wait_for_screen_where( - &format!("the scripted instance's TTFT cell leaves {SCRIPTED_TTFT_CELL:?}"), - |screen| { - scripted_ttft_cell(screen, &model).is_some_and(|cell| cell != SCRIPTED_TTFT_CELL) - }, + &format!("the header warnings badge grows past {before}"), + |screen| warning_badge_count(screen) > before, default_timeout(), ) .await From bbe0553160638c7d18c713046c4a7f5079e51415 Mon Sep 17 00:00:00 2001 From: Eugene Volen Date: Fri, 30 Oct 2026 12:39:23 +0200 Subject: [PATCH 2/4] feat(dash): split shared-GPU power among the instances on it (ROCMAI-62) Every instance was credited with the full board power of each GPU it touches, so two models on one GPU each claimed all of it: per-instance POWER overstated and tok/W understated, and the rows no longer summed to the board. attribute_power_w splits each GPU's power across its occupants in proportion to per-container VRAM (the per-process amd-smi attribution), spreads a multi-GPU instance's VRAM evenly over its GPUs, and falls back to an equal split when any occupant has no reading rather than guessing a skew. Power on a GPU nobody occupies is attributed to no instance but still counts toward node power. The function takes only snapshot data, so it adds no amd-smi dependency and works the same for any source that fills GpuMetrics and per-instance VRAM. Instance gains power_w (the attributed share, serde-default). The runner and demo use it for tokens_per_watt, and the instances table shows it in the POWER column. Signed-off-by: Eugene Volen --- crates/rocm-dash-core/src/efficiency.rs | 237 +++++++++++------- crates/rocm-dash-core/src/metrics.rs | 5 + crates/rocm-dash-core/src/traits.rs | 1 + crates/rocm-dash-daemon/src/demo.rs | 12 +- crates/rocm-dash-daemon/src/registry.rs | 31 ++- crates/rocm-dash-daemon/src/runner.rs | 11 +- .../tests/telemetry_contract.rs | 61 +++++ crates/rocm-dash-tui/src/ui/tabs/instances.rs | 40 ++- 8 files changed, 301 insertions(+), 97 deletions(-) diff --git a/crates/rocm-dash-core/src/efficiency.rs b/crates/rocm-dash-core/src/efficiency.rs index 072b022639..6781c86024 100644 --- a/crates/rocm-dash-core/src/efficiency.rs +++ b/crates/rocm-dash-core/src/efficiency.rs @@ -7,7 +7,10 @@ //! This is the join that turns vLLM throughput + amd-smi power into a //! per-instance capacity number. See `../wiki/concepts/metric-registry.md`. -use crate::metrics::GpuMetrics; +use std::collections::HashMap; +use std::hash::BuildHasher; + +use crate::metrics::{GpuMetrics, Instance}; /// Normalize a GPU identifier to a bare index for joining. /// @@ -34,44 +37,82 @@ pub fn gpu_ids_overlap(gpu_ids: &[String], gpus: &[GpuMetrics]) -> bool { gpus.iter().any(|g| device_in(&g.device_id, gpu_ids)) } -/// Summed board power (W) of the GPUs one instance occupies. +/// Board power (W) attributed to each instance, index-aligned with `instances`. +/// +/// A GPU's power is split among the instances that occupy it, so the instance +/// figures never add up to more than the board reports. Two instances sharing +/// one GPU each get a share, not the whole board. +/// +/// The split is proportional to each occupant's VRAM from +/// `per_container_vram_mb` (the per-process `amd-smi` attribution). A +/// multi-GPU instance's VRAM is spread evenly over its GPUs. If any occupant of +/// a GPU has no per-container reading the split on that GPU is equal, rather +/// than guessing a skew from partial data. A sole occupant gets the whole GPU. /// -/// Uses the same id-normalizing join as [`tokens_per_watt`]. `None` when no -/// matched GPU reports power (id join found nothing / amd-smi unavailable) so -/// the Observe table shows `—` rather than a fabricated `0 W`. +/// Power on a GPU no instance occupies (idle, or a process outside the managed +/// set) is attributed to nobody; it still counts toward node power. An entry is +/// `None` when none of its GPUs matched a GPU reporting power (id join found +/// nothing, power telemetry unavailable), so the UI shows `—` instead of a +/// fabricated `0 W`. #[must_use] -pub fn instance_power_w(gpu_ids: &[String], gpus: &[GpuMetrics]) -> Option { - let total: f64 = gpus - .iter() - .filter(|g| device_in(&g.device_id, gpu_ids)) - .map(|g| f64::from(g.power_w)) - .sum(); - (total > 0.0).then_some(total) +pub fn attribute_power_w( + instances: &[Instance], + gpus: &[GpuMetrics], + per_container_vram_mb: &HashMap, +) -> Vec> { + let mut watts = vec![0.0_f64; instances.len()]; + let mut matched = vec![false; instances.len()]; + for gpu in gpus { + let power = f64::from(gpu.power_w); + if !power.is_finite() || power <= 0.0 { + continue; + } + let occupants: Vec = instances + .iter() + .enumerate() + .filter(|(_, i)| device_in(&gpu.device_id, &i.gpu_ids)) + .map(|(k, _)| k) + .collect(); + if occupants.is_empty() { + continue; + } + let weights: Option> = occupants + .iter() + .map(|&k| { + let inst = &instances[k]; + per_container_vram_mb + .get(&inst.container_id) + .filter(|&&mb| mb > 0) + .map(|&mb| mb as f64 / inst.gpu_ids.len().max(1) as f64) + }) + .collect(); + let weight_sum = weights.as_ref().map(|w| w.iter().sum::()); + for (pos, &k) in occupants.iter().enumerate() { + let share = match (&weights, weight_sum) { + (Some(w), Some(sum)) if sum > 0.0 => w[pos] / sum, + _ => 1.0 / occupants.len() as f64, + }; + watts[k] = power.mul_add(share, watts[k]); + matched[k] = true; + } + } + watts + .into_iter() + .zip(matched) + .map(|(w, m)| m.then_some(w)) + .collect() } /// Tokens-per-watt for one serving instance: generation throughput (tok/s) -/// divided by the summed power (W) of the GPUs it occupies. +/// divided by the power (W) attributed to it by [`attribute_power_w`]. /// -/// Returns `None` when throughput is unknown (`gen_tps` is `None`) or the -/// matched GPUs report no power (e.g. amd-smi unavailable, or the id join -/// found nothing) — never a divide-by-zero or a negative number. A zero -/// throughput with live GPUs is a real `Some(0.0)`, not `None`. -pub fn tokens_per_watt( - gen_tps: Option, - gpu_ids: &[String], - gpus: &[GpuMetrics], -) -> Option { - let tps = gen_tps?; - let total_w: f64 = gpus - .iter() - .filter(|g| device_in(&g.device_id, gpu_ids)) - .map(|g| f64::from(g.power_w)) - .sum(); - if total_w > 0.0 { - Some(tps / total_w) - } else { - None - } +/// Returns `None` when throughput or attributed power is unknown, or power is +/// not positive — never a divide-by-zero or a negative number. A zero +/// throughput with live power is a real `Some(0.0)`, not `None`. +#[must_use] +pub fn tokens_per_watt(gen_tps: Option, power_w: Option) -> Option { + let (tps, power) = (gen_tps?, power_w?); + (power > 0.0).then(|| tps / power) } #[cfg(test)] @@ -94,60 +135,112 @@ mod tests { assert_eq!(normalize_gpu_id("3"), "3"); } + fn inst(id: &str, gpu_ids: &[&str]) -> Instance { + Instance { + container_id: id.into(), + gpu_ids: gpu_ids.iter().map(|s| (*s).to_string()).collect(), + ..Instance::default() + } + } + + fn vram(entries: &[(&str, u64)]) -> HashMap { + entries + .iter() + .map(|(k, v)| ((*k).to_string(), *v)) + .collect() + } + #[test] - fn joins_bare_index_to_prefixed_device_id() { + fn sole_occupant_gets_whole_board_via_bare_index_join() { // The crux: instance "0" must match amd-smi "gpu-0". - let gpus = [gpu("gpu-0", 250.0)]; - let r = tokens_per_watt(Some(500.0), &["0".into()], &gpus); - assert_eq!(r, Some(2.0)); + let p = attribute_power_w(&[inst("a", &["0"])], &[gpu("gpu-0", 250.0)], &vram(&[])); + assert_eq!(p, vec![Some(250.0)]); } #[test] - fn sums_power_across_tensor_parallel_gpus() { + fn tensor_parallel_instance_sums_its_gpus_and_ignores_others() { let gpus = [ gpu("gpu-0", 200.0), gpu("gpu-1", 300.0), gpu("gpu-2", 999.0), ]; - // Instance occupies gpus 0 and 1 → 500 W; gpu-2 excluded. - let r = tokens_per_watt(Some(1000.0), &["0".into(), "1".into()], &gpus); - assert_eq!(r, Some(2.0)); + let p = attribute_power_w(&[inst("a", &["0", "1"])], &gpus, &vram(&[])); + assert_eq!(p, vec![Some(500.0)]); } #[test] - fn none_throughput_yields_none() { - let gpus = [gpu("gpu-0", 250.0)]; - assert_eq!(tokens_per_watt(None, &["0".into()], &gpus), None); + fn shared_gpu_splits_by_vram_and_conserves_board_power() { + let gpus = [gpu("gpu-0", 400.0)]; + let insts = [inst("a", &["0"]), inst("b", &["0"])]; + let p = attribute_power_w(&insts, &gpus, &vram(&[("a", 30_000), ("b", 10_000)])); + assert_eq!(p, vec![Some(300.0), Some(100.0)]); + let total: f64 = p.iter().flatten().sum(); + assert!((total - 400.0).abs() < 1e-9, "must not exceed board power"); } #[test] - fn zero_total_power_yields_none() { - let gpus = [gpu("gpu-0", 0.0)]; - assert_eq!(tokens_per_watt(Some(500.0), &["0".into()], &gpus), None); + fn shared_gpu_splits_equally_without_complete_vram() { + let gpus = [gpu("gpu-0", 400.0)]; + let insts = [inst("a", &["0"]), inst("b", &["0"])]; + // No readings, or one occupant missing: equal split, no guessed skew. + for v in [ + vram(&[]), + vram(&[("a", 30_000)]), + vram(&[("a", 30_000), ("b", 0)]), + ] { + assert_eq!( + attribute_power_w(&insts, &gpus, &v), + vec![Some(200.0), Some(200.0)] + ); + } } #[test] - fn unmatched_ids_yield_none() { - let gpus = [gpu("gpu-5", 250.0)]; - assert_eq!(tokens_per_watt(Some(500.0), &["0".into()], &gpus), None); + fn multi_gpu_instance_vram_is_spread_across_its_gpus() { + // `a` spans gpu 0+1 (40 GB total -> 20 GB each); `b` is alone on gpu 1 + // with 20 GB. gpu-0 is all `a`; gpu-1 is split 50/50. + let gpus = [gpu("gpu-0", 100.0), gpu("gpu-1", 100.0)]; + let insts = [inst("a", &["0", "1"]), inst("b", &["1"])]; + let p = attribute_power_w(&insts, &gpus, &vram(&[("a", 40_000), ("b", 20_000)])); + assert_eq!(p, vec![Some(150.0), Some(50.0)]); } #[test] - fn empty_gpu_ids_yield_none() { - let gpus = [gpu("gpu-0", 250.0)]; - assert_eq!(tokens_per_watt(Some(500.0), &[], &gpus), None); + fn unoccupied_gpu_power_is_attributed_to_nobody() { + let gpus = [gpu("gpu-0", 100.0), gpu("gpu-1", 500.0)]; + let p = attribute_power_w(&[inst("a", &["0"])], &gpus, &vram(&[])); + assert_eq!(p, vec![Some(100.0)]); } #[test] - fn no_gpu_telemetry_yields_none() { - assert_eq!(tokens_per_watt(Some(500.0), &["0".into()], &[]), None); + fn unmatched_empty_or_unpowered_yield_none() { + let v = vram(&[]); + let a0 = [inst("a", &["0"])]; + assert_eq!( + attribute_power_w(&a0, &[gpu("gpu-5", 250.0)], &v), + vec![None] + ); + assert_eq!( + attribute_power_w(&[inst("a", &[])], &[gpu("gpu-0", 250.0)], &v), + vec![None] + ); + assert_eq!(attribute_power_w(&a0, &[], &v), vec![None]); + // Zero / NaN power means "no telemetry" (e.g. Windows without ADLX). + assert_eq!(attribute_power_w(&a0, &[gpu("gpu-0", 0.0)], &v), vec![None]); + assert_eq!( + attribute_power_w(&a0, &[gpu("gpu-0", f32::NAN)], &v), + vec![None] + ); } #[test] - fn zero_throughput_with_live_gpus_is_some_zero() { - // Idle instance, GPUs powered: 0 tok/W is a real reading, not "-". - let gpus = [gpu("gpu-0", 250.0)]; - assert_eq!(tokens_per_watt(Some(0.0), &["0".into()], &gpus), Some(0.0)); + fn tokens_per_watt_divides_and_guards() { + assert_eq!(tokens_per_watt(Some(500.0), Some(250.0)), Some(2.0)); + assert_eq!(tokens_per_watt(None, Some(250.0)), None); + assert_eq!(tokens_per_watt(Some(500.0), None), None); + assert_eq!(tokens_per_watt(Some(500.0), Some(0.0)), None); + // Idle instance, powered GPU: 0 tok/W is a real reading, not "-". + assert_eq!(tokens_per_watt(Some(0.0), Some(250.0)), Some(0.0)); } #[test] @@ -158,30 +251,4 @@ mod tests { assert!(!gpu_ids_overlap(&[], &gpus)); assert!(!gpu_ids_overlap(&["0".into()], &[])); } - - #[test] - fn instance_power_sums_matching_gpus() { - let gpus = [ - gpu("gpu-0", 200.0), - gpu("gpu-1", 300.0), - gpu("gpu-2", 999.0), - ]; - // Instance occupies gpus 0 and 1 → 500 W; gpu-2 is excluded. - assert_eq!( - instance_power_w(&["0".into(), "1".into()], &gpus), - Some(500.0) - ); - } - - #[test] - fn instance_power_no_match_yields_none() { - let gpus = [gpu("gpu-5", 250.0)]; - assert_eq!(instance_power_w(&["0".into()], &gpus), None); - } - - #[test] - fn instance_power_empty_ids_yields_none() { - let gpus = [gpu("gpu-0", 250.0)]; - assert_eq!(instance_power_w(&[], &gpus), None); - } } diff --git a/crates/rocm-dash-core/src/metrics.rs b/crates/rocm-dash-core/src/metrics.rs index 8686c4b5df..78dd44743e 100644 --- a/crates/rocm-dash-core/src/metrics.rs +++ b/crates/rocm-dash-core/src/metrics.rs @@ -219,6 +219,11 @@ pub struct Instance { /// occupies. `None` when throughput or GPU power telemetry is unavailable. #[serde(default)] pub tokens_per_watt: Option, + /// Board power (W) attributed to this instance: each GPU's power split among + /// the instances sharing it (see `efficiency::attribute_power_w`). `None` + /// when no occupied GPU reports power. Serde-default for replay back-compat. + #[serde(default)] + pub power_w: Option, /// Live average time-to-first-token (ms), windowed from the vLLM TTFT /// histogram. `None` until two scrapes (or absent for non-vLLM engines); /// Observe shows `—`. Serde-default for NDJSON replay back-compat. diff --git a/crates/rocm-dash-core/src/traits.rs b/crates/rocm-dash-core/src/traits.rs index b7dd893693..3719b91ec8 100644 --- a/crates/rocm-dash-core/src/traits.rs +++ b/crates/rocm-dash-core/src/traits.rs @@ -159,6 +159,7 @@ pub fn merge_instance( // a counter-rate delta is computed. gen_tps_observation: None, tokens_per_watt: None, + power_w: None, // Derived at the runner by windowing the sample's cumulative histogram // readings (mirrors gen_tps); left None at merge time. ttft_ms: None, diff --git a/crates/rocm-dash-daemon/src/demo.rs b/crates/rocm-dash-daemon/src/demo.rs index c9bbc3357d..4ed555f256 100644 --- a/crates/rocm-dash-daemon/src/demo.rs +++ b/crates/rocm-dash-daemon/src/demo.rs @@ -628,6 +628,7 @@ fn build_instances(t_s: f64, seed: &mut u64) -> Vec { gen_tps: Some(gen_tps), gen_tps_observation: None, tokens_per_watt: None, + power_w: None, ttft_ms, ttft_observation: None, tpot_ms, @@ -646,9 +647,14 @@ fn build_snapshot(t_s: f64, base_secs: i64, seed: &mut u64) -> Snapshot { let mut instances = build_instances(t_s, seed); // Mirror the runner: derive tokens_per_watt once both throughput and GPU // power are in hand, exercising the same id-normalizing join. - for inst in &mut instances { - inst.tokens_per_watt = - rocm_dash_core::efficiency::tokens_per_watt(inst.gen_tps, &inst.gpu_ids, &gpus); + let attributed = rocm_dash_core::efficiency::attribute_power_w( + &instances, + &gpus, + &std::collections::HashMap::new(), + ); + for (inst, power_w) in instances.iter_mut().zip(attributed) { + inst.power_w = power_w; + inst.tokens_per_watt = rocm_dash_core::efficiency::tokens_per_watt(inst.gen_tps, power_w); } Snapshot { timestamp, diff --git a/crates/rocm-dash-daemon/src/registry.rs b/crates/rocm-dash-daemon/src/registry.rs index 6f02aa8f4c..44e6275fb5 100644 --- a/crates/rocm-dash-daemon/src/registry.rs +++ b/crates/rocm-dash-daemon/src/registry.rs @@ -65,6 +65,11 @@ pub struct ServiceRecord { /// Absent on older records and once the service reaches `ready`. #[serde(default)] pub startup_phase: Option, + /// GPU indices the service was launched on (`rocm serve --gpu`), empty when + /// none were recorded. These become the instance's `gpu_ids`, which is what + /// joins it to GPU power and VRAM. + #[serde(default)] + pub gpu_indices: Vec, #[serde(default)] pub created_at_unix_ms: u128, } @@ -144,10 +149,9 @@ pub fn load_service_records(services_dir: &Path) -> Vec { /// collector pipeline, or `None` when the record is not in a scrapeable state. /// /// The port is taken verbatim from the registry (`record.port`) — the registry -/// is the authority, never a hardcoded default. `gpu_ids` is left empty because -/// the record carries no concrete device ids; VRAM and tokens/W attribution -/// degrade gracefully (device-summed / `None`) for managed services until a -/// richer device mapping is recorded. +/// is the authority, never a hardcoded default. `gpu_ids` come from the +/// record's `gpu_indices`; when none were recorded they stay empty and VRAM and +/// tokens/W attribution degrade gracefully (device-summed / `None`). pub fn discovered_from_record(record: &ServiceRecord) -> Option { if !is_scrapeable_status(&record.status) { return None; @@ -168,6 +172,7 @@ pub fn discovered_from_record(record: &ServiceRecord) -> Option(64); + let opts = RunnerOptions { + amd_smi_binary: Some(fake.into_os_string()), + amd_smi_skip_device_preflight: true, + ..fast_opts(services_dir) + }; + let _runner = spawn_runner(tx, opts).await; + + let (a, b) = wait_for_snapshot(&mut rx, |snap| { + let power = |id: &str| { + snap.instances + .iter() + .find(|i| i.container_id == id) + .and_then(|i| i.power_w) + }; + Some((power("shared-a")?, power("shared-b")?)) + }) + .await + .unwrap_or_else(|e| panic!("attributed power never appeared: {e}")); + + // No per-process VRAM was reported, so the board is split equally and the + // two shares add back up to what the GPU actually draws. + assert!((a - 200.0).abs() < 1e-6, "shared-a got {a} W"); + assert!((b - 200.0).abs() < 1e-6, "shared-b got {b} W"); +} + // ── Scenario 5: Omitted counter → None (not zero, not panic) ─────────────── /// Contract: when the `/metrics` endpoint returns HTTP 200 but the body omits diff --git a/crates/rocm-dash-tui/src/ui/tabs/instances.rs b/crates/rocm-dash-tui/src/ui/tabs/instances.rs index 86be1c6a5e..6538ef6ded 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/instances.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/instances.rs @@ -35,8 +35,6 @@ pub fn draw_table(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { } let instances = sorted_instances(&state.instances); let sel = clamp_sel(state.instance_sel, instances.len()); - let gpus: &[rocm_dash_core::metrics::GpuMetrics] = - state.latest.as_ref().map_or(&[], |s| s.gpus.as_slice()); let inner = panel::bento( f, @@ -119,7 +117,9 @@ pub fn draw_table(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { format::tokens_per_watt_cell(inst.tokens_per_watt, inst.gen_tps_observation.as_ref()); let ttft = format::latency_cell(inst.ttft_ms, inst.ttft_observation.as_ref()); let tpot = format::latency_cell(inst.tpot_ms, inst.tpot_observation.as_ref()); - let power = rocm_dash_core::efficiency::instance_power_w(&inst.gpu_ids, gpus) + let power = inst + .power_w + .filter(|w| w.is_finite()) .map_or_else(|| dash.to_string(), |w| format!("{w:.0}W")); let queue = match (inst.running_reqs, inst.waiting_reqs) { (None, None) => dash.to_string(), @@ -245,6 +245,7 @@ pub(in crate::ui) mod tests { gen_tps: None, gen_tps_observation: None, tokens_per_watt: None, + power_w: None, ttft_ms: None, ttft_observation: None, tpot_ms: None, @@ -726,6 +727,39 @@ pub(in crate::ui) mod tests { ); } + #[test] + fn table_power_shows_attributed_share_not_board_power() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + + // The GPU board may report far more than this instance's share; the + // POWER column must show the attributed figure carried on the instance. + let mut inst = mk_inst_obs("shared", Some(10.0), None); + inst.power_w = Some(123.0); + let mut state = state_with_snap(inst); + if let Some(snap) = state.latest.as_mut() { + snap.gpus = vec![rocm_dash_core::metrics::GpuMetrics { + device_id: "gpu-0".into(), + power_w: 999.0, + ..Default::default() + }]; + } + let mut term = Terminal::new(TestBackend::new(160, 20)).unwrap(); + term.draw(|f| draw_table(f, f.area(), &state, &state.theme)) + .unwrap(); + let out = buffer_text(&term); + assert!( + out.contains("123W"), + "attributed power missing; got: +{out}" + ); + assert!( + !out.contains("999W"), + "board power leaked into row; got: +{out}" + ); + } + #[test] fn table_held_legend_absent_when_all_fresh() { use ratatui::Terminal; From f08f974adaf17728643fa9ec446b2e5098f96bab Mon Sep 17 00:00:00 2001 From: Eugene Volen Date: Fri, 30 Oct 2026 12:43:30 +0200 Subject: [PATCH 3/4] feat(dash): report latency freshness and attributed power to the assistant (ROCMAI-62) The assistant's list_instances tool carried freshness only for gen_tps, so it could present a held TTFT as live and had no per-instance power. Add ttft_ms/tpot_ms with their freshness ("fresh", "held", or null when unknown) and the attributed power_w, matching what the instances table shows. Signed-off-by: Eugene Volen --- crates/rocm-dash-tui/src/agent/snapshot.rs | 48 +++++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/crates/rocm-dash-tui/src/agent/snapshot.rs b/crates/rocm-dash-tui/src/agent/snapshot.rs index 5e3ce15c17..d72f4dd74f 100644 --- a/crates/rocm-dash-tui/src/agent/snapshot.rs +++ b/crates/rocm-dash-tui/src/agent/snapshot.rs @@ -10,7 +10,7 @@ use serde_json::{Value, json}; -use rocm_dash_core::metrics::{GpuMetrics, ObservationFreshness}; +use rocm_dash_core::metrics::{GpuMetrics, ObservationFreshness, ObservationMetadata}; use super::StateSnapshot; @@ -42,6 +42,19 @@ pub fn gpu_status_json(snap: &StateSnapshot, gpu_index: Option) -> Value } } +/// `"fresh"` | `"held"` | `null` (unknown, e.g. a legacy snapshot). +fn freshness_label(obs: Option<&ObservationMetadata>) -> Value { + obs.map_or(Value::Null, |m| { + Value::String( + match m.freshness { + ObservationFreshness::Fresh => "fresh", + ObservationFreshness::Held => "held", + } + .into(), + ) + }) +} + /// Discovered serving instances with gen_tps, daemon-provided tok/W, /// freshness metadata, and observed_at timestamp. /// @@ -77,6 +90,11 @@ pub fn list_instances_json(snap: &StateSnapshot) -> Value { "waiting_reqs": i.waiting_reqs, "gen_tps": i.gen_tps, "tokens_per_watt": i.tokens_per_watt, + "power_w": i.power_w, + "ttft_ms": i.ttft_ms, + "ttft_freshness": freshness_label(i.ttft_observation.as_ref()), + "tpot_ms": i.tpot_ms, + "tpot_freshness": freshness_label(i.tpot_observation.as_ref()), "freshness": freshness, "observed_at": observed_at, }) @@ -239,6 +257,34 @@ mod tests { ); } + #[test] + fn list_instances_json_reports_latency_freshness_and_attributed_power() { + use chrono::Utc; + use rocm_dash_core::metrics::ObservationMetadata; + let held = ObservationMetadata { + observed_at: Utc::now(), + freshness: ObservationFreshness::Held, + }; + let inst = Instance { + container_name: "vllm-lat".into(), + ttft_ms: Some(150.0), + ttft_observation: Some(held), + tpot_ms: Some(22.0), + power_w: Some(123.0), + ..Default::default() + }; + let mut snap = fixture_snapshot(); + snap.instances = vec![inst]; + let v = list_instances_json(&snap); + let i = &v["instances"][0]; + assert_eq!(i["ttft_ms"], 150.0); + assert_eq!(i["ttft_freshness"], "held"); + // No metadata is unknown, never reported as fresh. + assert_eq!(i["tpot_ms"], 22.0); + assert!(i["tpot_freshness"].is_null()); + assert_eq!(i["power_w"], 123.0); + } + #[test] fn list_instances_json_held_freshness_field() { let snap = fixture_with_observation(rocm_dash_core::metrics::ObservationFreshness::Held); From 3ac884e571f82299b019d20d0b36cf83c9aae4ba Mon Sep 17 00:00:00 2001 From: Eugene Volen Date: Fri, 30 Oct 2026 12:52:22 +0200 Subject: [PATCH 4/4] feat(dash): derive GPU power warning thresholds from the device's power limit (ROCMAI-62) Board-power colors compared against fixed 525/700 W constants tuned for MI355X-class parts, so the same reading looked alarming or fine depending on which GPU produced it. GpuMetrics gains power_limit_w (serde-default; None means unknown, never 0). The daemon reads each GPU's cap from `amd-smi static` (the cap currently in force, else the part's maximum) on the existing 30 s system-info cadence and stamps it onto every tick's metrics; a failed fetch keeps the previous caps. power_style now warns at 70% and goes critical at 93% of that limit, which is where the old constants sat on a 750 W part. With no known limit the value is drawn in the plain foreground color instead of being judged against some other part's TDP. Platforms with no limit source (Windows today) therefore render uncolored rather than wrongly colored. The amd-smi `limit` schema is parsed defensively and an unrecognised shape yields no limit; it has not yet been checked against a real MI-series host. Signed-off-by: Eugene Volen --- crates/rocm-dash-collectors/src/amd_smi.rs | 80 ++++++++++++++ crates/rocm-dash-collectors/src/bench_load.rs | 7 +- crates/rocm-dash-collectors/src/cgroup.rs | 20 ++++ crates/rocm-dash-collectors/src/winperf.rs | 3 + crates/rocm-dash-core/src/metrics.rs | 32 +++++- crates/rocm-dash-daemon/src/demo.rs | 2 + crates/rocm-dash-daemon/src/runner.rs | 104 ++++++++++++++++-- .../tests/telemetry_contract.rs | 36 ++++-- crates/rocm-dash-tui/src/agent/snapshot.rs | 39 ++++++- crates/rocm-dash-tui/src/agent/tools.rs | 9 +- crates/rocm-dash-tui/src/ui/tabs/hardware.rs | 15 ++- crates/rocm-dash-tui/src/ui/tabs/home.rs | 5 + crates/rocm-dash-tui/src/ui/tabs/instances.rs | 35 +++++- crates/rocm-dash-tui/src/ui/tabs/observe.rs | 1 + crates/rocm-dash-tui/src/ui/widgets.rs | 100 +++++++++++++---- .../tests/dash_characterization.rs | 1 + crates/rocm-dash-tui/tests/dash_journeys.rs | 1 + tests/e2e-cucumber/src/mock_server.rs | 2 +- tests/e2e-cucumber/tests/e2e/dash_steps.rs | 4 +- 19 files changed, 432 insertions(+), 64 deletions(-) diff --git a/crates/rocm-dash-collectors/src/amd_smi.rs b/crates/rocm-dash-collectors/src/amd_smi.rs index 03a8ffab22..5c36933d76 100644 --- a/crates/rocm-dash-collectors/src/amd_smi.rs +++ b/crates/rocm-dash-collectors/src/amd_smi.rs @@ -129,6 +129,13 @@ impl AmdSmiCollector { Ok(parse_metrics(&v)) } + /// Per-GPU power caps (W) from `amd-smi static --json`, keyed by the same + /// `gpu-N` id [`metrics`] uses. GPUs with no readable cap are omitted. + pub async fn power_limits(&self) -> io::Result> { + let v = self.run_json(&["static", "--json"]).await?; + Ok(parse_power_limits(&v)) + } + /// Per-process GPU VRAM usage from `amd-smi process --json`. /// /// Inherent async (mirrors [`metrics`]); the runner uses this directly @@ -258,11 +265,51 @@ pub fn parse_metrics(v: &Value) -> Vec { temperature_c, power_w: val_f32(nested(g, &["power", "socket_power", "value"])).unwrap_or(0.0), clock_mhz: val_f32(nested(g, &["clock", "gfx_0", "clk", "value"])), + // Filled by the runner from `power_limits`; not in `metric`. + power_limit_w: None, } }) .collect() } +/// Candidate keys under a GPU's `limit` object for its power cap, in priority +/// order: the cap currently in force, then the maximum the part allows. +const POWER_LIMIT_KEYS: [&str; 2] = ["socket_power", "max_power"]; + +/// Parse `amd-smi static --json` into per-GPU power caps (W), keyed `gpu-N`. +/// +/// Assumed schema (not verified against every amd-smi release): top level +/// `{ "gpu_data": [ , ... ] }` or a bare array; each `` has +/// `{ "gpu": N, "limit": { "socket_power" | "max_power": } }` where the +/// value is a raw number or a `{ "value": W, "unit": "W" }` wrapper. A missing, +/// non-numeric, non-finite or non-positive cap (`N/A`, `0`) is omitted, so an +/// unrecognised schema yields an empty map and callers treat the cap as +/// unknown rather than guessing one. +pub fn parse_power_limits(v: &Value) -> std::collections::HashMap { + let entries: Vec<&Value> = if let Some(arr) = v.get("gpu_data").and_then(|x| x.as_array()) { + arr.iter().collect() + } else if let Some(arr) = v.as_array() { + arr.iter().collect() + } else { + return std::collections::HashMap::new(); + }; + entries + .into_iter() + .filter_map(|g| { + let id = g.get("gpu").and_then(serde_json::Value::as_u64)?; + let limit = g.get("limit")?; + // Validity is judged per key, so an unreadable or zero current cap + // falls through to the part's maximum instead of masking it. + let watts = POWER_LIMIT_KEYS.iter().find_map(|k| { + let raw = limit.get(*k)?; + let w = val_f32(Some(raw)).or_else(|| val_f32(raw.get("value")))?; + (w.is_finite() && w > 0.0).then_some(w) + })?; + Some((format!("gpu-{id}"), watts)) + }) + .collect() +} + /// Candidate `(parent, child)` JSON paths for a process's VRAM figure, in /// priority order. amd-smi naming varies by version, so we probe several. const VRAM_PATHS: &[(&str, &str)] = &[ @@ -504,6 +551,39 @@ mod tests { ); } + #[test] + fn power_limits_prefer_current_cap_and_skip_unreadable() { + let v: Value = serde_json::from_str( + r#"{ "gpu_data": [ + { "gpu": 0, "limit": { "socket_power": { "value": 600, "unit": "W" }, + "max_power": { "value": 750, "unit": "W" } } }, + { "gpu": 1, "limit": { "max_power": 750 } }, + { "gpu": 2, "limit": { "socket_power": "N/A" } }, + { "gpu": 3, "limit": { "max_power": { "value": 0, "unit": "W" } } }, + { "gpu": 4 } + ] }"#, + ) + .unwrap(); + let m = parse_power_limits(&v); + assert_eq!(m.get("gpu-0"), Some(&600.0), "current cap wins over max"); + assert_eq!(m.get("gpu-1"), Some(&750.0), "raw number falls back to max"); + assert_eq!(m.len(), 2, "N/A, zero and absent caps are unknown: {m:?}"); + // Unrecognised shapes are an empty map, not a panic. + assert!(parse_power_limits(&Value::Null).is_empty()); + } + + #[test] + fn a_zero_current_cap_falls_through_to_the_maximum() { + let v: Value = serde_json::from_str( + r#"[ { "gpu": 0, "limit": { "socket_power": 0, "max_power": 750 } }, + { "gpu": 1, "limit": { "socket_power": "N/A", "max_power": 600 } } ]"#, + ) + .unwrap(); + let m = parse_power_limits(&v); + assert_eq!(m.get("gpu-0"), Some(&750.0), "0 must not mask max_power"); + assert_eq!(m.get("gpu-1"), Some(&600.0), "N/A falls through too"); + } + const SAMPLE_METRIC: &str = r#"{ "gpu_data": [ { diff --git a/crates/rocm-dash-collectors/src/bench_load.rs b/crates/rocm-dash-collectors/src/bench_load.rs index c0a906c8f1..d875834dc2 100644 --- a/crates/rocm-dash-collectors/src/bench_load.rs +++ b/crates/rocm-dash-collectors/src/bench_load.rs @@ -542,10 +542,9 @@ fn prom_latency( /// ever handled — earlier ramp cells and other clients included — so writing it /// into this cell's immutable CSV row would describe a different population than /// the row names, with nothing to flag it. A blank column therefore honestly -/// means "not measured for this cell". (This is narrower than the telemetry -/// daemon's `avg_ms_from_histogram`, which *does* fall back: it is a repeated -/// live poll with a rolling baseline that self-corrects on the next tick and -/// persists nothing, so the trade-offs differ.) +/// means "not measured for this cell". (The telemetry daemon's live TTFT/TPOT +/// follow the same rule: only a window with new requests yields a value, and +/// it is held for a bounded time and then cleared.) fn latency_ms( sum_before: Option, count_before: Option, diff --git a/crates/rocm-dash-collectors/src/cgroup.rs b/crates/rocm-dash-collectors/src/cgroup.rs index 3598d3afcf..4cf7bf151d 100644 --- a/crates/rocm-dash-collectors/src/cgroup.rs +++ b/crates/rocm-dash-collectors/src/cgroup.rs @@ -40,6 +40,15 @@ pub fn parse_container_id_from_cgroup(contents: &str) -> Option { .map(str::to_string) } +/// Whether `id` has the shape [`parse_container_id_from_cgroup`] produces. +/// +/// That shape is 64 lowercase-hex characters. A managed `rocm serve` service is named by its +/// service id, which never has this shape, so it can never be a key of the +/// per-container VRAM map built from `/proc//cgroup`. +pub fn is_container_id(id: &str) -> bool { + id.len() == 64 && id.chars().all(is_lower_hex) +} + /// Resolve the docker container id owning host process `pid` by reading `/proc//cgroup`. /// /// Returns `None` on any read error (process gone, @@ -119,6 +128,17 @@ mod tests { ); } + #[test] + fn is_container_id_accepts_only_the_cgroup_join_shape() { + assert!(is_container_id(ID)); + assert!(!is_container_id( + "vllm-qwen-qwen2-5-0-5b-instru-1793361705247" + )); + assert!(!is_container_id(&ID.to_uppercase())); + assert!(!is_container_id(&ID[..63])); + assert!(!is_container_id("")); + } + #[test] fn container_id_for_pid_unreadable_is_none_no_panic() { // A PID that cannot exist → unreadable /proc path → None (no panic). diff --git a/crates/rocm-dash-collectors/src/winperf.rs b/crates/rocm-dash-collectors/src/winperf.rs index 0cacaf327a..7114f2c1cb 100644 --- a/crates/rocm-dash-collectors/src/winperf.rs +++ b/crates/rocm-dash-collectors/src/winperf.rs @@ -352,6 +352,7 @@ fn parse_metrics(raw: &str, adlx: &[AdlxReading]) -> Vec { gpu_utilization_pct, temperature_c, power_w, + power_limit_w: None, clock_mhz, }]; } @@ -371,6 +372,8 @@ fn parse_metrics(raw: &str, adlx: &[AdlxReading]) -> Vec { gpu_utilization_pct: util.get(key).copied().unwrap_or(0.0).min(100.0), temperature_c, power_w, + // No limit source on Windows: unknown, so power is not colored. + power_limit_w: None, clock_mhz, }) .collect() diff --git a/crates/rocm-dash-core/src/metrics.rs b/crates/rocm-dash-core/src/metrics.rs index 78dd44743e..77d14afbe3 100644 --- a/crates/rocm-dash-core/src/metrics.rs +++ b/crates/rocm-dash-core/src/metrics.rs @@ -42,6 +42,12 @@ pub struct GpuMetrics { pub gpu_utilization_pct: f32, pub temperature_c: f32, pub power_w: f32, + /// The board's power cap (W), when the platform reports one. Warning + /// thresholds are derived from it; `None` means unknown (never `0`), and + /// consumers must not substitute another part's limit. Serde-default so + /// older recordings load. + #[serde(default)] + pub power_limit_w: Option, pub clock_mhz: Option, } @@ -215,8 +221,9 @@ pub struct Instance { /// Omitted from JSON when `None` to preserve wire back-compat. #[serde(default, skip_serializing_if = "Option::is_none")] pub gen_tps_observation: Option, - /// Efficiency: `gen_tps` ÷ summed power (W) of the GPUs this instance - /// occupies. `None` when throughput or GPU power telemetry is unavailable. + /// Efficiency: `gen_tps` ÷ the power (W) attributed to this instance + /// ([`Instance::power_w`]). `None` when throughput or GPU power telemetry + /// is unavailable. #[serde(default)] pub tokens_per_watt: Option, /// Board power (W) attributed to this instance: each GPU's power split among @@ -225,8 +232,8 @@ pub struct Instance { #[serde(default)] pub power_w: Option, /// Live average time-to-first-token (ms), windowed from the vLLM TTFT - /// histogram. `None` until two scrapes (or absent for non-vLLM engines); - /// Observe shows `—`. Serde-default for NDJSON replay back-compat. + /// histogram. `None` until a window sees new requests (or absent for + /// non-vLLM engines); Observe shows `—`. Serde-default for NDJSON replay back-compat. #[serde(default)] pub ttft_ms: Option, /// Freshness provenance for [`Instance::ttft_ms`]. A value with no new @@ -236,7 +243,8 @@ pub struct Instance { #[serde(default, skip_serializing_if = "Option::is_none")] pub ttft_observation: Option, /// Live average time-per-output-token (ms), windowed from the vLLM TPOT - /// histogram. `None` until two scrapes (or absent); Observe shows `—`. + /// histogram. `None` until a window sees new requests (or absent); + /// Observe shows `—`. #[serde(default)] pub tpot_ms: Option, /// Freshness provenance for [`Instance::tpot_ms`]; same semantics as @@ -272,6 +280,20 @@ mod tests { assert_eq!(inst.kv_cache_usage_pct, Some(42.0)); } + #[test] + fn gpu_metrics_deserializes_without_power_limit() { + // Replay back-compat: recordings made before `power_limit_w` existed + // must load with the limit unknown, never 0. + let legacy = r#"{ + "device_id": "gpu-0", "vram_used_mb": 1, "vram_total_mb": 2, + "gpu_utilization_pct": 3.0, "temperature_c": 4.0, "power_w": 5.0, + "clock_mhz": null + }"#; + let g: GpuMetrics = serde_json::from_str(legacy).expect("legacy GPU must parse"); + assert_eq!(g.power_limit_w, None); + assert!((g.power_w - 5.0).abs() < f32::EPSILON); + } + #[test] fn instance_round_trips_ttft_tpot_through_serde() { // The new live-latency fields survive a serialize → deserialize cycle. diff --git a/crates/rocm-dash-daemon/src/demo.rs b/crates/rocm-dash-daemon/src/demo.rs index 4ed555f256..50ede75a7e 100644 --- a/crates/rocm-dash-daemon/src/demo.rs +++ b/crates/rocm-dash-daemon/src/demo.rs @@ -540,6 +540,8 @@ fn build_gpu(idx: usize, t_s: f64, seed: &mut u64) -> GpuMetrics { gpu_utilization_pct: util, temperature_c: temp, power_w: power, + // MI355X-class board power cap. + power_limit_w: Some(750.0), clock_mhz: Some((jitter(seed) - 0.5).mul_add(80.0, 1850.0)), } } diff --git a/crates/rocm-dash-daemon/src/runner.rs b/crates/rocm-dash-daemon/src/runner.rs index 0d1af6f9cf..47225ddf85 100644 --- a/crates/rocm-dash-daemon/src/runner.rs +++ b/crates/rocm-dash-daemon/src/runner.rs @@ -408,6 +408,11 @@ pub async fn run_loop( let mut tick_count: u64 = 0; let mut last_sysinfo_refresh: u64 = 0; + // Static per-GPU power caps (`amd-smi static`), refreshed on the sysinfo + // cadence and stamped onto each tick's metrics. `None` until first fetched. + let mut power_limits: HashMap = HashMap::new(); + let mut last_power_limit_refresh: Option = None; + let mut warned_no_power_limits = false; // Discovery/scrape warnings are produced on their own (slower) cadences, so // holding them only for the tick that produced them makes the header ⚠ badge // blink on and off. Keep the last result until that cadence runs again. @@ -467,8 +472,42 @@ pub async fn run_loop( } let mut warnings = Vec::new(); - let gpus = if let Some(g) = &gpu { - match g.metrics().await { + // Power caps are static, so they refresh on the sysinfo cadence. Fetch + // them alongside this tick's metrics instead of ahead of it, so the + // refresh tick pays one amd-smi round trip, not two in a row. + let refresh_limits = gpu.is_some() + && last_power_limit_refresh + .is_none_or(|at| tick_count.saturating_sub(at) >= sysinfo_refresh_ticks); + let mut gpus = if let Some(g) = &gpu { + let (limits, metrics) = tokio::join!( + async { + if refresh_limits { + Some(g.power_limits().await) + } else { + None + } + }, + g.metrics() + ); + if let Some(res) = limits { + match res { + // An empty map (a schema the parser no longer matches) must + // not wipe caps we already hold, and is worth saying once. + Ok(l) if l.is_empty() => { + if power_limits.is_empty() && !warned_no_power_limits { + warn!( + "amd-smi static reported no readable power cap; power stays uncolored" + ); + warned_no_power_limits = true; + } + } + Ok(l) => power_limits = l, + // A failed fetch keeps the previous caps; retry next interval. + Err(_) => {} + } + last_power_limit_refresh = Some(tick_count); + } + match metrics { Ok(v) => v, Err(e) => { warnings.push(format!("amd-smi metric: {e}")); @@ -507,6 +546,10 @@ pub async fn run_loop( Vec::new() }; + for g in &mut gpus { + g.power_limit_w = power_limits.get(&g.device_id).copied(); + } + if gpu.is_some() && tick_count.saturating_sub(last_sysinfo_refresh) >= sysinfo_refresh_ticks && let Some(g) = &gpu @@ -972,14 +1015,7 @@ pub async fn run_loop( // Warn only when a real process scrape happened but an instance with // live GPUs still fell back to device-summed (cgroup join missed). if procs_nonempty { - let vram_fallbacks = instances - .iter() - .filter(|i| { - !i.gpu_ids.is_empty() - && rocm_dash_core::efficiency::gpu_ids_overlap(&i.gpu_ids, &gpus) - && !per_container_used.contains_key(&i.container_id) - }) - .count(); + let vram_fallbacks = count_vram_join_misses(&instances, &gpus, &per_container_used); if vram_fallbacks > 0 { scrape_warns.push(format!( "vram attribution: {vram_fallbacks} instance(s) fell back to device-summed \ @@ -1028,6 +1064,29 @@ pub async fn run_loop( } } +/// Instances that *could* have joined the per-process VRAM map but did not. +/// +/// The map is keyed by docker/containerd ids read from `/proc//cgroup`, +/// so only an instance whose id has that shape can ever hit it. A managed +/// `rocm serve` service (named by its service id) falls back to device-summed +/// VRAM by design, and counting it would raise a false "check /proc access" +/// warning on every healthy managed instance. +fn count_vram_join_misses( + instances: &[Instance], + gpus: &[GpuMetrics], + per_container_used: &HashMap, +) -> usize { + instances + .iter() + .filter(|i| { + rocm_dash_collectors::cgroup::is_container_id(&i.container_id) + && !i.gpu_ids.is_empty() + && rocm_dash_core::efficiency::gpu_ids_overlap(&i.gpu_ids, gpus) + && !per_container_used.contains_key(&i.container_id) + }) + .count() +} + /// Remove all per-service telemetry state for `id`: tracker, TTFT, and TPOT /// baselines. Call on explicit removal (Docker Gone, Lemonade Gone, managed /// Gone) and whenever an identity change requires a clean slate. @@ -1406,6 +1465,31 @@ mod tests { (v, m.map(|m| m.freshness)) } + fn inst_on(id: &str, gpu: &str) -> Instance { + Instance { + container_id: id.into(), + gpu_ids: vec![gpu.into()], + ..Instance::default() + } + } + + #[test] + fn managed_instances_never_count_as_vram_join_misses() { + let docker_id = "ab".repeat(32); + let gpus = [gpu("gpu-0", 1000, 8000)]; + let insts = [ + inst_on("vllm-qwen-qwen2-5-0-5b-instru-1793361705247", "0"), + inst_on(&docker_id, "0"), + ]; + // A docker instance with no /proc match is a real miss; the managed + // service, which could never match, is not. + let empty: HashMap = HashMap::new(); + assert_eq!(count_vram_join_misses(&insts, &gpus, &empty), 1); + // Once the docker instance joins, nothing is missed. + let joined: HashMap = [(docker_id, 512)].into(); + assert_eq!(count_vram_join_misses(&insts, &gpus, &joined), 0); + } + #[test] fn latency_first_scrape_reports_nothing_then_windows() { let mut w = LatencyWindows::new(); diff --git a/crates/rocm-dash-daemon/tests/telemetry_contract.rs b/crates/rocm-dash-daemon/tests/telemetry_contract.rs index e1473d3f5b..c15589325d 100644 --- a/crates/rocm-dash-daemon/tests/telemetry_contract.rs +++ b/crates/rocm-dash-daemon/tests/telemetry_contract.rs @@ -734,9 +734,10 @@ async fn ttft_is_held_then_cleared_when_requests_stop() { // ── GPU power limit and shared-GPU attribution (ROCMAI-62) ────────────────── -/// Contract: with `amd-smi` reporting one 400 W GPU and two managed services -/// launched on that GPU, the broadcast snapshot splits the board power between -/// the two instances instead of crediting each with all of it. Drives the real runner with a +/// Contract: with `amd-smi` reporting one 400 W GPU capped at 750 W and two +/// managed services launched on that GPU, the broadcast snapshot carries the +/// cap on the GPU and splits the board power between the two instances +/// instead of crediting each with all of it. Drives the real runner with a /// fake `amd-smi`; amd-smi itself is not needed on the host. #[cfg(unix)] #[tokio::test] @@ -760,7 +761,8 @@ async fn shared_gpu_power_is_split_and_limit_is_reported() { r#"#!/bin/sh case "$1" in metric) echo '{"gpu_data":[{"gpu":0,"mem_usage":{"used_vram":{"value":1000},"total_vram":{"value":8000}},"usage":{"gfx_activity":{"value":50}},"temperature":{"hotspot":{"value":60}},"power":{"socket_power":{"value":400.0,"unit":"W"}}}]}' ;; - process) echo '{"gpu_data":[]}' ;; + static) echo '{"gpu_data":[{"gpu":0,"limit":{"max_power":{"value":750,"unit":"W"}}}]}' ;; + process) echo '{"gpu_data":[{"gpu":0,"process_list":[{"process_info":{"pid":1,"memory_usage":{"vram_mem":{"value":100,"unit":"MB"}}}}]}]}' ;; *) echo '{}' ;; esac "#, @@ -775,22 +777,38 @@ esac }; let _runner = spawn_runner(tx, opts).await; - let (a, b) = wait_for_snapshot(&mut rx, |snap| { + let (limit, a, b, warnings) = wait_for_snapshot(&mut rx, |snap| { + let gpu = snap.gpus.first()?; let power = |id: &str| { snap.instances .iter() .find(|i| i.container_id == id) .and_then(|i| i.power_w) }; - Some((power("shared-a")?, power("shared-b")?)) + Some(( + gpu.power_limit_w?, + power("shared-a")?, + power("shared-b")?, + snap.warnings.clone(), + )) }) .await - .unwrap_or_else(|e| panic!("attributed power never appeared: {e}")); + .unwrap_or_else(|e| panic!("limit and attributed power never appeared: {e}")); - // No per-process VRAM was reported, so the board is split equally and the - // two shares add back up to what the GPU actually draws. + assert!((limit - 750.0).abs() < f32::EPSILON, "limit was {limit}"); + // Neither service joins the per-container VRAM map (managed services can + // not), so the board is split equally and the two shares add back up to + // what the GPU actually draws. assert!((a - 200.0).abs() < 1e-6, "shared-a got {a} W"); assert!((b - 200.0).abs() < 1e-6, "shared-b got {b} W"); + // amd-smi lists a GPU process, so the VRAM scrape is "real", but that + // process is in no container the cgroup join can name. Managed services + // can never join that map by design, so this must not read as a /proc + // problem. + assert!( + !warnings.iter().any(|w| w.starts_with("vram attribution")), + "false vram-attribution warning for healthy managed services: {warnings:?}" + ); } // ── Scenario 5: Omitted counter → None (not zero, not panic) ─────────────── diff --git a/crates/rocm-dash-tui/src/agent/snapshot.rs b/crates/rocm-dash-tui/src/agent/snapshot.rs index d72f4dd74f..43ebe0d5b2 100644 --- a/crates/rocm-dash-tui/src/agent/snapshot.rs +++ b/crates/rocm-dash-tui/src/agent/snapshot.rs @@ -24,6 +24,7 @@ fn gpu_json(g: &GpuMetrics) -> Value { "gpu_utilization_pct": g.gpu_utilization_pct, "temperature_c": g.temperature_c, "power_w": g.power_w, + "power_limit_w": g.power_limit_w, "vram_used_mb": g.vram_used_mb, "vram_total_mb": g.vram_total_mb, }) @@ -61,7 +62,14 @@ fn freshness_label(obs: Option<&ObservationMetadata>) -> Value { /// Fields added by EAI-7960: /// - `gen_tps`: live generation throughput from the Instance (same as daemon) /// - `tokens_per_watt`: daemon-provided efficiency (never recomputed here) -/// - `freshness`: `"fresh"` | `"held"` | `null` (null for legacy/unknown) +/// - `attributed_power_w`: this instance's share of its GPUs' board power +/// (split among instances sharing a GPU). Not the same as `gpu_status`'s +/// `power_w`, which is the whole board. +/// - `ttft_ms` / `tpot_ms` with `ttft_freshness` / `tpot_freshness` +/// (`"fresh"` | `"held"` | `null`): a held latency is the last value from a +/// window that saw requests, not a live reading +/// - `freshness`: `"fresh"` | `"held"` | `null` (null for legacy/unknown), +/// for `gen_tps` /// - `observed_at`: ISO-8601 UTC string when present, `null` when absent pub fn list_instances_json(snap: &StateSnapshot) -> Value { let arr: Vec = snap @@ -90,7 +98,7 @@ pub fn list_instances_json(snap: &StateSnapshot) -> Value { "waiting_reqs": i.waiting_reqs, "gen_tps": i.gen_tps, "tokens_per_watt": i.tokens_per_watt, - "power_w": i.power_w, + "attributed_power_w": i.power_w, "ttft_ms": i.ttft_ms, "ttft_freshness": freshness_label(i.ttft_observation.as_ref()), "tpot_ms": i.tpot_ms, @@ -282,7 +290,32 @@ mod tests { // No metadata is unknown, never reported as fresh. assert_eq!(i["tpot_ms"], 22.0); assert!(i["tpot_freshness"].is_null()); - assert_eq!(i["power_w"], 123.0); + assert_eq!(i["attributed_power_w"], 123.0); + } + + #[test] + fn gpu_status_json_reports_the_power_limit_or_null() { + use rocm_dash_core::metrics::Snapshot; + let mut snap = fixture_snapshot(); + let mut g = GpuMetrics { + device_id: "gpu-0".into(), + power_w: 700.0, + power_limit_w: Some(750.0), + ..GpuMetrics::default() + }; + snap.latest = Some(Snapshot { + gpus: vec![g.clone()], + ..Snapshot::default() + }); + let v = gpu_status_json(&snap, None); + assert_eq!(v["gpus"][0]["power_limit_w"], 750.0); + // Unknown stays null: never 0, never another part's limit. + g.power_limit_w = None; + snap.latest = Some(Snapshot { + gpus: vec![g], + ..Snapshot::default() + }); + assert!(gpu_status_json(&snap, None)["gpus"][0]["power_limit_w"].is_null()); } #[test] diff --git a/crates/rocm-dash-tui/src/agent/tools.rs b/crates/rocm-dash-tui/src/agent/tools.rs index 113ece0518..1b3b19d4c7 100644 --- a/crates/rocm-dash-tui/src/agent/tools.rs +++ b/crates/rocm-dash-tui/src/agent/tools.rs @@ -112,7 +112,11 @@ impl Tool for ListInstancesTool { ToolDefinition { name: Self::NAME.to_string(), description: "List discovered serving instances: name, model, status, \ - KV-cache usage %, and running/waiting request counts." + KV-cache usage %, running/waiting request counts, \ + generation tok/s, TTFT/TPOT (each with a fresh/held \ + flag; held means the last value, not a live reading), \ + and attributed_power_w (this instance's share of its \ + GPUs' power, not whole-board power)." .to_string(), parameters: json!({ "type": "object", "properties": {} }), } @@ -166,7 +170,8 @@ impl Tool for TokensPerWattTool { ToolDefinition { name: Self::NAME.to_string(), description: "Per-instance efficiency: generation tokens/sec divided by \ - the summed power (W) of the GPUs each instance occupies." + the instance's attributed share of its GPUs' power (W). \ + GPUs shared by several instances are split between them." .to_string(), parameters: json!({ "type": "object", "properties": {} }), } diff --git a/crates/rocm-dash-tui/src/ui/tabs/hardware.rs b/crates/rocm-dash-tui/src/ui/tabs/hardware.rs index 4314a95bbc..2474b32437 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/hardware.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/hardware.rs @@ -393,7 +393,7 @@ pub fn gpu_compact_line( ), Span::styled( format!("{:>4.0} W", g.power_w), - power_style(g.power_w, theme), + power_style(g.power_w, g.power_limit_w, theme), ), ]; let base_w: usize = spans.iter().map(|s| s.content.chars().count()).sum(); @@ -673,6 +673,7 @@ mod tests { gpu_utilization_pct: util, temperature_c: temp, power_w: power, + power_limit_w: None, clock_mhz: Some(1850.0), } } @@ -716,7 +717,8 @@ mod tests { fn gpu_compact_line_colors_and_truncates() { let theme = Theme::default_dark(); // hot + high power → err color on both temp and power spans - let hot = mk_gpu("gpu-2", 95.0, 88.0, 740.0); + let mut hot = mk_gpu("gpu-2", 95.0, 88.0, 740.0); + hot.power_limit_w = Some(750.0); let line = gpu_compact_line(&hot, &theme, 120, true); let temp_span = line .spans @@ -726,6 +728,15 @@ mod tests { let pow_span = line.spans.iter().find(|s| s.content.contains('W')).unwrap(); assert_eq!(temp_span.style.fg, Some(theme.err)); assert_eq!(pow_span.style.fg, Some(theme.err)); + // same draw but no known power limit → power is not judged (plain fg) + hot.power_limit_w = None; + let unjudged = gpu_compact_line(&hot, &theme, 120, true); + let pow = unjudged + .spans + .iter() + .find(|s| s.content.contains('W')) + .unwrap(); + assert_eq!(pow.style.fg, Some(theme.fg)); // wide → vram segment included assert!(line.spans.iter().any(|s| s.content.contains("GiB"))); // narrow → vram segment dropped (truncation by omission) diff --git a/crates/rocm-dash-tui/src/ui/tabs/home.rs b/crates/rocm-dash-tui/src/ui/tabs/home.rs index b4a4ba043b..e49cdeb872 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/home.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/home.rs @@ -724,6 +724,7 @@ mod tests { gpu_utilization_pct: 62.0, temperature_c: 51.0, power_w: 420.0, + power_limit_w: None, clock_mhz: Some(2100.0), }], gpu_system_info: Some(GpuSystemInfo { @@ -785,6 +786,7 @@ mod tests { gpu_utilization_pct: 0.0, temperature_c: 0.0, power_w: 0.0, + power_limit_w: None, clock_mhz: None, }], gpu_system_info: None, @@ -837,6 +839,7 @@ mod tests { gpu_utilization_pct: 12.0, temperature_c: 0.0, power_w: 0.0, + power_limit_w: None, clock_mhz: None, }], gpu_system_info: Some(GpuSystemInfo { @@ -978,6 +981,7 @@ mod tests { gpu_utilization_pct: 0.5, temperature_c: 0.0, power_w: 0.0, + power_limit_w: None, clock_mhz: None, }], ..Default::default() @@ -995,6 +999,7 @@ mod tests { gpu_utilization_pct: 0.0, temperature_c: 0.0, power_w: 0.0, + power_limit_w: None, clock_mhz: None, }; let snap = Snapshot { diff --git a/crates/rocm-dash-tui/src/ui/tabs/instances.rs b/crates/rocm-dash-tui/src/ui/tabs/instances.rs index 6538ef6ded..ce7370da83 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/instances.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/instances.rs @@ -158,8 +158,10 @@ pub fn draw_table(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { // Wide enough for "{v:.2} tok/W*" (tokens_per_watt_cell's longest // rendering) so the held marker never gets clipped mid-unit. Constraint::Length(12), - Constraint::Length(7), - Constraint::Length(7), + // Wide enough for a held 5-digit latency ("10000ms*") so the marker is + // never the part that gets clipped. + Constraint::Length(8), + Constraint::Length(8), Constraint::Length(7), Constraint::Length(7), Constraint::Length(5), @@ -760,6 +762,35 @@ pub(in crate::ui) mod tests { ); } + #[test] + fn table_held_tpot_shows_marker_and_legend_and_five_digit_latency_is_not_clipped() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + + let mut inst = mk_inst_obs( + "idle", + Some(456.0), + Some(obs(ObservationFreshness::Fresh, 5)), + ); + // TPOT alone is held, and long enough that a 7-wide column would clip + // the trailing marker. + inst.tpot_ms = Some(12345.0); + inst.tpot_observation = Some(obs(ObservationFreshness::Held, 30)); + let state = state_with_snap(inst); + let mut term = Terminal::new(TestBackend::new(160, 20)).unwrap(); + term.draw(|f| draw_table(f, f.area(), &state, &state.theme)) + .unwrap(); + let out = buffer_text(&term); + assert!( + out.contains("12345ms*"), + "held TPOT marker missing or clipped; got:\n{out}" + ); + assert!( + out.contains(format::HELD_LEGEND), + "legend must explain a held TPOT marker; got:\n{out}" + ); + } + #[test] fn table_held_legend_absent_when_all_fresh() { use ratatui::Terminal; diff --git a/crates/rocm-dash-tui/src/ui/tabs/observe.rs b/crates/rocm-dash-tui/src/ui/tabs/observe.rs index 15554f6bd2..f200ef3b81 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/observe.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/observe.rs @@ -291,6 +291,7 @@ mod tests { gpu_utilization_pct: 50.0, temperature_c: 55.0, power_w: 300.0, + power_limit_w: None, clock_mhz: Some(2000.0), }], ..Default::default() diff --git a/crates/rocm-dash-tui/src/ui/widgets.rs b/crates/rocm-dash-tui/src/ui/widgets.rs index ca5da108fe..f429ecdbda 100644 --- a/crates/rocm-dash-tui/src/ui/widgets.rs +++ b/crates/rocm-dash-tui/src/ui/widgets.rs @@ -16,15 +16,15 @@ pub const TEMP_WARN_C: f32 = 60.0; /// Temperature critical threshold (°C). At or above → err color. pub const TEMP_CRIT_C: f32 = 80.0; -/// Board-power warning threshold (W). At or above → warn color. +/// Fraction of a GPU's power limit at or above which power is colored warn. /// /// Fixed semantic thresholds make temperature and power colors mean "near the limit" -/// rather than "near the largest value seen this session". Tuned for -/// MI355X-class parts (TDP ~750 W); the demo generator peaks ~740 W at 100% -/// util. No per-GPU TDP exists in the data model, so these are constants. -pub const POWER_WARN_W: f32 = 525.0; -/// Board-power critical threshold (W). At or above → err color. ~TDP-adjacent. -pub const POWER_CRIT_W: f32 = 700.0; +/// rather than "near the largest value seen this session". The power thresholds +/// are relative to the device's own cap, so the colors mean "near this part's +/// limit" on any GPU, not "near an MI355X's". +pub const POWER_WARN_FRAC: f32 = 0.70; +/// Fraction of the power limit at or above which board power is colored err. +pub const POWER_CRIT_FRAC: f32 = 0.93; /// Truncate `s` to at most `n` characters (not bytes). pub fn trunc(s: &str, n: usize) -> String { @@ -80,15 +80,19 @@ pub fn temperature_style(c: f32, theme: &Theme) -> Style { Style::default().fg(color) } -/// Color a board-power value against the fixed [`POWER_WARN_W`] / -/// [`POWER_CRIT_W`] thresholds, mirroring [`temperature_style`]. -pub fn power_style(w: f32, theme: &Theme) -> Style { - let color = if w >= POWER_CRIT_W { - theme.err - } else if w >= POWER_WARN_W { - theme.warn - } else { - theme.ok +/// Color a board-power value against the GPU's own power limit +/// ([`POWER_WARN_FRAC`] / [`POWER_CRIT_FRAC`] of it), mirroring +/// [`temperature_style`]. +/// +/// With no known limit (or a non-positive one) there is nothing to compare +/// against, so the value is drawn in the plain foreground color rather than +/// judged against another part's TDP. +pub fn power_style(w: f32, limit_w: Option, theme: &Theme) -> Style { + let color = match limit_w.filter(|l| l.is_finite() && *l > 0.0) { + None => theme.fg, + Some(limit) if w >= limit * POWER_CRIT_FRAC => theme.err, + Some(limit) if w >= limit * POWER_WARN_FRAC => theme.warn, + Some(_) => theme.ok, }; Style::default().fg(color) } @@ -172,9 +176,13 @@ pub fn gpu_stats_line<'a>(g: &'a GpuMetrics, theme: &Theme) -> Line<'a> { Style::default().fg(theme.muted), ), Span::styled( - format!(" {:>5.1}°C {:>5.1} W", g.temperature_c, g.power_w), + format!(" {:>5.1}°C", g.temperature_c), temperature_style(g.temperature_c, theme), ), + Span::styled( + format!(" {:>5.1} W", g.power_w), + power_style(g.power_w, g.power_limit_w, theme), + ), ]) } @@ -216,14 +224,58 @@ mod tests { } #[test] - fn power_style_uses_fixed_thresholds() { + fn power_style_is_relative_to_the_gpu_limit() { let theme = Theme::default_dark(); - assert_eq!(power_style(720.0, &theme).fg, Some(theme.err)); - assert_eq!(power_style(600.0, &theme).fg, Some(theme.warn)); - assert_eq!(power_style(300.0, &theme).fg, Some(theme.ok)); - // boundaries are inclusive at the threshold - assert_eq!(power_style(POWER_CRIT_W, &theme).fg, Some(theme.err)); - assert_eq!(power_style(POWER_WARN_W, &theme).fg, Some(theme.warn)); + // Same wattage, different parts: 600 W is critical on a 600 W card but + // comfortably fine on a 1000 W one. + assert_eq!(power_style(600.0, Some(600.0), &theme).fg, Some(theme.err)); + assert_eq!(power_style(600.0, Some(1000.0), &theme).fg, Some(theme.ok)); + assert_eq!(power_style(500.0, Some(600.0), &theme).fg, Some(theme.warn)); + assert_eq!(power_style(300.0, Some(600.0), &theme).fg, Some(theme.ok)); + // Boundaries are inclusive. + assert_eq!( + power_style(1000.0 * POWER_CRIT_FRAC, Some(1000.0), &theme).fg, + Some(theme.err) + ); + assert_eq!( + power_style(1000.0 * POWER_WARN_FRAC, Some(1000.0), &theme).fg, + Some(theme.warn) + ); + } + + #[test] + fn power_style_without_a_limit_is_neutral_not_judged() { + let theme = Theme::default_dark(); + for limit in [None, Some(0.0), Some(f32::NAN), Some(-5.0)] { + assert_eq!(power_style(720.0, limit, &theme).fg, Some(theme.fg)); + } + } + + #[test] + fn full_panel_colors_power_by_limit_not_by_temperature() { + let theme = Theme::default_dark(); + // A cool GPU at 95% of its cap: power is critical even though the + // temperature is fine, the same as in the compact row. + let g = GpuMetrics { + device_id: "gpu-0".into(), + temperature_c: 40.0, + power_w: 712.5, + power_limit_w: Some(750.0), + ..GpuMetrics::default() + }; + let line = gpu_stats_line(&g, &theme); + let temp = line + .spans + .iter() + .find(|s| s.content.contains("°C")) + .unwrap(); + let power = line + .spans + .iter() + .find(|s| s.content.contains(" W")) + .unwrap(); + assert_eq!(temp.style.fg, Some(theme.ok)); + assert_eq!(power.style.fg, Some(theme.err)); } #[test] diff --git a/crates/rocm-dash-tui/tests/dash_characterization.rs b/crates/rocm-dash-tui/tests/dash_characterization.rs index ffca5d6b12..fc7f8e659a 100644 --- a/crates/rocm-dash-tui/tests/dash_characterization.rs +++ b/crates/rocm-dash-tui/tests/dash_characterization.rs @@ -41,6 +41,7 @@ fn synthetic_snapshot() -> Snapshot { gpu_utilization_pct: 72.0, temperature_c: 58.0, power_w: 420.0, + power_limit_w: None, clock_mhz: Some(2100.0), }], gpu_system_info: Some(GpuSystemInfo { diff --git a/crates/rocm-dash-tui/tests/dash_journeys.rs b/crates/rocm-dash-tui/tests/dash_journeys.rs index 31117a8b41..2a8e1ecffc 100644 --- a/crates/rocm-dash-tui/tests/dash_journeys.rs +++ b/crates/rocm-dash-tui/tests/dash_journeys.rs @@ -39,6 +39,7 @@ fn synthetic_snapshot() -> Snapshot { gpu_utilization_pct: 66.0, temperature_c: 61.0, power_w: 410.0, + power_limit_w: None, clock_mhz: Some(2050.0), }], gpu_system_info: Some(GpuSystemInfo { diff --git a/tests/e2e-cucumber/src/mock_server.rs b/tests/e2e-cucumber/src/mock_server.rs index 3a2844d87f..60fb9d2950 100644 --- a/tests/e2e-cucumber/src/mock_server.rs +++ b/tests/e2e-cucumber/src/mock_server.rs @@ -378,7 +378,7 @@ struct ServerState { /// Deterministic, monotonically-advancing state behind the mock `/metrics` /// route, so successive scrapes exercise the daemon's rate/average windowing -/// (`gen_tps_from_delta`, `avg_ms_from_histogram` in +/// (`gen_tps_from_delta`, `observe_latency` in /// `rocm-dash-daemon::runner`) the same way a real vLLM would: /// * `vllm:generation_tokens_total` strictly increases every scrape, so the /// counter-delta windowing yields a positive, visible generation rate diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index f906992243..1f87dc8581 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -2499,8 +2499,8 @@ async fn metrics_endpoint_fails(world: &mut E2eWorld) { /// EAI-7960 principal regression assertion. /// -/// The frame under assertion is provably post-failure (the TTFT cell it used to -/// show is gone) and the logical clock is held, so the only way "tok/s" can be +/// The frame under assertion is provably post-failure (the header warnings +/// badge has grown past its pre-failure count) and the logical clock is held, so the only way "tok/s" can be /// missing here is the regression itself: the daemon clearing a held rate on a /// failed scrape instead of keeping it for the validity window. #[then("generation throughput remains visible within the validity window")]