diff --git a/crates/rocm-dash-collectors/src/amd_smi.rs b/crates/rocm-dash-collectors/src/amd_smi.rs index 03a8ffab22..5c36933d76 100644 --- a/crates/rocm-dash-collectors/src/amd_smi.rs +++ b/crates/rocm-dash-collectors/src/amd_smi.rs @@ -129,6 +129,13 @@ impl AmdSmiCollector { Ok(parse_metrics(&v)) } + /// Per-GPU power caps (W) from `amd-smi static --json`, keyed by the same + /// `gpu-N` id [`metrics`] uses. GPUs with no readable cap are omitted. + pub async fn power_limits(&self) -> io::Result> { + let v = self.run_json(&["static", "--json"]).await?; + Ok(parse_power_limits(&v)) + } + /// Per-process GPU VRAM usage from `amd-smi process --json`. /// /// Inherent async (mirrors [`metrics`]); the runner uses this directly @@ -258,11 +265,51 @@ pub fn parse_metrics(v: &Value) -> Vec { temperature_c, power_w: val_f32(nested(g, &["power", "socket_power", "value"])).unwrap_or(0.0), clock_mhz: val_f32(nested(g, &["clock", "gfx_0", "clk", "value"])), + // Filled by the runner from `power_limits`; not in `metric`. + power_limit_w: None, } }) .collect() } +/// Candidate keys under a GPU's `limit` object for its power cap, in priority +/// order: the cap currently in force, then the maximum the part allows. +const POWER_LIMIT_KEYS: [&str; 2] = ["socket_power", "max_power"]; + +/// Parse `amd-smi static --json` into per-GPU power caps (W), keyed `gpu-N`. +/// +/// Assumed schema (not verified against every amd-smi release): top level +/// `{ "gpu_data": [ , ... ] }` or a bare array; each `` has +/// `{ "gpu": N, "limit": { "socket_power" | "max_power": } }` where the +/// value is a raw number or a `{ "value": W, "unit": "W" }` wrapper. A missing, +/// non-numeric, non-finite or non-positive cap (`N/A`, `0`) is omitted, so an +/// unrecognised schema yields an empty map and callers treat the cap as +/// unknown rather than guessing one. +pub fn parse_power_limits(v: &Value) -> std::collections::HashMap { + let entries: Vec<&Value> = if let Some(arr) = v.get("gpu_data").and_then(|x| x.as_array()) { + arr.iter().collect() + } else if let Some(arr) = v.as_array() { + arr.iter().collect() + } else { + return std::collections::HashMap::new(); + }; + entries + .into_iter() + .filter_map(|g| { + let id = g.get("gpu").and_then(serde_json::Value::as_u64)?; + let limit = g.get("limit")?; + // Validity is judged per key, so an unreadable or zero current cap + // falls through to the part's maximum instead of masking it. + let watts = POWER_LIMIT_KEYS.iter().find_map(|k| { + let raw = limit.get(*k)?; + let w = val_f32(Some(raw)).or_else(|| val_f32(raw.get("value")))?; + (w.is_finite() && w > 0.0).then_some(w) + })?; + Some((format!("gpu-{id}"), watts)) + }) + .collect() +} + /// Candidate `(parent, child)` JSON paths for a process's VRAM figure, in /// priority order. amd-smi naming varies by version, so we probe several. const VRAM_PATHS: &[(&str, &str)] = &[ @@ -504,6 +551,39 @@ mod tests { ); } + #[test] + fn power_limits_prefer_current_cap_and_skip_unreadable() { + let v: Value = serde_json::from_str( + r#"{ "gpu_data": [ + { "gpu": 0, "limit": { "socket_power": { "value": 600, "unit": "W" }, + "max_power": { "value": 750, "unit": "W" } } }, + { "gpu": 1, "limit": { "max_power": 750 } }, + { "gpu": 2, "limit": { "socket_power": "N/A" } }, + { "gpu": 3, "limit": { "max_power": { "value": 0, "unit": "W" } } }, + { "gpu": 4 } + ] }"#, + ) + .unwrap(); + let m = parse_power_limits(&v); + assert_eq!(m.get("gpu-0"), Some(&600.0), "current cap wins over max"); + assert_eq!(m.get("gpu-1"), Some(&750.0), "raw number falls back to max"); + assert_eq!(m.len(), 2, "N/A, zero and absent caps are unknown: {m:?}"); + // Unrecognised shapes are an empty map, not a panic. + assert!(parse_power_limits(&Value::Null).is_empty()); + } + + #[test] + fn a_zero_current_cap_falls_through_to_the_maximum() { + let v: Value = serde_json::from_str( + r#"[ { "gpu": 0, "limit": { "socket_power": 0, "max_power": 750 } }, + { "gpu": 1, "limit": { "socket_power": "N/A", "max_power": 600 } } ]"#, + ) + .unwrap(); + let m = parse_power_limits(&v); + assert_eq!(m.get("gpu-0"), Some(&750.0), "0 must not mask max_power"); + assert_eq!(m.get("gpu-1"), Some(&600.0), "N/A falls through too"); + } + const SAMPLE_METRIC: &str = r#"{ "gpu_data": [ { diff --git a/crates/rocm-dash-collectors/src/bench_load.rs b/crates/rocm-dash-collectors/src/bench_load.rs index c0a906c8f1..d875834dc2 100644 --- a/crates/rocm-dash-collectors/src/bench_load.rs +++ b/crates/rocm-dash-collectors/src/bench_load.rs @@ -542,10 +542,9 @@ fn prom_latency( /// ever handled — earlier ramp cells and other clients included — so writing it /// into this cell's immutable CSV row would describe a different population than /// the row names, with nothing to flag it. A blank column therefore honestly -/// means "not measured for this cell". (This is narrower than the telemetry -/// daemon's `avg_ms_from_histogram`, which *does* fall back: it is a repeated -/// live poll with a rolling baseline that self-corrects on the next tick and -/// persists nothing, so the trade-offs differ.) +/// means "not measured for this cell". (The telemetry daemon's live TTFT/TPOT +/// follow the same rule: only a window with new requests yields a value, and +/// it is held for a bounded time and then cleared.) fn latency_ms( sum_before: Option, count_before: Option, diff --git a/crates/rocm-dash-collectors/src/cgroup.rs b/crates/rocm-dash-collectors/src/cgroup.rs index 3598d3afcf..4cf7bf151d 100644 --- a/crates/rocm-dash-collectors/src/cgroup.rs +++ b/crates/rocm-dash-collectors/src/cgroup.rs @@ -40,6 +40,15 @@ pub fn parse_container_id_from_cgroup(contents: &str) -> Option { .map(str::to_string) } +/// Whether `id` has the shape [`parse_container_id_from_cgroup`] produces. +/// +/// That shape is 64 lowercase-hex characters. A managed `rocm serve` service is named by its +/// service id, which never has this shape, so it can never be a key of the +/// per-container VRAM map built from `/proc//cgroup`. +pub fn is_container_id(id: &str) -> bool { + id.len() == 64 && id.chars().all(is_lower_hex) +} + /// Resolve the docker container id owning host process `pid` by reading `/proc//cgroup`. /// /// Returns `None` on any read error (process gone, @@ -119,6 +128,17 @@ mod tests { ); } + #[test] + fn is_container_id_accepts_only_the_cgroup_join_shape() { + assert!(is_container_id(ID)); + assert!(!is_container_id( + "vllm-qwen-qwen2-5-0-5b-instru-1793361705247" + )); + assert!(!is_container_id(&ID.to_uppercase())); + assert!(!is_container_id(&ID[..63])); + assert!(!is_container_id("")); + } + #[test] fn container_id_for_pid_unreadable_is_none_no_panic() { // A PID that cannot exist → unreadable /proc path → None (no panic). diff --git a/crates/rocm-dash-collectors/src/winperf.rs b/crates/rocm-dash-collectors/src/winperf.rs index 0cacaf327a..7114f2c1cb 100644 --- a/crates/rocm-dash-collectors/src/winperf.rs +++ b/crates/rocm-dash-collectors/src/winperf.rs @@ -352,6 +352,7 @@ fn parse_metrics(raw: &str, adlx: &[AdlxReading]) -> Vec { gpu_utilization_pct, temperature_c, power_w, + power_limit_w: None, clock_mhz, }]; } @@ -371,6 +372,8 @@ fn parse_metrics(raw: &str, adlx: &[AdlxReading]) -> Vec { gpu_utilization_pct: util.get(key).copied().unwrap_or(0.0).min(100.0), temperature_c, power_w, + // No limit source on Windows: unknown, so power is not colored. + power_limit_w: None, clock_mhz, }) .collect() diff --git a/crates/rocm-dash-core/src/efficiency.rs b/crates/rocm-dash-core/src/efficiency.rs index 072b022639..6781c86024 100644 --- a/crates/rocm-dash-core/src/efficiency.rs +++ b/crates/rocm-dash-core/src/efficiency.rs @@ -7,7 +7,10 @@ //! This is the join that turns vLLM throughput + amd-smi power into a //! per-instance capacity number. See `../wiki/concepts/metric-registry.md`. -use crate::metrics::GpuMetrics; +use std::collections::HashMap; +use std::hash::BuildHasher; + +use crate::metrics::{GpuMetrics, Instance}; /// Normalize a GPU identifier to a bare index for joining. /// @@ -34,44 +37,82 @@ pub fn gpu_ids_overlap(gpu_ids: &[String], gpus: &[GpuMetrics]) -> bool { gpus.iter().any(|g| device_in(&g.device_id, gpu_ids)) } -/// Summed board power (W) of the GPUs one instance occupies. +/// Board power (W) attributed to each instance, index-aligned with `instances`. +/// +/// A GPU's power is split among the instances that occupy it, so the instance +/// figures never add up to more than the board reports. Two instances sharing +/// one GPU each get a share, not the whole board. +/// +/// The split is proportional to each occupant's VRAM from +/// `per_container_vram_mb` (the per-process `amd-smi` attribution). A +/// multi-GPU instance's VRAM is spread evenly over its GPUs. If any occupant of +/// a GPU has no per-container reading the split on that GPU is equal, rather +/// than guessing a skew from partial data. A sole occupant gets the whole GPU. /// -/// Uses the same id-normalizing join as [`tokens_per_watt`]. `None` when no -/// matched GPU reports power (id join found nothing / amd-smi unavailable) so -/// the Observe table shows `—` rather than a fabricated `0 W`. +/// Power on a GPU no instance occupies (idle, or a process outside the managed +/// set) is attributed to nobody; it still counts toward node power. An entry is +/// `None` when none of its GPUs matched a GPU reporting power (id join found +/// nothing, power telemetry unavailable), so the UI shows `—` instead of a +/// fabricated `0 W`. #[must_use] -pub fn instance_power_w(gpu_ids: &[String], gpus: &[GpuMetrics]) -> Option { - let total: f64 = gpus - .iter() - .filter(|g| device_in(&g.device_id, gpu_ids)) - .map(|g| f64::from(g.power_w)) - .sum(); - (total > 0.0).then_some(total) +pub fn attribute_power_w( + instances: &[Instance], + gpus: &[GpuMetrics], + per_container_vram_mb: &HashMap, +) -> Vec> { + let mut watts = vec![0.0_f64; instances.len()]; + let mut matched = vec![false; instances.len()]; + for gpu in gpus { + let power = f64::from(gpu.power_w); + if !power.is_finite() || power <= 0.0 { + continue; + } + let occupants: Vec = instances + .iter() + .enumerate() + .filter(|(_, i)| device_in(&gpu.device_id, &i.gpu_ids)) + .map(|(k, _)| k) + .collect(); + if occupants.is_empty() { + continue; + } + let weights: Option> = occupants + .iter() + .map(|&k| { + let inst = &instances[k]; + per_container_vram_mb + .get(&inst.container_id) + .filter(|&&mb| mb > 0) + .map(|&mb| mb as f64 / inst.gpu_ids.len().max(1) as f64) + }) + .collect(); + let weight_sum = weights.as_ref().map(|w| w.iter().sum::()); + for (pos, &k) in occupants.iter().enumerate() { + let share = match (&weights, weight_sum) { + (Some(w), Some(sum)) if sum > 0.0 => w[pos] / sum, + _ => 1.0 / occupants.len() as f64, + }; + watts[k] = power.mul_add(share, watts[k]); + matched[k] = true; + } + } + watts + .into_iter() + .zip(matched) + .map(|(w, m)| m.then_some(w)) + .collect() } /// Tokens-per-watt for one serving instance: generation throughput (tok/s) -/// divided by the summed power (W) of the GPUs it occupies. +/// divided by the power (W) attributed to it by [`attribute_power_w`]. /// -/// Returns `None` when throughput is unknown (`gen_tps` is `None`) or the -/// matched GPUs report no power (e.g. amd-smi unavailable, or the id join -/// found nothing) — never a divide-by-zero or a negative number. A zero -/// throughput with live GPUs is a real `Some(0.0)`, not `None`. -pub fn tokens_per_watt( - gen_tps: Option, - gpu_ids: &[String], - gpus: &[GpuMetrics], -) -> Option { - let tps = gen_tps?; - let total_w: f64 = gpus - .iter() - .filter(|g| device_in(&g.device_id, gpu_ids)) - .map(|g| f64::from(g.power_w)) - .sum(); - if total_w > 0.0 { - Some(tps / total_w) - } else { - None - } +/// Returns `None` when throughput or attributed power is unknown, or power is +/// not positive — never a divide-by-zero or a negative number. A zero +/// throughput with live power is a real `Some(0.0)`, not `None`. +#[must_use] +pub fn tokens_per_watt(gen_tps: Option, power_w: Option) -> Option { + let (tps, power) = (gen_tps?, power_w?); + (power > 0.0).then(|| tps / power) } #[cfg(test)] @@ -94,60 +135,112 @@ mod tests { assert_eq!(normalize_gpu_id("3"), "3"); } + fn inst(id: &str, gpu_ids: &[&str]) -> Instance { + Instance { + container_id: id.into(), + gpu_ids: gpu_ids.iter().map(|s| (*s).to_string()).collect(), + ..Instance::default() + } + } + + fn vram(entries: &[(&str, u64)]) -> HashMap { + entries + .iter() + .map(|(k, v)| ((*k).to_string(), *v)) + .collect() + } + #[test] - fn joins_bare_index_to_prefixed_device_id() { + fn sole_occupant_gets_whole_board_via_bare_index_join() { // The crux: instance "0" must match amd-smi "gpu-0". - let gpus = [gpu("gpu-0", 250.0)]; - let r = tokens_per_watt(Some(500.0), &["0".into()], &gpus); - assert_eq!(r, Some(2.0)); + let p = attribute_power_w(&[inst("a", &["0"])], &[gpu("gpu-0", 250.0)], &vram(&[])); + assert_eq!(p, vec![Some(250.0)]); } #[test] - fn sums_power_across_tensor_parallel_gpus() { + fn tensor_parallel_instance_sums_its_gpus_and_ignores_others() { let gpus = [ gpu("gpu-0", 200.0), gpu("gpu-1", 300.0), gpu("gpu-2", 999.0), ]; - // Instance occupies gpus 0 and 1 → 500 W; gpu-2 excluded. - let r = tokens_per_watt(Some(1000.0), &["0".into(), "1".into()], &gpus); - assert_eq!(r, Some(2.0)); + let p = attribute_power_w(&[inst("a", &["0", "1"])], &gpus, &vram(&[])); + assert_eq!(p, vec![Some(500.0)]); } #[test] - fn none_throughput_yields_none() { - let gpus = [gpu("gpu-0", 250.0)]; - assert_eq!(tokens_per_watt(None, &["0".into()], &gpus), None); + fn shared_gpu_splits_by_vram_and_conserves_board_power() { + let gpus = [gpu("gpu-0", 400.0)]; + let insts = [inst("a", &["0"]), inst("b", &["0"])]; + let p = attribute_power_w(&insts, &gpus, &vram(&[("a", 30_000), ("b", 10_000)])); + assert_eq!(p, vec![Some(300.0), Some(100.0)]); + let total: f64 = p.iter().flatten().sum(); + assert!((total - 400.0).abs() < 1e-9, "must not exceed board power"); } #[test] - fn zero_total_power_yields_none() { - let gpus = [gpu("gpu-0", 0.0)]; - assert_eq!(tokens_per_watt(Some(500.0), &["0".into()], &gpus), None); + fn shared_gpu_splits_equally_without_complete_vram() { + let gpus = [gpu("gpu-0", 400.0)]; + let insts = [inst("a", &["0"]), inst("b", &["0"])]; + // No readings, or one occupant missing: equal split, no guessed skew. + for v in [ + vram(&[]), + vram(&[("a", 30_000)]), + vram(&[("a", 30_000), ("b", 0)]), + ] { + assert_eq!( + attribute_power_w(&insts, &gpus, &v), + vec![Some(200.0), Some(200.0)] + ); + } } #[test] - fn unmatched_ids_yield_none() { - let gpus = [gpu("gpu-5", 250.0)]; - assert_eq!(tokens_per_watt(Some(500.0), &["0".into()], &gpus), None); + fn multi_gpu_instance_vram_is_spread_across_its_gpus() { + // `a` spans gpu 0+1 (40 GB total -> 20 GB each); `b` is alone on gpu 1 + // with 20 GB. gpu-0 is all `a`; gpu-1 is split 50/50. + let gpus = [gpu("gpu-0", 100.0), gpu("gpu-1", 100.0)]; + let insts = [inst("a", &["0", "1"]), inst("b", &["1"])]; + let p = attribute_power_w(&insts, &gpus, &vram(&[("a", 40_000), ("b", 20_000)])); + assert_eq!(p, vec![Some(150.0), Some(50.0)]); } #[test] - fn empty_gpu_ids_yield_none() { - let gpus = [gpu("gpu-0", 250.0)]; - assert_eq!(tokens_per_watt(Some(500.0), &[], &gpus), None); + fn unoccupied_gpu_power_is_attributed_to_nobody() { + let gpus = [gpu("gpu-0", 100.0), gpu("gpu-1", 500.0)]; + let p = attribute_power_w(&[inst("a", &["0"])], &gpus, &vram(&[])); + assert_eq!(p, vec![Some(100.0)]); } #[test] - fn no_gpu_telemetry_yields_none() { - assert_eq!(tokens_per_watt(Some(500.0), &["0".into()], &[]), None); + fn unmatched_empty_or_unpowered_yield_none() { + let v = vram(&[]); + let a0 = [inst("a", &["0"])]; + assert_eq!( + attribute_power_w(&a0, &[gpu("gpu-5", 250.0)], &v), + vec![None] + ); + assert_eq!( + attribute_power_w(&[inst("a", &[])], &[gpu("gpu-0", 250.0)], &v), + vec![None] + ); + assert_eq!(attribute_power_w(&a0, &[], &v), vec![None]); + // Zero / NaN power means "no telemetry" (e.g. Windows without ADLX). + assert_eq!(attribute_power_w(&a0, &[gpu("gpu-0", 0.0)], &v), vec![None]); + assert_eq!( + attribute_power_w(&a0, &[gpu("gpu-0", f32::NAN)], &v), + vec![None] + ); } #[test] - fn zero_throughput_with_live_gpus_is_some_zero() { - // Idle instance, GPUs powered: 0 tok/W is a real reading, not "-". - let gpus = [gpu("gpu-0", 250.0)]; - assert_eq!(tokens_per_watt(Some(0.0), &["0".into()], &gpus), Some(0.0)); + fn tokens_per_watt_divides_and_guards() { + assert_eq!(tokens_per_watt(Some(500.0), Some(250.0)), Some(2.0)); + assert_eq!(tokens_per_watt(None, Some(250.0)), None); + assert_eq!(tokens_per_watt(Some(500.0), None), None); + assert_eq!(tokens_per_watt(Some(500.0), Some(0.0)), None); + // Idle instance, powered GPU: 0 tok/W is a real reading, not "-". + assert_eq!(tokens_per_watt(Some(0.0), Some(250.0)), Some(0.0)); } #[test] @@ -158,30 +251,4 @@ mod tests { assert!(!gpu_ids_overlap(&[], &gpus)); assert!(!gpu_ids_overlap(&["0".into()], &[])); } - - #[test] - fn instance_power_sums_matching_gpus() { - let gpus = [ - gpu("gpu-0", 200.0), - gpu("gpu-1", 300.0), - gpu("gpu-2", 999.0), - ]; - // Instance occupies gpus 0 and 1 → 500 W; gpu-2 is excluded. - assert_eq!( - instance_power_w(&["0".into(), "1".into()], &gpus), - Some(500.0) - ); - } - - #[test] - fn instance_power_no_match_yields_none() { - let gpus = [gpu("gpu-5", 250.0)]; - assert_eq!(instance_power_w(&["0".into()], &gpus), None); - } - - #[test] - fn instance_power_empty_ids_yields_none() { - let gpus = [gpu("gpu-0", 250.0)]; - assert_eq!(instance_power_w(&[], &gpus), None); - } } diff --git a/crates/rocm-dash-core/src/metrics.rs b/crates/rocm-dash-core/src/metrics.rs index b81ef252c4..77d14afbe3 100644 --- a/crates/rocm-dash-core/src/metrics.rs +++ b/crates/rocm-dash-core/src/metrics.rs @@ -42,6 +42,12 @@ pub struct GpuMetrics { pub gpu_utilization_pct: f32, pub temperature_c: f32, pub power_w: f32, + /// The board's power cap (W), when the platform reports one. Warning + /// thresholds are derived from it; `None` means unknown (never `0`), and + /// consumers must not substitute another part's limit. Serde-default so + /// older recordings load. + #[serde(default)] + pub power_limit_w: Option, pub clock_mhz: Option, } @@ -215,19 +221,36 @@ pub struct Instance { /// Omitted from JSON when `None` to preserve wire back-compat. #[serde(default, skip_serializing_if = "Option::is_none")] pub gen_tps_observation: Option, - /// Efficiency: `gen_tps` ÷ summed power (W) of the GPUs this instance - /// occupies. `None` when throughput or GPU power telemetry is unavailable. + /// Efficiency: `gen_tps` ÷ the power (W) attributed to this instance + /// ([`Instance::power_w`]). `None` when throughput or GPU power telemetry + /// is unavailable. #[serde(default)] pub tokens_per_watt: Option, + /// Board power (W) attributed to this instance: each GPU's power split among + /// the instances sharing it (see `efficiency::attribute_power_w`). `None` + /// when no occupied GPU reports power. Serde-default for replay back-compat. + #[serde(default)] + pub power_w: Option, /// Live average time-to-first-token (ms), windowed from the vLLM TTFT - /// histogram. `None` until two scrapes (or absent for non-vLLM engines); - /// Observe shows `—`. Serde-default for NDJSON replay back-compat. + /// histogram. `None` until a window sees new requests (or absent for + /// non-vLLM engines); Observe shows `—`. Serde-default for NDJSON replay back-compat. #[serde(default)] pub ttft_ms: Option, + /// Freshness provenance for [`Instance::ttft_ms`]. A value with no new + /// requests in the latest scrape window is `Held` until the observation + /// validity window lapses, then cleared. `None` when unknown (legacy + /// snapshots). Omitted from JSON when `None`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub ttft_observation: Option, /// Live average time-per-output-token (ms), windowed from the vLLM TPOT - /// histogram. `None` until two scrapes (or absent); Observe shows `—`. + /// histogram. `None` until a window sees new requests (or absent); + /// Observe shows `—`. #[serde(default)] pub tpot_ms: Option, + /// Freshness provenance for [`Instance::tpot_ms`]; same semantics as + /// [`Instance::ttft_observation`]. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tpot_observation: Option, pub launch_args: Vec, pub env_vars: std::collections::BTreeMap, pub log_file: Option, @@ -257,6 +280,20 @@ mod tests { assert_eq!(inst.kv_cache_usage_pct, Some(42.0)); } + #[test] + fn gpu_metrics_deserializes_without_power_limit() { + // Replay back-compat: recordings made before `power_limit_w` existed + // must load with the limit unknown, never 0. + let legacy = r#"{ + "device_id": "gpu-0", "vram_used_mb": 1, "vram_total_mb": 2, + "gpu_utilization_pct": 3.0, "temperature_c": 4.0, "power_w": 5.0, + "clock_mhz": null + }"#; + let g: GpuMetrics = serde_json::from_str(legacy).expect("legacy GPU must parse"); + assert_eq!(g.power_limit_w, None); + assert!((g.power_w - 5.0).abs() < f32::EPSILON); + } + #[test] fn instance_round_trips_ttft_tpot_through_serde() { // The new live-latency fields survive a serialize → deserialize cycle. @@ -272,6 +309,29 @@ mod tests { assert_eq!(back.tpot_ms, Some(22.0)); } + #[test] + fn instance_latency_observation_is_omitted_when_unknown_and_round_trips() { + // Legacy payloads carry no freshness: it stays `None`, never `Fresh`, + // and is not written back out. + let json = serde_json::to_string(&Instance::default()).expect("serialize"); + assert!(!json.contains("ttft_observation") && !json.contains("tpot_observation")); + let inst = Instance { + ttft_ms: Some(100.0), + ttft_observation: Some(ObservationMetadata { + observed_at: Utc::now(), + freshness: ObservationFreshness::Held, + }), + ..Default::default() + }; + let back: Instance = + serde_json::from_str(&serde_json::to_string(&inst).expect("serialize")).expect("parse"); + assert_eq!( + back.ttft_observation.map(|m| m.freshness), + Some(ObservationFreshness::Held) + ); + assert!(back.tpot_observation.is_none()); + } + #[test] fn instance_status_ready_round_trips_as_lowercase_json() { // The new `Ready` variant must serialize/deserialize the same way the diff --git a/crates/rocm-dash-core/src/traits.rs b/crates/rocm-dash-core/src/traits.rs index 59105f5f03..3719b91ec8 100644 --- a/crates/rocm-dash-core/src/traits.rs +++ b/crates/rocm-dash-core/src/traits.rs @@ -159,10 +159,13 @@ pub fn merge_instance( // a counter-rate delta is computed. gen_tps_observation: None, tokens_per_watt: None, + power_w: None, // Derived at the runner by windowing the sample's cumulative histogram // readings (mirrors gen_tps); left None at merge time. ttft_ms: None, + ttft_observation: None, tpot_ms: None, + tpot_observation: None, launch_args: svc.launch_args.clone(), env_vars: svc.env_vars.clone(), log_file: svc.log_file.clone(), diff --git a/crates/rocm-dash-daemon/src/demo.rs b/crates/rocm-dash-daemon/src/demo.rs index 2690aa4ef9..50ede75a7e 100644 --- a/crates/rocm-dash-daemon/src/demo.rs +++ b/crates/rocm-dash-daemon/src/demo.rs @@ -540,6 +540,8 @@ fn build_gpu(idx: usize, t_s: f64, seed: &mut u64) -> GpuMetrics { gpu_utilization_pct: util, temperature_c: temp, power_w: power, + // MI355X-class board power cap. + power_limit_w: Some(750.0), clock_mhz: Some((jitter(seed) - 0.5).mul_add(80.0, 1850.0)), } } @@ -628,8 +630,11 @@ fn build_instances(t_s: f64, seed: &mut u64) -> Vec { gen_tps: Some(gen_tps), gen_tps_observation: None, tokens_per_watt: None, + power_w: None, ttft_ms, + ttft_observation: None, tpot_ms, + tpot_observation: None, launch_args: c.launch_args.iter().map(|s| (*s).to_string()).collect(), env_vars, log_file: Some(format!("/var/log/vllm/{}.log", c.container_name)), @@ -644,9 +649,14 @@ fn build_snapshot(t_s: f64, base_secs: i64, seed: &mut u64) -> Snapshot { let mut instances = build_instances(t_s, seed); // Mirror the runner: derive tokens_per_watt once both throughput and GPU // power are in hand, exercising the same id-normalizing join. - for inst in &mut instances { - inst.tokens_per_watt = - rocm_dash_core::efficiency::tokens_per_watt(inst.gen_tps, &inst.gpu_ids, &gpus); + let attributed = rocm_dash_core::efficiency::attribute_power_w( + &instances, + &gpus, + &std::collections::HashMap::new(), + ); + for (inst, power_w) in instances.iter_mut().zip(attributed) { + inst.power_w = power_w; + inst.tokens_per_watt = rocm_dash_core::efficiency::tokens_per_watt(inst.gen_tps, power_w); } Snapshot { timestamp, diff --git a/crates/rocm-dash-daemon/src/registry.rs b/crates/rocm-dash-daemon/src/registry.rs index 6f02aa8f4c..44e6275fb5 100644 --- a/crates/rocm-dash-daemon/src/registry.rs +++ b/crates/rocm-dash-daemon/src/registry.rs @@ -65,6 +65,11 @@ pub struct ServiceRecord { /// Absent on older records and once the service reaches `ready`. #[serde(default)] pub startup_phase: Option, + /// GPU indices the service was launched on (`rocm serve --gpu`), empty when + /// none were recorded. These become the instance's `gpu_ids`, which is what + /// joins it to GPU power and VRAM. + #[serde(default)] + pub gpu_indices: Vec, #[serde(default)] pub created_at_unix_ms: u128, } @@ -144,10 +149,9 @@ pub fn load_service_records(services_dir: &Path) -> Vec { /// collector pipeline, or `None` when the record is not in a scrapeable state. /// /// The port is taken verbatim from the registry (`record.port`) — the registry -/// is the authority, never a hardcoded default. `gpu_ids` is left empty because -/// the record carries no concrete device ids; VRAM and tokens/W attribution -/// degrade gracefully (device-summed / `None`) for managed services until a -/// richer device mapping is recorded. +/// is the authority, never a hardcoded default. `gpu_ids` come from the +/// record's `gpu_indices`; when none were recorded they stay empty and VRAM and +/// tokens/W attribution degrade gracefully (device-summed / `None`). pub fn discovered_from_record(record: &ServiceRecord) -> Option { if !is_scrapeable_status(&record.status) { return None; @@ -168,6 +172,7 @@ pub fn discovered_from_record(record: &ServiceRecord) -> Option = HashMap::new(); - // Previous TTFT / TPOT histogram readings (sum_s, count, at) per instance, - // for the windowed avg-ms derivation (mirrors `prev_gen_tokens`). - let mut prev_ttft: HashMap)> = HashMap::new(); - let mut prev_tpot: HashMap)> = HashMap::new(); + // Previous TTFT / TPOT histogram readings plus a freshness tracker per + // instance, for the windowed avg-ms derivation (mirrors `gen_trackers`). + let mut prev_ttft: LatencyWindows = HashMap::new(); + let mut prev_tpot: LatencyWindows = HashMap::new(); // Per-container VRAM (MB) from the last amd-smi `process` scrape. Refreshed // on the instance cadence and reused every tick so the attributed value is // stable between scrapes (mirrors how GPU power drives tokens_per_watt). @@ -406,6 +408,11 @@ pub async fn run_loop( let mut tick_count: u64 = 0; let mut last_sysinfo_refresh: u64 = 0; + // Static per-GPU power caps (`amd-smi static`), refreshed on the sysinfo + // cadence and stamped onto each tick's metrics. `None` until first fetched. + let mut power_limits: HashMap = HashMap::new(); + let mut last_power_limit_refresh: Option = None; + let mut warned_no_power_limits = false; // Discovery/scrape warnings are produced on their own (slower) cadences, so // holding them only for the tick that produced them makes the header ⚠ badge // blink on and off. Keep the last result until that cadence runs again. @@ -465,8 +472,42 @@ pub async fn run_loop( } let mut warnings = Vec::new(); - let gpus = if let Some(g) = &gpu { - match g.metrics().await { + // Power caps are static, so they refresh on the sysinfo cadence. Fetch + // them alongside this tick's metrics instead of ahead of it, so the + // refresh tick pays one amd-smi round trip, not two in a row. + let refresh_limits = gpu.is_some() + && last_power_limit_refresh + .is_none_or(|at| tick_count.saturating_sub(at) >= sysinfo_refresh_ticks); + let mut gpus = if let Some(g) = &gpu { + let (limits, metrics) = tokio::join!( + async { + if refresh_limits { + Some(g.power_limits().await) + } else { + None + } + }, + g.metrics() + ); + if let Some(res) = limits { + match res { + // An empty map (a schema the parser no longer matches) must + // not wipe caps we already hold, and is worth saying once. + Ok(l) if l.is_empty() => { + if power_limits.is_empty() && !warned_no_power_limits { + warn!( + "amd-smi static reported no readable power cap; power stays uncolored" + ); + warned_no_power_limits = true; + } + } + Ok(l) => power_limits = l, + // A failed fetch keeps the previous caps; retry next interval. + Err(_) => {} + } + last_power_limit_refresh = Some(tick_count); + } + match metrics { Ok(v) => v, Err(e) => { warnings.push(format!("amd-smi metric: {e}")); @@ -505,6 +546,10 @@ pub async fn run_loop( Vec::new() }; + for g in &mut gpus { + g.power_limit_w = power_limits.get(&g.device_id).copied(); + } + if gpu.is_some() && tick_count.saturating_sub(last_sysinfo_refresh) >= sysinfo_refresh_ticks && let Some(g) = &gpu @@ -775,17 +820,21 @@ pub async fn run_loop( // assembly below (tracker.snapshot per cycle_at). // // Windowed avg latency (ms) from the cumulative TTFT/TPOT - // histograms; cumulative-average fallback on first scrape. - let ttft_ms = avg_ms_from_histogram( + // histograms. Only a window with new requests produces a + // value; the tracker holds it, then expires it, in the + // snapshot assembly below. + observe_latency( &mut prev_ttft, &id, + opts.instance_tick, sample.ttft_sum_s, sample.ttft_count, cycle_at, ); - let tpot_ms = avg_ms_from_histogram( + observe_latency( &mut prev_tpot, &id, + opts.instance_tick, sample.tpot_sum_s, sample.tpot_count, cycle_at, @@ -794,8 +843,6 @@ pub async fn run_loop( inst.kv_cache_usage_pct = sample.kv_cache_usage_pct; inst.running_reqs = sample.running_reqs; inst.waiting_reqs = sample.waiting_reqs; - inst.ttft_ms = ttft_ms; - inst.tpot_ms = tpot_ms; // Docker-discovered instances have no // `ManagedServiceRecord` to promote their status, so // they start life as `Starting`. A successful vLLM @@ -818,18 +865,11 @@ pub async fn run_loop( // window (clamp(3×instance_tick, 6s, 30s)) and only clears // it after expiry — satisfying the "held" contract. // - // Clear TTFT/TPOT baselines so recovery re-baselines from - // the next successful scrape (latency clearing semantics - // are unchanged; freshness generalization is out of scope). - prev_ttft.remove(&id); - prev_tpot.remove(&id); - if let Some(mut inst) = runner.state.instances.get(&id).cloned() - && (inst.ttft_ms.is_some() || inst.tpot_ms.is_some()) - { - inst.ttft_ms = None; - inst.tpot_ms = None; - runner.state.apply(StateEvent::InstanceUpserted(inst)); - } + // Drop the TTFT/TPOT baselines so recovery re-baselines + // from the next successful scrape; the trackers keep the + // last windowed value Held until its validity lapses. + rebaseline_latency(&mut prev_ttft, &id); + rebaseline_latency(&mut prev_tpot, &id); } } } @@ -921,6 +961,10 @@ pub async fn run_loop( .map_or((None, None), |t| t.snapshot(cycle_at)); inst.gen_tps = gen_tps; inst.gen_tps_observation = meta; + (inst.ttft_ms, inst.ttft_observation) = + snapshot_latency(&mut prev_ttft, &inst.container_id, cycle_at); + (inst.tpot_ms, inst.tpot_observation) = + snapshot_latency(&mut prev_tpot, &inst.container_id, cycle_at); } // Derive per-instance efficiency now that this tick's GPU power is known // and gen_tps reflects the tracker state for this cycle. @@ -928,9 +972,14 @@ pub async fn run_loop( // don't line up with any amd-smi device_id — the silent-None failure // mode on real hardware, surfaced via the header ⚠ badge. let mut id_join_misses = 0usize; - for inst in &mut instances { + // Split each GPU's power among the instances sharing it, so two + // instances on one board don't both claim all of it. + let attributed = + rocm_dash_core::efficiency::attribute_power_w(&instances, &gpus, &per_container_used); + for (inst, power_w) in instances.iter_mut().zip(attributed) { + inst.power_w = power_w; inst.tokens_per_watt = - rocm_dash_core::efficiency::tokens_per_watt(inst.gen_tps, &inst.gpu_ids, &gpus); + rocm_dash_core::efficiency::tokens_per_watt(inst.gen_tps, power_w); if inst.gen_tps.is_some() && !inst.gpu_ids.is_empty() && !gpus.is_empty() @@ -952,7 +1001,12 @@ pub async fn run_loop( if let Some(entry) = runner.state.instances.get_mut(&inst.container_id) { entry.gen_tps = inst.gen_tps; entry.gen_tps_observation = inst.gen_tps_observation.clone(); + entry.ttft_ms = inst.ttft_ms; + entry.ttft_observation = inst.ttft_observation.clone(); + entry.tpot_ms = inst.tpot_ms; + entry.tpot_observation = inst.tpot_observation.clone(); entry.tokens_per_watt = inst.tokens_per_watt; + entry.power_w = inst.power_w; } } // Attribute per-instance VRAM (per-process where the cgroup join hit, @@ -961,14 +1015,7 @@ pub async fn run_loop( // Warn only when a real process scrape happened but an instance with // live GPUs still fell back to device-summed (cgroup join missed). if procs_nonempty { - let vram_fallbacks = instances - .iter() - .filter(|i| { - !i.gpu_ids.is_empty() - && rocm_dash_core::efficiency::gpu_ids_overlap(&i.gpu_ids, &gpus) - && !per_container_used.contains_key(&i.container_id) - }) - .count(); + let vram_fallbacks = count_vram_join_misses(&instances, &gpus, &per_container_used); if vram_fallbacks > 0 { scrape_warns.push(format!( "vram attribution: {vram_fallbacks} instance(s) fell back to device-summed \ @@ -1017,14 +1064,37 @@ pub async fn run_loop( } } +/// Instances that *could* have joined the per-process VRAM map but did not. +/// +/// The map is keyed by docker/containerd ids read from `/proc//cgroup`, +/// so only an instance whose id has that shape can ever hit it. A managed +/// `rocm serve` service (named by its service id) falls back to device-summed +/// VRAM by design, and counting it would raise a false "check /proc access" +/// warning on every healthy managed instance. +fn count_vram_join_misses( + instances: &[Instance], + gpus: &[GpuMetrics], + per_container_used: &HashMap, +) -> usize { + instances + .iter() + .filter(|i| { + rocm_dash_collectors::cgroup::is_container_id(&i.container_id) + && !i.gpu_ids.is_empty() + && rocm_dash_core::efficiency::gpu_ids_overlap(&i.gpu_ids, gpus) + && !per_container_used.contains_key(&i.container_id) + }) + .count() +} + /// Remove all per-service telemetry state for `id`: tracker, TTFT, and TPOT /// baselines. Call on explicit removal (Docker Gone, Lemonade Gone, managed /// Gone) and whenever an identity change requires a clean slate. pub(crate) fn purge_telemetry_state( id: &str, gen_trackers: &mut HashMap, - prev_ttft: &mut HashMap)>, - prev_tpot: &mut HashMap)>, + prev_ttft: &mut LatencyWindows, + prev_tpot: &mut LatencyWindows, ) { gen_trackers.remove(id); prev_ttft.remove(id); @@ -1038,8 +1108,8 @@ pub(crate) fn purge_telemetry_state( pub(crate) fn invalidate_on_endpoint_change( id: &str, gen_trackers: &mut HashMap, - prev_ttft: &mut HashMap)>, - prev_tpot: &mut HashMap)>, + prev_ttft: &mut LatencyWindows, + prev_tpot: &mut LatencyWindows, ) { if let Some(t) = gen_trackers.get_mut(id) { t.invalidate(); @@ -1085,8 +1155,11 @@ pub(crate) fn merge_discovery_refresh( inst.gen_tps = ex.gen_tps; inst.gen_tps_observation = ex.gen_tps_observation.clone(); inst.tokens_per_watt = ex.tokens_per_watt; + inst.power_w = ex.power_w; inst.ttft_ms = ex.ttft_ms; + inst.ttft_observation = ex.ttft_observation.clone(); inst.tpot_ms = ex.tpot_ms; + inst.tpot_observation = ex.tpot_observation.clone(); } inst } @@ -1137,46 +1210,78 @@ fn ticks_per(period: Duration, tick: Duration) -> u64 { n.max(1) as u64 } -/// Upper bound on the scrape interval accepted by `avg_ms_from_histogram`. +/// Upper bound on the scrape interval accepted by [`observe_latency`]. /// Applies to TTFT/TPOT latency histograms only; generation-counter staleness /// is managed by `GenerationObservationTracker` (see `rocm-dash-core`). -/// A gap longer than this makes the windowed delta stale (outage or forward -/// wall-clock jump / NTP / VM resume), so the histogram falls back to the -/// cumulative average for that sample. +/// A gap longer than this (outage, forward wall-clock jump / NTP / VM resume) +/// makes the windowed delta meaningless, so that sample yields no value and +/// only re-baselines. const MAX_RATE_INTERVAL_S: f64 = 60.0; -/// Average latency (ms) from a cumulative histogram's `_sum` (seconds) and -/// `_count` series, windowed over successive scrapes: `(Δsum / Δcount) × 1000`. -/// Updates the per-instance baseline in `prev`. On the FIRST reading (or after a -/// counter reset / stale interval / no new requests) it falls back to the -/// cumulative average `sum/count × 1000`, so a single scrape already yields a -/// value. Returns `None` when the histogram is absent (`sum`/`count` `None`) or -/// `count == 0` — Observe then shows `—` (never a fabricated number). -fn avg_ms_from_histogram( - prev: &mut HashMap)>, +/// Per-instance latency state for one histogram: the previous cumulative +/// reading and the tracker that holds and then expires the derived average. +pub(crate) struct LatencyWindow { + /// Previous `(sum_s, count, at)` reading; `None` until baselined. + prev: Option<(f64, f64, DateTime)>, + /// Holds the last windowed average for the observation validity window. + tracker: GenerationObservationTracker, +} + +pub(crate) type LatencyWindows = HashMap; + +/// Feed one scrape of a cumulative latency histogram (`_sum` in seconds, +/// `_count`) into `id`'s window. A value is recorded only when the window saw +/// new requests: `(Δsum / Δcount) × 1000` ms. The first reading, a counter +/// reset, a stale interval, or a window with no new requests only moves the +/// baseline; the lifetime average is never reported, so an idle server's old +/// latency ages out instead of persisting. An absent histogram (`None`) leaves +/// the window untouched. +fn observe_latency( + windows: &mut LatencyWindows, id: &str, + instance_tick: Duration, sum_s: Option, count: Option, now: DateTime, -) -> Option { - let (sum_s, count) = (sum_s?, count?); - let last = prev.insert(id.to_string(), (sum_s, count, now)); - if let Some((psum, pcount, pat)) = last { +) { + let (Some(sum_s), Some(count)) = (sum_s, count) else { + return; + }; + let w = windows + .entry(id.to_string()) + .or_insert_with(|| LatencyWindow { + prev: None, + tracker: GenerationObservationTracker::new(instance_tick), + }); + if let Some((psum, pcount, pat)) = w.prev.replace((sum_s, count, now)) { let dcount = count - pcount; let dsum = sum_s - psum; let dt = (now - pat).num_milliseconds() as f64 / 1000.0; if dcount > 0.0 && dsum >= 0.0 && dt > 0.0 && dt <= MAX_RATE_INTERVAL_S { - return Some(dsum / dcount * 1000.0); + w.tracker.observe_direct(dsum / dcount * 1000.0, now); } } - // Cumulative-average fallback: first scrape, no new requests, or a reset. - if count > 0.0 { - Some(sum_s / count * 1000.0) - } else { - None +} + +/// Forget the previous reading after a failed scrape so recovery re-baselines, +/// while the tracker keeps the last value held until it expires. +fn rebaseline_latency(windows: &mut LatencyWindows, id: &str) { + if let Some(w) = windows.get_mut(id) { + w.prev = None; } } +/// Current held/fresh/expired latency for `id` at `now`. +fn snapshot_latency( + windows: &mut LatencyWindows, + id: &str, + now: DateTime, +) -> (Option, Option) { + windows + .get_mut(id) + .map_or((None, None), |w| w.tracker.snapshot(now)) +} + #[cfg(test)] mod tests { use super::*; @@ -1340,36 +1445,120 @@ mod tests { Utc.timestamp_opt(secs, 0).unwrap() } + use rocm_dash_core::metrics::ObservationFreshness; + + const TICK: Duration = Duration::from_secs(2); + + fn baseline(sum_s: f64, count: f64) -> LatencyWindow { + LatencyWindow { + prev: Some((sum_s, count, at(10))), + tracker: GenerationObservationTracker::new(TICK), + } + } + + fn snap_at( + w: &mut LatencyWindows, + id: &str, + secs: i64, + ) -> (Option, Option) { + let (v, m) = snapshot_latency(w, id, at(secs)); + (v, m.map(|m| m.freshness)) + } + + fn inst_on(id: &str, gpu: &str) -> Instance { + Instance { + container_id: id.into(), + gpu_ids: vec![gpu.into()], + ..Instance::default() + } + } + #[test] - fn avg_ms_first_scrape_uses_cumulative_then_windows() { - let mut prev = HashMap::new(); - // First scrape: cumulative average = 2.0s / 4 × 1000 = 500 ms. + fn managed_instances_never_count_as_vram_join_misses() { + let docker_id = "ab".repeat(32); + let gpus = [gpu("gpu-0", 1000, 8000)]; + let insts = [ + inst_on("vllm-qwen-qwen2-5-0-5b-instru-1793361705247", "0"), + inst_on(&docker_id, "0"), + ]; + // A docker instance with no /proc match is a real miss; the managed + // service, which could never match, is not. + let empty: HashMap = HashMap::new(); + assert_eq!(count_vram_join_misses(&insts, &gpus, &empty), 1); + // Once the docker instance joins, nothing is missed. + let joined: HashMap = [(docker_id, 512)].into(); + assert_eq!(count_vram_join_misses(&insts, &gpus, &joined), 0); + } + + #[test] + fn latency_first_scrape_reports_nothing_then_windows() { + let mut w = LatencyWindows::new(); + // First scrape only baselines: the lifetime average is not reported. + observe_latency(&mut w, "a", TICK, Some(2.0), Some(4.0), at(10)); + assert_eq!(snap_at(&mut w, "a", 10), (None, None)); + // Second scrape: Δsum=1.0s over Δcount=10 → 100 ms, fresh. + observe_latency(&mut w, "a", TICK, Some(3.0), Some(14.0), at(12)); assert_eq!( - avg_ms_from_histogram(&mut prev, "a", Some(2.0), Some(4.0), at(10)), - Some(500.0) + snap_at(&mut w, "a", 12), + (Some(100.0), Some(ObservationFreshness::Fresh)) ); - // Second scrape: Δsum=1.0s over Δcount=10 → 100 ms (windowed). + } + + #[test] + fn latency_is_held_without_new_requests_then_expires() { + let mut w = LatencyWindows::new(); + observe_latency(&mut w, "a", TICK, Some(2.0), Some(4.0), at(10)); + observe_latency(&mut w, "a", TICK, Some(3.0), Some(14.0), at(12)); + // Idle scrapes: counters unchanged → no new value, previous is held. + observe_latency(&mut w, "a", TICK, Some(3.0), Some(14.0), at(14)); assert_eq!( - avg_ms_from_histogram(&mut prev, "a", Some(3.0), Some(14.0), at(12)), - Some(100.0) + snap_at(&mut w, "a", 14), + (Some(100.0), Some(ObservationFreshness::Held)) ); - // Absent histogram → None (Observe shows `—`). + // Validity for a 2 s tick is the 6 s floor: still held at +6 s... assert_eq!( - avg_ms_from_histogram(&mut prev, "b", None, None, at(12)), - None + snap_at(&mut w, "a", 18), + (Some(100.0), Some(ObservationFreshness::Held)) ); - // count == 0 (no requests yet) → None, never a divide-by-zero number. + // ...and gone just past it. The stale lifetime average never returns. + observe_latency(&mut w, "a", TICK, Some(3.0), Some(14.0), at(19)); + assert_eq!(snap_at(&mut w, "a", 19), (None, None)); + } + + #[test] + fn latency_absent_empty_reset_and_stale_interval_report_nothing() { + let mut w = LatencyWindows::new(); + // Absent histogram / zero count never fabricate a number. + observe_latency(&mut w, "b", TICK, None, None, at(12)); + assert_eq!(snap_at(&mut w, "b", 12), (None, None)); + observe_latency(&mut w, "c", TICK, Some(0.0), Some(0.0), at(12)); + observe_latency(&mut w, "c", TICK, Some(0.0), Some(0.0), at(14)); + assert_eq!(snap_at(&mut w, "c", 14), (None, None)); + // A counter reset (sum drops) only re-baselines. + observe_latency(&mut w, "d", TICK, Some(10.0), Some(100.0), at(10)); + observe_latency(&mut w, "d", TICK, Some(0.2), Some(2.0), at(12)); + assert_eq!(snap_at(&mut w, "d", 12), (None, None)); + // A gap beyond MAX_RATE_INTERVAL_S is not a usable window. + observe_latency(&mut w, "e", TICK, Some(1.0), Some(10.0), at(0)); + observe_latency(&mut w, "e", TICK, Some(2.0), Some(20.0), at(100)); + assert_eq!(snap_at(&mut w, "e", 100), (None, None)); + } + + #[test] + fn failed_scrape_rebaselines_but_keeps_value_held() { + let mut w = LatencyWindows::new(); + observe_latency(&mut w, "a", TICK, Some(2.0), Some(4.0), at(10)); + observe_latency(&mut w, "a", TICK, Some(3.0), Some(14.0), at(12)); + rebaseline_latency(&mut w, "a"); assert_eq!( - avg_ms_from_histogram(&mut prev, "c", Some(0.0), Some(0.0), at(12)), - None + snap_at(&mut w, "a", 14), + (Some(100.0), Some(ObservationFreshness::Held)) ); - // A counter reset (sum drops) re-bases to the cumulative average. - let mut p2 = HashMap::new(); - avg_ms_from_histogram(&mut p2, "d", Some(10.0), Some(100.0), at(10)); + // Recovery re-baselines: the first post-failure scrape yields nothing new. + observe_latency(&mut w, "a", TICK, Some(9.0), Some(50.0), at(15)); assert_eq!( - avg_ms_from_histogram(&mut p2, "d", Some(0.2), Some(2.0), at(12)), - Some(100.0), - "reset falls back to cumulative 0.2/2×1000" + snap_at(&mut w, "a", 15), + (Some(100.0), Some(ObservationFreshness::Held)) ); } @@ -1497,16 +1686,16 @@ mod tests { #[test] fn purge_telemetry_state_removes_all_three_maps_for_id() { let mut trackers: HashMap = HashMap::new(); - let mut ttft: HashMap)> = HashMap::new(); - let mut tpot: HashMap)> = HashMap::new(); + let mut ttft: LatencyWindows = HashMap::new(); + let mut tpot: LatencyWindows = HashMap::new(); // Populate two ids. trackers.insert("a".into(), make_tracker(2)); trackers.insert("b".into(), make_tracker(2)); - ttft.insert("a".into(), (1.0, 1.0, at(10))); - ttft.insert("b".into(), (2.0, 2.0, at(10))); - tpot.insert("a".into(), (3.0, 3.0, at(10))); - tpot.insert("b".into(), (4.0, 4.0, at(10))); + ttft.insert("a".into(), baseline(1.0, 1.0)); + ttft.insert("b".into(), baseline(2.0, 2.0)); + tpot.insert("a".into(), baseline(3.0, 3.0)); + tpot.insert("b".into(), baseline(4.0, 4.0)); purge_telemetry_state("a", &mut trackers, &mut ttft, &mut tpot); @@ -1526,16 +1715,16 @@ mod tests { #[test] fn invalidate_on_endpoint_change_resets_tracker_and_clears_latency() { let mut trackers: HashMap = HashMap::new(); - let mut ttft: HashMap)> = HashMap::new(); - let mut tpot: HashMap)> = HashMap::new(); + let mut ttft: LatencyWindows = HashMap::new(); + let mut tpot: LatencyWindows = HashMap::new(); let mut tracker = make_tracker(2); // Give the tracker a reading so it has a non-None baseline. tracker.observe_counter(100.0, at(10)); tracker.observe_counter(200.0, at(12)); trackers.insert("svc".into(), tracker); - ttft.insert("svc".into(), (1.0, 10.0, at(10))); - tpot.insert("svc".into(), (0.5, 8.0, at(10))); + ttft.insert("svc".into(), baseline(1.0, 10.0)); + tpot.insert("svc".into(), baseline(0.5, 8.0)); invalidate_on_endpoint_change("svc", &mut trackers, &mut ttft, &mut tpot); diff --git a/crates/rocm-dash-daemon/tests/telemetry_contract.rs b/crates/rocm-dash-daemon/tests/telemetry_contract.rs index 577ab6f8b5..c15589325d 100644 --- a/crates/rocm-dash-daemon/tests/telemetry_contract.rs +++ b/crates/rocm-dash-daemon/tests/telemetry_contract.rs @@ -649,6 +649,168 @@ async fn gen_tps_held_for_validity_window_after_single_failure() { ); } +// ── TTFT freshness (ROCMAI-62) ────────────────────────────────────────────── + +/// Extract the `contract-svc` instance's TTFT and its freshness. +/// Outer `None` = instance absent. +#[allow(clippy::type_complexity)] +fn svc_ttft( + snap: &rocm_dash_core::metrics::Snapshot, +) -> Option<( + Option, + Option, +)> { + snap.instances + .iter() + .find(|i| i.container_id == SVC_ID) + .map(|i| (i.ttft_ms, i.ttft_observation.as_ref().map(|m| m.freshness))) +} + +/// Contract: once new requests stop arriving, the last windowed TTFT is held +/// (`Held`) for the observation validity window and then cleared. It is never +/// replaced by the histogram's lifetime average, which the mock's constant +/// `Frozen` body would otherwise report as a steady 50 ms indefinitely. +/// +/// Exercises the real runner loop: scrape, per-instance tracker, snapshot. +#[tokio::test] +async fn ttft_is_held_then_cleared_when_requests_stop() { + use rocm_dash_core::metrics::ObservationFreshness::{Fresh, Held}; + + let tmp = tempfile::TempDir::new().unwrap(); + let services_dir = tmp.path().join("services"); + + let mode = Arc::new(Mutex::new(MockMode::Growing)); + let ticks = Arc::new(AtomicU64::new(0)); + let stop = Arc::new(AtomicBool::new(false)); + let failure_count = Arc::new(AtomicU64::new(0)); + let port = spawn_mock_server( + Arc::clone(&mode), + Arc::clone(&ticks), + Arc::clone(&failure_count), + Arc::clone(&stop), + ) + .await; + + write_service_record(&services_dir, port); + let (tx, mut rx) = broadcast::channel::(64); + let _runner = spawn_runner(tx, fast_opts(services_dir)).await; + + // Step 1: traffic is flowing, so TTFT is a live (Fresh) windowed value. + let live = wait_for_snapshot(&mut rx, |snap| { + svc_ttft(snap).and_then(|(v, f)| (f == Some(Fresh)).then_some(v).flatten()) + }) + .await + .unwrap_or_else(|e| panic!("fresh TTFT never established: {e}")); + assert!(live > 0.0, "windowed TTFT should be positive, got {live}"); + + // Step 2: the engine goes idle: histograms stop advancing. + drain_snapshots(&mut rx); + *mode + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = MockMode::Frozen(1000); + + // Step 3: the value is held, marked as such, rather than dropped or + // silently presented as live. + let held = wait_for_snapshot(&mut rx, |snap| { + svc_ttft(snap).and_then(|(v, f)| (f == Some(Held)).then_some(v).flatten()) + }) + .await + .unwrap_or_else(|e| panic!("TTFT was never reported as Held once idle: {e}")); + assert!(held > 0.0); + + // Step 4: after the validity window it clears, and stays clear. The + // pre-fix behaviour would keep reporting the 50 ms lifetime average here. + let cleared = wait_for_snapshot(&mut rx, |snap| { + svc_ttft(snap).and_then(|(v, _)| v.is_none().then_some(())) + }) + .await; + stop.store(true, Ordering::Relaxed); + assert!( + cleared.is_ok(), + "idle TTFT was never cleared: stale latency persisted past the validity window ({:?})", + observation_validity_window() + ); +} + +// ── GPU power limit and shared-GPU attribution (ROCMAI-62) ────────────────── + +/// Contract: with `amd-smi` reporting one 400 W GPU capped at 750 W and two +/// managed services launched on that GPU, the broadcast snapshot carries the +/// cap on the GPU and splits the board power between the two instances +/// instead of crediting each with all of it. Drives the real runner with a +/// fake `amd-smi`; amd-smi itself is not needed on the host. +#[cfg(unix)] +#[tokio::test] +async fn shared_gpu_power_is_split_and_limit_is_reported() { + use rocm_core::fake_executable::write_fake_executable; + + let tmp = tempfile::TempDir::new().unwrap(); + let services_dir = tmp.path().join("services"); + std::fs::create_dir_all(&services_dir).unwrap(); + for id in ["shared-a", "shared-b"] { + // Unscraped ports: this scenario is about power, not Prometheus. + let json = format!( + r#"{{"service_id":"{id}","engine":"vllm","model_ref":"M/{id}","canonical_model_id":"M/{id}","host":"127.0.0.1","port":9,"status":"ready","gpu_indices":[0],"created_at_unix_ms":0}}"# + ); + std::fs::write(services_dir.join(format!("{id}.json")), json).unwrap(); + } + + let fake = tmp.path().join("amd-smi-fake"); + write_fake_executable( + &fake, + r#"#!/bin/sh +case "$1" in + metric) echo '{"gpu_data":[{"gpu":0,"mem_usage":{"used_vram":{"value":1000},"total_vram":{"value":8000}},"usage":{"gfx_activity":{"value":50}},"temperature":{"hotspot":{"value":60}},"power":{"socket_power":{"value":400.0,"unit":"W"}}}]}' ;; + static) echo '{"gpu_data":[{"gpu":0,"limit":{"max_power":{"value":750,"unit":"W"}}}]}' ;; + process) echo '{"gpu_data":[{"gpu":0,"process_list":[{"process_info":{"pid":1,"memory_usage":{"vram_mem":{"value":100,"unit":"MB"}}}}]}]}' ;; + *) echo '{}' ;; +esac +"#, + ) + .unwrap(); + + let (tx, mut rx) = broadcast::channel::(64); + let opts = RunnerOptions { + amd_smi_binary: Some(fake.into_os_string()), + amd_smi_skip_device_preflight: true, + ..fast_opts(services_dir) + }; + let _runner = spawn_runner(tx, opts).await; + + let (limit, a, b, warnings) = wait_for_snapshot(&mut rx, |snap| { + let gpu = snap.gpus.first()?; + let power = |id: &str| { + snap.instances + .iter() + .find(|i| i.container_id == id) + .and_then(|i| i.power_w) + }; + Some(( + gpu.power_limit_w?, + power("shared-a")?, + power("shared-b")?, + snap.warnings.clone(), + )) + }) + .await + .unwrap_or_else(|e| panic!("limit and attributed power never appeared: {e}")); + + assert!((limit - 750.0).abs() < f32::EPSILON, "limit was {limit}"); + // Neither service joins the per-container VRAM map (managed services can + // not), so the board is split equally and the two shares add back up to + // what the GPU actually draws. + assert!((a - 200.0).abs() < 1e-6, "shared-a got {a} W"); + assert!((b - 200.0).abs() < 1e-6, "shared-b got {b} W"); + // amd-smi lists a GPU process, so the VRAM scrape is "real", but that + // process is in no container the cgroup join can name. Managed services + // can never join that map by design, so this must not read as a /proc + // problem. + assert!( + !warnings.iter().any(|w| w.starts_with("vram attribution")), + "false vram-attribution warning for healthy managed services: {warnings:?}" + ); +} + // ── Scenario 5: Omitted counter → None (not zero, not panic) ─────────────── /// Contract: when the `/metrics` endpoint returns HTTP 200 but the body omits diff --git a/crates/rocm-dash-tui/src/agent/snapshot.rs b/crates/rocm-dash-tui/src/agent/snapshot.rs index 5e3ce15c17..43ebe0d5b2 100644 --- a/crates/rocm-dash-tui/src/agent/snapshot.rs +++ b/crates/rocm-dash-tui/src/agent/snapshot.rs @@ -10,7 +10,7 @@ use serde_json::{Value, json}; -use rocm_dash_core::metrics::{GpuMetrics, ObservationFreshness}; +use rocm_dash_core::metrics::{GpuMetrics, ObservationFreshness, ObservationMetadata}; use super::StateSnapshot; @@ -24,6 +24,7 @@ fn gpu_json(g: &GpuMetrics) -> Value { "gpu_utilization_pct": g.gpu_utilization_pct, "temperature_c": g.temperature_c, "power_w": g.power_w, + "power_limit_w": g.power_limit_w, "vram_used_mb": g.vram_used_mb, "vram_total_mb": g.vram_total_mb, }) @@ -42,13 +43,33 @@ pub fn gpu_status_json(snap: &StateSnapshot, gpu_index: Option) -> Value } } +/// `"fresh"` | `"held"` | `null` (unknown, e.g. a legacy snapshot). +fn freshness_label(obs: Option<&ObservationMetadata>) -> Value { + obs.map_or(Value::Null, |m| { + Value::String( + match m.freshness { + ObservationFreshness::Fresh => "fresh", + ObservationFreshness::Held => "held", + } + .into(), + ) + }) +} + /// Discovered serving instances with gen_tps, daemon-provided tok/W, /// freshness metadata, and observed_at timestamp. /// /// Fields added by EAI-7960: /// - `gen_tps`: live generation throughput from the Instance (same as daemon) /// - `tokens_per_watt`: daemon-provided efficiency (never recomputed here) -/// - `freshness`: `"fresh"` | `"held"` | `null` (null for legacy/unknown) +/// - `attributed_power_w`: this instance's share of its GPUs' board power +/// (split among instances sharing a GPU). Not the same as `gpu_status`'s +/// `power_w`, which is the whole board. +/// - `ttft_ms` / `tpot_ms` with `ttft_freshness` / `tpot_freshness` +/// (`"fresh"` | `"held"` | `null`): a held latency is the last value from a +/// window that saw requests, not a live reading +/// - `freshness`: `"fresh"` | `"held"` | `null` (null for legacy/unknown), +/// for `gen_tps` /// - `observed_at`: ISO-8601 UTC string when present, `null` when absent pub fn list_instances_json(snap: &StateSnapshot) -> Value { let arr: Vec = snap @@ -77,6 +98,11 @@ pub fn list_instances_json(snap: &StateSnapshot) -> Value { "waiting_reqs": i.waiting_reqs, "gen_tps": i.gen_tps, "tokens_per_watt": i.tokens_per_watt, + "attributed_power_w": i.power_w, + "ttft_ms": i.ttft_ms, + "ttft_freshness": freshness_label(i.ttft_observation.as_ref()), + "tpot_ms": i.tpot_ms, + "tpot_freshness": freshness_label(i.tpot_observation.as_ref()), "freshness": freshness, "observed_at": observed_at, }) @@ -239,6 +265,59 @@ mod tests { ); } + #[test] + fn list_instances_json_reports_latency_freshness_and_attributed_power() { + use chrono::Utc; + use rocm_dash_core::metrics::ObservationMetadata; + let held = ObservationMetadata { + observed_at: Utc::now(), + freshness: ObservationFreshness::Held, + }; + let inst = Instance { + container_name: "vllm-lat".into(), + ttft_ms: Some(150.0), + ttft_observation: Some(held), + tpot_ms: Some(22.0), + power_w: Some(123.0), + ..Default::default() + }; + let mut snap = fixture_snapshot(); + snap.instances = vec![inst]; + let v = list_instances_json(&snap); + let i = &v["instances"][0]; + assert_eq!(i["ttft_ms"], 150.0); + assert_eq!(i["ttft_freshness"], "held"); + // No metadata is unknown, never reported as fresh. + assert_eq!(i["tpot_ms"], 22.0); + assert!(i["tpot_freshness"].is_null()); + assert_eq!(i["attributed_power_w"], 123.0); + } + + #[test] + fn gpu_status_json_reports_the_power_limit_or_null() { + use rocm_dash_core::metrics::Snapshot; + let mut snap = fixture_snapshot(); + let mut g = GpuMetrics { + device_id: "gpu-0".into(), + power_w: 700.0, + power_limit_w: Some(750.0), + ..GpuMetrics::default() + }; + snap.latest = Some(Snapshot { + gpus: vec![g.clone()], + ..Snapshot::default() + }); + let v = gpu_status_json(&snap, None); + assert_eq!(v["gpus"][0]["power_limit_w"], 750.0); + // Unknown stays null: never 0, never another part's limit. + g.power_limit_w = None; + snap.latest = Some(Snapshot { + gpus: vec![g], + ..Snapshot::default() + }); + assert!(gpu_status_json(&snap, None)["gpus"][0]["power_limit_w"].is_null()); + } + #[test] fn list_instances_json_held_freshness_field() { let snap = fixture_with_observation(rocm_dash_core::metrics::ObservationFreshness::Held); diff --git a/crates/rocm-dash-tui/src/agent/tools.rs b/crates/rocm-dash-tui/src/agent/tools.rs index 113ece0518..1b3b19d4c7 100644 --- a/crates/rocm-dash-tui/src/agent/tools.rs +++ b/crates/rocm-dash-tui/src/agent/tools.rs @@ -112,7 +112,11 @@ impl Tool for ListInstancesTool { ToolDefinition { name: Self::NAME.to_string(), description: "List discovered serving instances: name, model, status, \ - KV-cache usage %, and running/waiting request counts." + KV-cache usage %, running/waiting request counts, \ + generation tok/s, TTFT/TPOT (each with a fresh/held \ + flag; held means the last value, not a live reading), \ + and attributed_power_w (this instance's share of its \ + GPUs' power, not whole-board power)." .to_string(), parameters: json!({ "type": "object", "properties": {} }), } @@ -166,7 +170,8 @@ impl Tool for TokensPerWattTool { ToolDefinition { name: Self::NAME.to_string(), description: "Per-instance efficiency: generation tokens/sec divided by \ - the summed power (W) of the GPUs each instance occupies." + the instance's attributed share of its GPUs' power (W). \ + GPUs shared by several instances are split between them." .to_string(), parameters: json!({ "type": "object", "properties": {} }), } diff --git a/crates/rocm-dash-tui/src/ui/format.rs b/crates/rocm-dash-tui/src/ui/format.rs index cd6dbaa62a..6eb0fcc1c5 100644 --- a/crates/rocm-dash-tui/src/ui/format.rs +++ b/crates/rocm-dash-tui/src/ui/format.rs @@ -245,6 +245,25 @@ pub fn gen_tps_cell(tps: Option, obs: Option<&ObservationMetadata>) -> Stri } } +/// Latency (TTFT/TPOT) for narrow table cells. +/// +/// - `None` / non-finite → `"—"` +/// - `Some(v)`, Held → `"{v:.0}ms*"` +/// - `Some(v)`, Fresh or unknown metadata → `"{v:.0}ms"` +pub fn latency_cell(ms: Option, obs: Option<&ObservationMetadata>) -> String { + match ms.filter(|v| v.is_finite()) { + None => "—".to_string(), + Some(v) => { + let base = format!("{v:.0}ms"); + if obs.is_some_and(|m| m.freshness == ObservationFreshness::Held) { + format!("{base}{HELD_MARKER}") + } else { + base + } + } + } +} + /// Full-unit gen_tps for pane rows and services views. /// Mirrors [`tps_opt`] format and appends [`HELD_MARKER`] when metadata is Held. /// @@ -609,6 +628,16 @@ mod tests { } } + #[test] + fn latency_cell_marks_held_and_never_prints_non_finite() { + assert_eq!(latency_cell(None, None), "—"); + assert_eq!(latency_cell(Some(f64::NAN), Some(&held_obs())), "—"); + assert_eq!(latency_cell(Some(150.4), Some(&held_obs())), "150ms*"); + assert_eq!(latency_cell(Some(150.4), Some(&fresh_obs())), "150ms"); + // Unknown freshness (legacy snapshot) is never shown as held. + assert_eq!(latency_cell(Some(150.4), None), "150ms"); + } + fn held_obs() -> ObservationMetadata { ObservationMetadata { observed_at: DateTime::from_timestamp(1_700_000_000, 0).unwrap(), diff --git a/crates/rocm-dash-tui/src/ui/tabs/hardware.rs b/crates/rocm-dash-tui/src/ui/tabs/hardware.rs index 4314a95bbc..2474b32437 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/hardware.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/hardware.rs @@ -393,7 +393,7 @@ pub fn gpu_compact_line( ), Span::styled( format!("{:>4.0} W", g.power_w), - power_style(g.power_w, theme), + power_style(g.power_w, g.power_limit_w, theme), ), ]; let base_w: usize = spans.iter().map(|s| s.content.chars().count()).sum(); @@ -673,6 +673,7 @@ mod tests { gpu_utilization_pct: util, temperature_c: temp, power_w: power, + power_limit_w: None, clock_mhz: Some(1850.0), } } @@ -716,7 +717,8 @@ mod tests { fn gpu_compact_line_colors_and_truncates() { let theme = Theme::default_dark(); // hot + high power → err color on both temp and power spans - let hot = mk_gpu("gpu-2", 95.0, 88.0, 740.0); + let mut hot = mk_gpu("gpu-2", 95.0, 88.0, 740.0); + hot.power_limit_w = Some(750.0); let line = gpu_compact_line(&hot, &theme, 120, true); let temp_span = line .spans @@ -726,6 +728,15 @@ mod tests { let pow_span = line.spans.iter().find(|s| s.content.contains('W')).unwrap(); assert_eq!(temp_span.style.fg, Some(theme.err)); assert_eq!(pow_span.style.fg, Some(theme.err)); + // same draw but no known power limit → power is not judged (plain fg) + hot.power_limit_w = None; + let unjudged = gpu_compact_line(&hot, &theme, 120, true); + let pow = unjudged + .spans + .iter() + .find(|s| s.content.contains('W')) + .unwrap(); + assert_eq!(pow.style.fg, Some(theme.fg)); // wide → vram segment included assert!(line.spans.iter().any(|s| s.content.contains("GiB"))); // narrow → vram segment dropped (truncation by omission) diff --git a/crates/rocm-dash-tui/src/ui/tabs/home.rs b/crates/rocm-dash-tui/src/ui/tabs/home.rs index b4a4ba043b..e49cdeb872 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/home.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/home.rs @@ -724,6 +724,7 @@ mod tests { gpu_utilization_pct: 62.0, temperature_c: 51.0, power_w: 420.0, + power_limit_w: None, clock_mhz: Some(2100.0), }], gpu_system_info: Some(GpuSystemInfo { @@ -785,6 +786,7 @@ mod tests { gpu_utilization_pct: 0.0, temperature_c: 0.0, power_w: 0.0, + power_limit_w: None, clock_mhz: None, }], gpu_system_info: None, @@ -837,6 +839,7 @@ mod tests { gpu_utilization_pct: 12.0, temperature_c: 0.0, power_w: 0.0, + power_limit_w: None, clock_mhz: None, }], gpu_system_info: Some(GpuSystemInfo { @@ -978,6 +981,7 @@ mod tests { gpu_utilization_pct: 0.5, temperature_c: 0.0, power_w: 0.0, + power_limit_w: None, clock_mhz: None, }], ..Default::default() @@ -995,6 +999,7 @@ mod tests { gpu_utilization_pct: 0.0, temperature_c: 0.0, power_w: 0.0, + power_limit_w: None, clock_mhz: None, }; let snap = Snapshot { diff --git a/crates/rocm-dash-tui/src/ui/tabs/instances.rs b/crates/rocm-dash-tui/src/ui/tabs/instances.rs index d549767119..ce7370da83 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/instances.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/instances.rs @@ -12,7 +12,9 @@ use ratatui::style::{Modifier, Style}; use ratatui::text::{Line, Span}; use ratatui::widgets::{Cell, Paragraph, Row, Table}; -use rocm_dash_core::metrics::{Instance, InstanceStatus, ObservationFreshness}; +use rocm_dash_core::metrics::{ + Instance, InstanceStatus, ObservationFreshness, ObservationMetadata, +}; use crate::app::{AppState, ConnState}; use crate::ui::format; @@ -33,8 +35,6 @@ pub fn draw_table(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { } let instances = sorted_instances(&state.instances); let sel = clamp_sel(state.instance_sel, instances.len()); - let gpus: &[rocm_dash_core::metrics::GpuMetrics] = - state.latest.as_ref().map_or(&[], |s| s.gpus.as_slice()); let inner = panel::bento( f, @@ -80,8 +80,15 @@ pub fn draw_table(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { .gen_tps_observation .as_ref() .is_some_and(|m| m.freshness == ObservationFreshness::Held); - held && (inst.gen_tps.is_some_and(f64::is_finite) - || inst.tokens_per_watt.is_some_and(f64::is_finite)) + let is_held = |m: &Option| { + m.as_ref() + .is_some_and(|m| m.freshness == ObservationFreshness::Held) + }; + (held + && (inst.gen_tps.is_some_and(f64::is_finite) + || inst.tokens_per_watt.is_some_and(f64::is_finite))) + || (is_held(&inst.ttft_observation) && inst.ttft_ms.is_some_and(f64::is_finite)) + || (is_held(&inst.tpot_observation) && inst.tpot_ms.is_some_and(f64::is_finite)) }); let (table_area, legend_area) = if any_held { let split = Layout::default() @@ -108,15 +115,11 @@ pub fn draw_table(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { let tps = format::gen_tps_cell(inst.gen_tps, inst.gen_tps_observation.as_ref()); let tpw = format::tokens_per_watt_cell(inst.tokens_per_watt, inst.gen_tps_observation.as_ref()); - let ttft = inst - .ttft_ms - .filter(|v| v.is_finite()) - .map_or_else(|| dash.to_string(), |v| format!("{v:.0}ms")); - let tpot = inst - .tpot_ms - .filter(|v| v.is_finite()) - .map_or_else(|| dash.to_string(), |v| format!("{v:.0}ms")); - let power = rocm_dash_core::efficiency::instance_power_w(&inst.gpu_ids, gpus) + let ttft = format::latency_cell(inst.ttft_ms, inst.ttft_observation.as_ref()); + let tpot = format::latency_cell(inst.tpot_ms, inst.tpot_observation.as_ref()); + let power = inst + .power_w + .filter(|w| w.is_finite()) .map_or_else(|| dash.to_string(), |w| format!("{w:.0}W")); let queue = match (inst.running_reqs, inst.waiting_reqs) { (None, None) => dash.to_string(), @@ -155,8 +158,10 @@ pub fn draw_table(f: &mut Frame, area: Rect, state: &AppState, theme: &Theme) { // Wide enough for "{v:.2} tok/W*" (tokens_per_watt_cell's longest // rendering) so the held marker never gets clipped mid-unit. Constraint::Length(12), - Constraint::Length(7), - Constraint::Length(7), + // Wide enough for a held 5-digit latency ("10000ms*") so the marker is + // never the part that gets clipped. + Constraint::Length(8), + Constraint::Length(8), Constraint::Length(7), Constraint::Length(7), Constraint::Length(5), @@ -242,8 +247,11 @@ pub(in crate::ui) mod tests { gen_tps: None, gen_tps_observation: None, tokens_per_watt: None, + power_w: None, ttft_ms: None, + ttft_observation: None, tpot_ms: None, + tpot_observation: None, launch_args: vec![], env_vars: BTreeMap::new(), log_file: None, @@ -692,6 +700,97 @@ pub(in crate::ui) mod tests { ); } + #[test] + fn table_held_ttft_shows_marker_and_legend_even_with_fresh_gen_tps() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + + let mut inst = mk_inst_obs( + "idle", + Some(456.0), + Some(obs(ObservationFreshness::Fresh, 5)), + ); + inst.ttft_ms = Some(150.0); + inst.ttft_observation = Some(obs(ObservationFreshness::Held, 30)); + let state = state_with_snap(inst); + let mut term = Terminal::new(TestBackend::new(160, 20)).unwrap(); + term.draw(|f| draw_table(f, f.area(), &state, &state.theme)) + .unwrap(); + let out = buffer_text(&term); + assert!( + out.contains("150ms*"), + "held TTFT must be marked; got: +{out}" + ); + assert!( + out.contains(format::HELD_LEGEND), + "legend must explain a held TTFT marker; got: +{out}" + ); + } + + #[test] + fn table_power_shows_attributed_share_not_board_power() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + + // The GPU board may report far more than this instance's share; the + // POWER column must show the attributed figure carried on the instance. + let mut inst = mk_inst_obs("shared", Some(10.0), None); + inst.power_w = Some(123.0); + let mut state = state_with_snap(inst); + if let Some(snap) = state.latest.as_mut() { + snap.gpus = vec![rocm_dash_core::metrics::GpuMetrics { + device_id: "gpu-0".into(), + power_w: 999.0, + ..Default::default() + }]; + } + let mut term = Terminal::new(TestBackend::new(160, 20)).unwrap(); + term.draw(|f| draw_table(f, f.area(), &state, &state.theme)) + .unwrap(); + let out = buffer_text(&term); + assert!( + out.contains("123W"), + "attributed power missing; got: +{out}" + ); + assert!( + !out.contains("999W"), + "board power leaked into row; got: +{out}" + ); + } + + #[test] + fn table_held_tpot_shows_marker_and_legend_and_five_digit_latency_is_not_clipped() { + use ratatui::Terminal; + use ratatui::backend::TestBackend; + + let mut inst = mk_inst_obs( + "idle", + Some(456.0), + Some(obs(ObservationFreshness::Fresh, 5)), + ); + // TPOT alone is held, and long enough that a 7-wide column would clip + // the trailing marker. + inst.tpot_ms = Some(12345.0); + inst.tpot_observation = Some(obs(ObservationFreshness::Held, 30)); + let state = state_with_snap(inst); + let mut term = Terminal::new(TestBackend::new(160, 20)).unwrap(); + term.draw(|f| draw_table(f, f.area(), &state, &state.theme)) + .unwrap(); + let out = buffer_text(&term); + assert!( + out.contains("12345ms*"), + "held TPOT marker missing or clipped; got:\n{out}" + ); + assert!( + out.contains(format::HELD_LEGEND), + "legend must explain a held TPOT marker; got:\n{out}" + ); + } + #[test] fn table_held_legend_absent_when_all_fresh() { use ratatui::Terminal; diff --git a/crates/rocm-dash-tui/src/ui/tabs/observe.rs b/crates/rocm-dash-tui/src/ui/tabs/observe.rs index 15554f6bd2..f200ef3b81 100644 --- a/crates/rocm-dash-tui/src/ui/tabs/observe.rs +++ b/crates/rocm-dash-tui/src/ui/tabs/observe.rs @@ -291,6 +291,7 @@ mod tests { gpu_utilization_pct: 50.0, temperature_c: 55.0, power_w: 300.0, + power_limit_w: None, clock_mhz: Some(2000.0), }], ..Default::default() diff --git a/crates/rocm-dash-tui/src/ui/widgets.rs b/crates/rocm-dash-tui/src/ui/widgets.rs index ca5da108fe..f429ecdbda 100644 --- a/crates/rocm-dash-tui/src/ui/widgets.rs +++ b/crates/rocm-dash-tui/src/ui/widgets.rs @@ -16,15 +16,15 @@ pub const TEMP_WARN_C: f32 = 60.0; /// Temperature critical threshold (°C). At or above → err color. pub const TEMP_CRIT_C: f32 = 80.0; -/// Board-power warning threshold (W). At or above → warn color. +/// Fraction of a GPU's power limit at or above which power is colored warn. /// /// Fixed semantic thresholds make temperature and power colors mean "near the limit" -/// rather than "near the largest value seen this session". Tuned for -/// MI355X-class parts (TDP ~750 W); the demo generator peaks ~740 W at 100% -/// util. No per-GPU TDP exists in the data model, so these are constants. -pub const POWER_WARN_W: f32 = 525.0; -/// Board-power critical threshold (W). At or above → err color. ~TDP-adjacent. -pub const POWER_CRIT_W: f32 = 700.0; +/// rather than "near the largest value seen this session". The power thresholds +/// are relative to the device's own cap, so the colors mean "near this part's +/// limit" on any GPU, not "near an MI355X's". +pub const POWER_WARN_FRAC: f32 = 0.70; +/// Fraction of the power limit at or above which board power is colored err. +pub const POWER_CRIT_FRAC: f32 = 0.93; /// Truncate `s` to at most `n` characters (not bytes). pub fn trunc(s: &str, n: usize) -> String { @@ -80,15 +80,19 @@ pub fn temperature_style(c: f32, theme: &Theme) -> Style { Style::default().fg(color) } -/// Color a board-power value against the fixed [`POWER_WARN_W`] / -/// [`POWER_CRIT_W`] thresholds, mirroring [`temperature_style`]. -pub fn power_style(w: f32, theme: &Theme) -> Style { - let color = if w >= POWER_CRIT_W { - theme.err - } else if w >= POWER_WARN_W { - theme.warn - } else { - theme.ok +/// Color a board-power value against the GPU's own power limit +/// ([`POWER_WARN_FRAC`] / [`POWER_CRIT_FRAC`] of it), mirroring +/// [`temperature_style`]. +/// +/// With no known limit (or a non-positive one) there is nothing to compare +/// against, so the value is drawn in the plain foreground color rather than +/// judged against another part's TDP. +pub fn power_style(w: f32, limit_w: Option, theme: &Theme) -> Style { + let color = match limit_w.filter(|l| l.is_finite() && *l > 0.0) { + None => theme.fg, + Some(limit) if w >= limit * POWER_CRIT_FRAC => theme.err, + Some(limit) if w >= limit * POWER_WARN_FRAC => theme.warn, + Some(_) => theme.ok, }; Style::default().fg(color) } @@ -172,9 +176,13 @@ pub fn gpu_stats_line<'a>(g: &'a GpuMetrics, theme: &Theme) -> Line<'a> { Style::default().fg(theme.muted), ), Span::styled( - format!(" {:>5.1}°C {:>5.1} W", g.temperature_c, g.power_w), + format!(" {:>5.1}°C", g.temperature_c), temperature_style(g.temperature_c, theme), ), + Span::styled( + format!(" {:>5.1} W", g.power_w), + power_style(g.power_w, g.power_limit_w, theme), + ), ]) } @@ -216,14 +224,58 @@ mod tests { } #[test] - fn power_style_uses_fixed_thresholds() { + fn power_style_is_relative_to_the_gpu_limit() { let theme = Theme::default_dark(); - assert_eq!(power_style(720.0, &theme).fg, Some(theme.err)); - assert_eq!(power_style(600.0, &theme).fg, Some(theme.warn)); - assert_eq!(power_style(300.0, &theme).fg, Some(theme.ok)); - // boundaries are inclusive at the threshold - assert_eq!(power_style(POWER_CRIT_W, &theme).fg, Some(theme.err)); - assert_eq!(power_style(POWER_WARN_W, &theme).fg, Some(theme.warn)); + // Same wattage, different parts: 600 W is critical on a 600 W card but + // comfortably fine on a 1000 W one. + assert_eq!(power_style(600.0, Some(600.0), &theme).fg, Some(theme.err)); + assert_eq!(power_style(600.0, Some(1000.0), &theme).fg, Some(theme.ok)); + assert_eq!(power_style(500.0, Some(600.0), &theme).fg, Some(theme.warn)); + assert_eq!(power_style(300.0, Some(600.0), &theme).fg, Some(theme.ok)); + // Boundaries are inclusive. + assert_eq!( + power_style(1000.0 * POWER_CRIT_FRAC, Some(1000.0), &theme).fg, + Some(theme.err) + ); + assert_eq!( + power_style(1000.0 * POWER_WARN_FRAC, Some(1000.0), &theme).fg, + Some(theme.warn) + ); + } + + #[test] + fn power_style_without_a_limit_is_neutral_not_judged() { + let theme = Theme::default_dark(); + for limit in [None, Some(0.0), Some(f32::NAN), Some(-5.0)] { + assert_eq!(power_style(720.0, limit, &theme).fg, Some(theme.fg)); + } + } + + #[test] + fn full_panel_colors_power_by_limit_not_by_temperature() { + let theme = Theme::default_dark(); + // A cool GPU at 95% of its cap: power is critical even though the + // temperature is fine, the same as in the compact row. + let g = GpuMetrics { + device_id: "gpu-0".into(), + temperature_c: 40.0, + power_w: 712.5, + power_limit_w: Some(750.0), + ..GpuMetrics::default() + }; + let line = gpu_stats_line(&g, &theme); + let temp = line + .spans + .iter() + .find(|s| s.content.contains("°C")) + .unwrap(); + let power = line + .spans + .iter() + .find(|s| s.content.contains(" W")) + .unwrap(); + assert_eq!(temp.style.fg, Some(theme.ok)); + assert_eq!(power.style.fg, Some(theme.err)); } #[test] diff --git a/crates/rocm-dash-tui/tests/dash_characterization.rs b/crates/rocm-dash-tui/tests/dash_characterization.rs index ffca5d6b12..fc7f8e659a 100644 --- a/crates/rocm-dash-tui/tests/dash_characterization.rs +++ b/crates/rocm-dash-tui/tests/dash_characterization.rs @@ -41,6 +41,7 @@ fn synthetic_snapshot() -> Snapshot { gpu_utilization_pct: 72.0, temperature_c: 58.0, power_w: 420.0, + power_limit_w: None, clock_mhz: Some(2100.0), }], gpu_system_info: Some(GpuSystemInfo { diff --git a/crates/rocm-dash-tui/tests/dash_journeys.rs b/crates/rocm-dash-tui/tests/dash_journeys.rs index 31117a8b41..2a8e1ecffc 100644 --- a/crates/rocm-dash-tui/tests/dash_journeys.rs +++ b/crates/rocm-dash-tui/tests/dash_journeys.rs @@ -39,6 +39,7 @@ fn synthetic_snapshot() -> Snapshot { gpu_utilization_pct: 66.0, temperature_c: 61.0, power_w: 410.0, + power_limit_w: None, clock_mhz: Some(2050.0), }], gpu_system_info: Some(GpuSystemInfo { diff --git a/tests/e2e-cucumber/src/mock_server.rs b/tests/e2e-cucumber/src/mock_server.rs index 3a2844d87f..60fb9d2950 100644 --- a/tests/e2e-cucumber/src/mock_server.rs +++ b/tests/e2e-cucumber/src/mock_server.rs @@ -378,7 +378,7 @@ struct ServerState { /// Deterministic, monotonically-advancing state behind the mock `/metrics` /// route, so successive scrapes exercise the daemon's rate/average windowing -/// (`gen_tps_from_delta`, `avg_ms_from_histogram` in +/// (`gen_tps_from_delta`, `observe_latency` in /// `rocm-dash-daemon::runner`) the same way a real vLLM would: /// * `vllm:generation_tokens_total` strictly increases every scrape, so the /// counter-delta windowing yields a positive, visible generation rate diff --git a/tests/e2e-cucumber/tests/e2e/dash_steps.rs b/tests/e2e-cucumber/tests/e2e/dash_steps.rs index 759e68b102..1f87dc8581 100644 --- a/tests/e2e-cucumber/tests/e2e/dash_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/dash_steps.rs @@ -23,27 +23,6 @@ const MANAGED_MODEL_PROMPT: &str = "hello from the terminal"; /// it proves both that the overlay opened and - as an absence - that it closed. const SERVICES_OVERLAY_TITLE: &str = "Services — managed inference servers"; -/// The Observe instances table's TTFT cell while the scripted mock is serving: -/// its histogram pins time-to-first-token at exactly 50 ms -/// (`ttft_sum_s = ticks × 0.050` over `ttft_count = ticks`), and the cell is -/// rendered `"{v:.0}ms"`. A *failed* scrape clears `ttft_ms`/`tpot_ms` -/// (`runner.rs`), so this cell changing is the screen's own proof that the -/// frame on display was assembled after the failure — the only frame the -/// held-throughput assertion is about. -const SCRIPTED_TTFT_CELL: &str = "50ms"; - -/// Zero-based index of the TTFT cell within an Observe instances row, counting -/// from the model id: `MODEL TOK/S TOK/W TTFT TPOT POWER QUEUE KV%` -/// (`instances.rs`). -/// -/// The cell is read by position on the scripted instance's own row rather than -/// matched as a substring of the whole screen. A bare substring cannot tell a -/// cleared cell from a surviving one: any future ms-suffixed value that merely -/// *contains* the scripted one (`150ms`, `250ms`), or a second row whose TTFT -/// is also 50 ms, would keep the marker on screen and time this step out for a -/// reason that has nothing to do with the scrape it synchronises on. -const TTFT_COLUMN: usize = 3; - /// Borrow the scenario's active TUI session, or fail clearly if none was opened. const fn session(world: &mut E2eWorld) -> &mut TuiSession { world @@ -2409,19 +2388,27 @@ fn write_dash_clock(path: &std::path::Path, directive: &str) { std::fs::rename(&tmp, path).expect("failed to publish the dashboard test clock"); } -/// The TTFT cell `model`'s row is currently rendering, or `None` while that row -/// is not on screen at all. +/// The count on the header's `⚠ N` warnings badge, or 0 while no badge is drawn. /// -/// Cells are whitespace-separated and a model id carries no spaces, so counting -/// fields from the id yields one field per column — including the `—` -/// placeholder a cleared cell renders, which keeps the columns aligned. -fn scripted_ttft_cell<'a>(screen: &'a str, model: &str) -> Option<&'a str> { +/// The badge counts the daemon's current warnings, and a failed `/metrics` +/// scrape adds a `vllm scrape: N failed` warning to the snapshot it publishes. +/// A TTFT or throughput cell cannot serve as proof that a failure reached the +/// screen: those values are deliberately *held* through a failed scrape (and, +/// with the clock held, stay marked fresh), so the badge growing is the +/// visible evidence that a frame assembled after the failure is on display. +fn warning_badge_count(screen: &str) -> usize { screen - .lines() - .find(|line| line.contains(model))? - .split_whitespace() - .skip_while(|field| *field != model) - .nth(TTFT_COLUMN) + .split('⚠') + .nth(1) + .map(|after| { + after + .trim_start() + .chars() + .take_while(char::is_ascii_digit) + .collect::() + }) + .and_then(|digits| digits.parse().ok()) + .unwrap_or(0) } /// The Observe tab's node-throughput hero shows the "tok/s" unit whenever @@ -2465,12 +2452,17 @@ async fn dashboard_observation_time_is_held(world: &mut E2eWorld) { /// Switch the scripted mock to Failure mode and wait for the failure to reach /// the screen: first the mock's own counter proves the daemon was served a 503, -/// then the cleared TTFT cell proves the frame on display is one the daemon -/// assembled after that scrape. Both waits are synchronizations on observed -/// events, not fixed sleeps, and — the clock being held — neither can consume -/// the validity window they precede. +/// then the header warnings badge growing proves the frame on display is one +/// the daemon assembled after that scrape. Both waits are synchronizations on +/// observed events, not fixed sleeps, and — the clock being held — neither can +/// consume the validity window they precede. #[when("the metrics endpoint fails transiently")] async fn metrics_endpoint_fails(world: &mut E2eWorld) { + // Read the badge before the failure is injected: the count that matters is + // the one this failure adds, and other warnings (e.g. no amd-smi on a + // GPU-less runner) are already present and constant. + let before = warning_badge_count(&session(world).screen_text()); + let mock = world.mock.as_ref().expect("no mock server running"); mock.set_metrics_mode(MetricsMode::Failure); @@ -2490,16 +2482,10 @@ async fn metrics_endpoint_fails(world: &mut E2eWorld) { tokio::time::sleep(Duration::from_millis(100)).await; } - let model = world - .model_name - .clone() - .expect("the scripted-metrics Given records the model this row belongs to"); session(world) .wait_for_screen_where( - &format!("the scripted instance's TTFT cell leaves {SCRIPTED_TTFT_CELL:?}"), - |screen| { - scripted_ttft_cell(screen, &model).is_some_and(|cell| cell != SCRIPTED_TTFT_CELL) - }, + &format!("the header warnings badge grows past {before}"), + |screen| warning_badge_count(screen) > before, default_timeout(), ) .await @@ -2513,8 +2499,8 @@ async fn metrics_endpoint_fails(world: &mut E2eWorld) { /// EAI-7960 principal regression assertion. /// -/// The frame under assertion is provably post-failure (the TTFT cell it used to -/// show is gone) and the logical clock is held, so the only way "tok/s" can be +/// The frame under assertion is provably post-failure (the header warnings +/// badge has grown past its pre-failure count) and the logical clock is held, so the only way "tok/s" can be /// missing here is the regression itself: the daemon clearing a held rate on a /// failed scrape instead of keeping it for the validity window. #[then("generation throughput remains visible within the validity window")]