Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
80 changes: 80 additions & 0 deletions crates/rocm-dash-collectors/src/amd_smi.rs
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,13 @@ impl AmdSmiCollector {
Ok(parse_metrics(&v))
}

/// Per-GPU power caps (W) from `amd-smi static --json`, keyed by the same
/// `gpu-N` id [`metrics`] uses. GPUs with no readable cap are omitted.
pub async fn power_limits(&self) -> io::Result<std::collections::HashMap<String, f32>> {
let v = self.run_json(&["static", "--json"]).await?;
Ok(parse_power_limits(&v))
}

/// Per-process GPU VRAM usage from `amd-smi process --json`.
///
/// Inherent async (mirrors [`metrics`]); the runner uses this directly
Expand Down Expand Up @@ -258,11 +265,51 @@ pub fn parse_metrics(v: &Value) -> Vec<GpuMetrics> {
temperature_c,
power_w: val_f32(nested(g, &["power", "socket_power", "value"])).unwrap_or(0.0),
clock_mhz: val_f32(nested(g, &["clock", "gfx_0", "clk", "value"])),
// Filled by the runner from `power_limits`; not in `metric`.
power_limit_w: None,
}
})
.collect()
}

/// Candidate keys under a GPU's `limit` object for its power cap, in priority
/// order: the cap currently in force, then the maximum the part allows.
const POWER_LIMIT_KEYS: [&str; 2] = ["socket_power", "max_power"];

/// Parse `amd-smi static --json` into per-GPU power caps (W), keyed `gpu-N`.
///
/// Assumed schema (not verified against every amd-smi release): top level
/// `{ "gpu_data": [ <gpu>, ... ] }` or a bare array; each `<gpu>` has
/// `{ "gpu": N, "limit": { "socket_power" | "max_power": <watts> } }` where the
/// value is a raw number or a `{ "value": W, "unit": "W" }` wrapper. A missing,
/// non-numeric, non-finite or non-positive cap (`N/A`, `0`) is omitted, so an
/// unrecognised schema yields an empty map and callers treat the cap as
/// unknown rather than guessing one.
pub fn parse_power_limits(v: &Value) -> std::collections::HashMap<String, f32> {
let entries: Vec<&Value> = if let Some(arr) = v.get("gpu_data").and_then(|x| x.as_array()) {
arr.iter().collect()
} else if let Some(arr) = v.as_array() {
arr.iter().collect()
} else {
return std::collections::HashMap::new();
};
entries
.into_iter()
.filter_map(|g| {
let id = g.get("gpu").and_then(serde_json::Value::as_u64)?;
let limit = g.get("limit")?;
// Validity is judged per key, so an unreadable or zero current cap
// falls through to the part's maximum instead of masking it.
let watts = POWER_LIMIT_KEYS.iter().find_map(|k| {
let raw = limit.get(*k)?;
let w = val_f32(Some(raw)).or_else(|| val_f32(raw.get("value")))?;
(w.is_finite() && w > 0.0).then_some(w)
})?;
Some((format!("gpu-{id}"), watts))
})
.collect()
}

/// Candidate `(parent, child)` JSON paths for a process's VRAM figure, in
/// priority order. amd-smi naming varies by version, so we probe several.
const VRAM_PATHS: &[(&str, &str)] = &[
Expand Down Expand Up @@ -504,6 +551,39 @@ mod tests {
);
}

#[test]
fn power_limits_prefer_current_cap_and_skip_unreadable() {
let v: Value = serde_json::from_str(
r#"{ "gpu_data": [
{ "gpu": 0, "limit": { "socket_power": { "value": 600, "unit": "W" },
"max_power": { "value": 750, "unit": "W" } } },
{ "gpu": 1, "limit": { "max_power": 750 } },
{ "gpu": 2, "limit": { "socket_power": "N/A" } },
{ "gpu": 3, "limit": { "max_power": { "value": 0, "unit": "W" } } },
{ "gpu": 4 }
] }"#,
)
.unwrap();
let m = parse_power_limits(&v);
assert_eq!(m.get("gpu-0"), Some(&600.0), "current cap wins over max");
assert_eq!(m.get("gpu-1"), Some(&750.0), "raw number falls back to max");
assert_eq!(m.len(), 2, "N/A, zero and absent caps are unknown: {m:?}");
// Unrecognised shapes are an empty map, not a panic.
assert!(parse_power_limits(&Value::Null).is_empty());
}

#[test]
fn a_zero_current_cap_falls_through_to_the_maximum() {
let v: Value = serde_json::from_str(
r#"[ { "gpu": 0, "limit": { "socket_power": 0, "max_power": 750 } },
{ "gpu": 1, "limit": { "socket_power": "N/A", "max_power": 600 } } ]"#,
)
.unwrap();
let m = parse_power_limits(&v);
assert_eq!(m.get("gpu-0"), Some(&750.0), "0 must not mask max_power");
assert_eq!(m.get("gpu-1"), Some(&600.0), "N/A falls through too");
}

const SAMPLE_METRIC: &str = r#"{
"gpu_data": [
{
Expand Down
7 changes: 3 additions & 4 deletions crates/rocm-dash-collectors/src/bench_load.rs
Original file line number Diff line number Diff line change
Expand Up @@ -542,10 +542,9 @@ fn prom_latency(
/// ever handled — earlier ramp cells and other clients included — so writing it
/// into this cell's immutable CSV row would describe a different population than
/// the row names, with nothing to flag it. A blank column therefore honestly
/// means "not measured for this cell". (This is narrower than the telemetry
/// daemon's `avg_ms_from_histogram`, which *does* fall back: it is a repeated
/// live poll with a rolling baseline that self-corrects on the next tick and
/// persists nothing, so the trade-offs differ.)
/// means "not measured for this cell". (The telemetry daemon's live TTFT/TPOT
/// follow the same rule: only a window with new requests yields a value, and
/// it is held for a bounded time and then cleared.)
fn latency_ms(
sum_before: Option<f64>,
count_before: Option<f64>,
Expand Down
20 changes: 20 additions & 0 deletions crates/rocm-dash-collectors/src/cgroup.rs
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,15 @@ pub fn parse_container_id_from_cgroup(contents: &str) -> Option<String> {
.map(str::to_string)
}

/// Whether `id` has the shape [`parse_container_id_from_cgroup`] produces.
///
/// That shape is 64 lowercase-hex characters. A managed `rocm serve` service is named by its
/// service id, which never has this shape, so it can never be a key of the
/// per-container VRAM map built from `/proc/<pid>/cgroup`.
pub fn is_container_id(id: &str) -> bool {
id.len() == 64 && id.chars().all(is_lower_hex)
}

/// Resolve the docker container id owning host process `pid` by reading `/proc/<pid>/cgroup`.
///
/// Returns `None` on any read error (process gone,
Expand Down Expand Up @@ -119,6 +128,17 @@ mod tests {
);
}

#[test]
fn is_container_id_accepts_only_the_cgroup_join_shape() {
assert!(is_container_id(ID));
assert!(!is_container_id(
"vllm-qwen-qwen2-5-0-5b-instru-1793361705247"
));
assert!(!is_container_id(&ID.to_uppercase()));
assert!(!is_container_id(&ID[..63]));
assert!(!is_container_id(""));
}

#[test]
fn container_id_for_pid_unreadable_is_none_no_panic() {
// A PID that cannot exist → unreadable /proc path → None (no panic).
Expand Down
3 changes: 3 additions & 0 deletions crates/rocm-dash-collectors/src/winperf.rs
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,7 @@ fn parse_metrics(raw: &str, adlx: &[AdlxReading]) -> Vec<GpuMetrics> {
gpu_utilization_pct,
temperature_c,
power_w,
power_limit_w: None,
clock_mhz,
}];
}
Expand All @@ -371,6 +372,8 @@ fn parse_metrics(raw: &str, adlx: &[AdlxReading]) -> Vec<GpuMetrics> {
gpu_utilization_pct: util.get(key).copied().unwrap_or(0.0).min(100.0),
temperature_c,
power_w,
// No limit source on Windows: unknown, so power is not colored.
power_limit_w: None,
clock_mhz,
})
.collect()
Expand Down
Loading
Loading