diff --git a/apps/rocm/src/examine_cmd.rs b/apps/rocm/src/examine_cmd.rs index 6ae1a22fae..06b1a2cab6 100644 --- a/apps/rocm/src/examine_cmd.rs +++ b/apps/rocm/src/examine_cmd.rs @@ -82,13 +82,13 @@ pub(crate) fn examine(json: bool, framework: rocm_core::FrameworkProbe) -> Resul // torch lives only in its site-packages, so probing PATH would report // `unknown` for a host that has one. let interpreter = rocm_core::active_managed_framework_interpreter(&paths, &config); - let mut examination = - rocm_core::Examination::probe_with_interpreter(framework, interpreter.as_ref()); - // `gather` rather than `examine_human_report`: the latter first runs - // `recover_setup_runtime_registration`, which writes. Asking a machine a - // question should not change it, and `--json` is the form tooling calls - // in a loop. - let host = ExamineSummary::gather()?; + // `probe_with_summary` rather than `examine_human_report`, whose recovery + // step writes: asking a machine a question should not change it, and + // `--json` is the form tooling calls in a loop. It gathers the summary + // from the examination it probes, so the kernel release is read once for + // both documents and the two fields cannot disagree. + let (mut examination, host) = + rocm_core::Examination::probe_with_summary(framework, interpreter.as_ref())?; // The human report's "AMD GPU detected" line reads this same // `detected_gfx_target`; WSL skips the probes that would set // `has_amd_gpu`, so take it from here rather than leave the two forms diff --git a/crates/rocm-core/src/examine.rs b/crates/rocm-core/src/examine.rs index fc339d767a..89f19847fa 100644 --- a/crates/rocm-core/src/examine.rs +++ b/crates/rocm-core/src/examine.rs @@ -545,6 +545,29 @@ impl Examination { Self::probe_with_interpreter(framework, None) } + /// Probe this host and gather its summary from the examination, returning + /// both: what `rocm examine --json` prints as one document. + /// + /// The pairing is made here so no caller can hand the summary an + /// examination of another host. The summary takes the examination's kernel + /// release (a failed read included) instead of running `uname -r` again, so + /// one inspection reads it once: `summary.kernel` and `kernel_release` + /// cannot disagree on a transient failure, and a hung `uname` costs one + /// timeout rather than two. Probing is as [`Self::probe_with_interpreter`]. + /// + /// # Errors + /// + /// Fails when the host summary cannot be gathered (the application + /// directories cannot be resolved); the examination itself never fails. + pub fn probe_with_summary( + framework: FrameworkProbe, + interpreter: Option<&FrameworkInterpreter>, + ) -> anyhow::Result<(Self, crate::ExamineSummary)> { + let examination = Self::probe_with_interpreter(framework, interpreter); + let summary = crate::ExamineSummary::gather_after(&examination)?; + Ok((examination, summary)) + } + /// As [`Self::probe`], but running the framework probe under `interpreter` /// when one is given. /// diff --git a/crates/rocm-core/src/examine/host_env.rs b/crates/rocm-core/src/examine/host_env.rs index da192981da..2311271487 100644 --- a/crates/rocm-core/src/examine/host_env.rs +++ b/crates/rocm-core/src/examine/host_env.rs @@ -54,7 +54,10 @@ pub(super) fn probe_os(e: &mut Examination) { }; if runtime_is_linux() { e.os_family = "linux".to_owned(); - e.kernel_release = run("uname", &["-r"], SHORT).1.trim().to_owned(); + // The summary's read, so `kernel:` and `kernel_release` come from one + // `uname -r`. A failed read is empty here, with no `probe_failures` + // entry, as before: a host without a usable `uname` is not a degraded one. + e.kernel_release = crate::host_gpu::detect_kernel_version().unwrap_or_default(); e.kernel_cmdline = read_host_text("/proc/cmdline").trim().to_owned(); record_distro( e, @@ -618,4 +621,251 @@ mod tests { ); assert_eq!(name.as_deref(), Some("Fallback 1.2")); } + + // The kernel-release read, the second pair of `docs/host-probing.md` + // ("Kernel version"). It used to be two readers: `detect_kernel_version` in + // the summary stack (1.5 s, strict UTF-8, `None` on any failure or non-zero + // exit) and `probe_os` here (5 s, lossy, exit code ignored, empty on + // failure). `probe_os` now calls `detect_kernel_version`, so these tests pin + // the one reader's behaviour through both entry points: a release is read + // as printed, a non-zero exit, output that is not UTF-8, a silent `uname` + // and one that never answers read as nothing, an answer within the 5 s + // limit is read, and a failed read leaves no `probe_failures` entry. Each + // case plants a fake `uname` first on `PATH` and asks both entry points in + // a child copy of this test binary, because the lookup of `uname` follows + // the process's own `PATH`, which a test thread cannot change without + // disturbing every other test. The child's entry point, + // [`kernel_pair_child_reports_both_answers`], does nothing in a normal run. + // Linux only, because `probe_os` reads `uname -r` only when + // `runtime_is_linux()`, and the fake is a shell script; other Unix hosts are + // not supported platforms. + + #[cfg(target_os = "linux")] + mod kernel_pair { + use super::*; + use std::process::Command; + use std::time::Instant; + + const CHILD_ENV: &str = "ROCM_CORE_TEST_KERNEL_PAIR"; + const ANSWER_PREFIX: &str = "kernel-pair-answers:"; + + /// What each stack read for `uname -r`. + #[derive(Debug, PartialEq, Eq)] + struct Pair { + /// `detect_kernel_version`, the summary's `kernel`. + summary: Option, + /// `probe_os`'s `kernel_release`. + examination: String, + } + + fn pair(summary: Option<&str>, examination: &str) -> Pair { + Pair { + summary: summary.map(str::to_owned), + examination: examination.to_owned(), + } + } + + /// The child's side of [`ask`]: print both answers as JSON. + #[test] + fn kernel_pair_child_reports_both_answers() { + let Some(mode) = std::env::var_os(CHILD_ENV) else { + return; + }; + if mode == "inspect" { + // One inspection through the entry point `rocm examine --json` uses. + let (e, summary) = + Examination::probe_with_summary(crate::FrameworkProbe::Skip, None) + .expect("probe and gather"); + let answers = serde_json::json!({ + "summary": summary.kernel, + "examination": e.kernel_release, + "probe_failures": e.probe_failures, + }); + println!("{ANSWER_PREFIX}{answers}"); + return; + } + let mut e = Examination::default(); + probe_os(&mut e); + let answers = serde_json::json!({ + "summary": crate::host_gpu::detect_kernel_version(), + "examination": e.kernel_release, + "probe_failures": e.probe_failures, + }); + println!("{ANSWER_PREFIX}{answers}"); + } + + /// Plant a `uname` whose `-r` output is `body` (a shell fragment) and ask + /// both stacks. `uname -v` answers a fixed string, so `probe_os` has + /// nothing else to wait for. + fn ask(body: &str) -> Pair { + let (json, _release_reads) = run_child(body, "pair"); + // A failed read is empty, not a failure: `probe_os` records no + // `probe_failures` entry at all in any case (whatever its wording + // would be), so a host with a missing or broken `uname` does not read + // as degraded. The fake host's os-release reads cleanly, so the list + // is empty rather than compared with a baseline. + assert_eq!(json["probe_failures"], serde_json::json!([]), "{json}"); + Pair { + summary: json["summary"].as_str().map(str::to_owned), + examination: json["examination"] + .as_str() + .expect("the examination answer is a string") + .to_owned(), + } + } + + /// Run the child in `mode` against a fake `uname` whose `-r` output is + /// `body`, and return its answers and how many times `-r` was asked. + /// `uname -v` answers a fixed string, so `probe_os` has nothing else to + /// wait for. + fn run_child(body: &str, mode: &str) -> (serde_json::Value, usize) { + let scratch = tempfile::tempdir().expect("scratch directory"); + let bin = scratch.path().join("bin"); + std::fs::create_dir_all(&bin).expect("create the fake bin directory"); + let calls = scratch.path().join("release-reads"); + crate::fake_executable::write_fake_executable( + &bin.join("uname"), + format!( + "#!/bin/sh\nif [ \"$1\" = -v ]; then echo '#1 SMP fake'; exit 0; fi\necho x >> '{}'\n{body}\n", + calls.display() + ), + ) + .expect("plant the fake uname"); + let this_test = module_path!() + .split_once("::") + .expect("module path has a crate prefix") + .1; + let output = Command::new(std::env::current_exe().expect("path of the test binary")) + .args([ + "--exact", + &format!("{this_test}::kernel_pair_child_reports_both_answers"), + "--nocapture", + "--test-threads=1", + ]) + .env(CHILD_ENV, mode) + // The fake first; the system directories so `sleep` and `printf` + // resolve inside it. + .env("PATH", format!("{}:/usr/bin:/bin", bin.display())) + .output() + .expect("start the test binary again"); + let stdout = String::from_utf8_lossy(&output.stdout); + let line = stdout + .lines() + .find_map(|line| line.split_once(ANSWER_PREFIX).map(|(_, json)| json)) + .unwrap_or_else(|| { + panic!( + "the child printed no answers (status {}):\n{stdout}\n{}", + output.status, + String::from_utf8_lossy(&output.stderr) + ) + }); + let json: serde_json::Value = serde_json::from_str(line).expect("answers are JSON"); + let reads = std::fs::read_to_string(&calls).map_or(0, |log| log.lines().count()); + (json, reads) + } + + /// One inspection reads the release once, for the examination and the + /// summary together, so the two fields agree and a hung `uname` costs + /// one timeout. The reads are counted by the fake `uname` itself, so a + /// second read shows even when both return the same string. A failed + /// read is shared too: both are empty and it is still one read. + #[test] + fn one_inspection_reads_the_kernel_release_once() { + let (json, reads) = run_child("echo 6.8.0-fake", "inspect"); + assert_eq!(json["summary"], "6.8.0-fake", "{json}"); + assert_eq!(json["examination"], "6.8.0-fake", "{json}"); + assert_eq!(reads, 1, "{json}"); + + let (json, reads) = run_child("echo 6.8.0-partial; exit 3", "inspect"); + assert_eq!(json["summary"], serde_json::Value::Null, "{json}"); + assert_eq!(json["examination"], "", "{json}"); + assert_eq!(reads, 1, "{json}"); + } + + #[test] + fn both_read_the_release_a_working_uname_prints() { + assert_eq!( + ask("echo 6.8.0-fake"), + pair(Some("6.8.0-fake"), "6.8.0-fake") + ); + } + + /// A WSL 1 and a WSL 2 style release are read as printed by both, so + /// `is_wsl1_kernel` (which reads `kernel_release`) sees what the summary + /// shows. The consumer's answer is pinned in `wsl.rs`. + #[test] + fn both_read_wsl_style_releases_as_printed() { + for release in [ + "4.4.0-19041-Microsoft", + "5.15.167.4-microsoft-standard-WSL2", + ] { + assert_eq!( + ask(&format!("echo {release}")), + pair(Some(release), release), + "{release}" + ); + } + } + + /// A `uname` that prints a release and exits non-zero is dropped by + /// both. The examination used to keep the release. + #[test] + fn a_non_zero_exit_with_output_is_dropped_by_both() { + assert_eq!(ask("echo 6.8.0-partial; exit 3"), pair(None, "")); + } + + /// Both read a successful but silent `uname` as nothing: `None` in the + /// summary (shown as ``), empty in the examination, with no + /// `probe_failures` entry in either case. + #[test] + fn a_silent_uname_reads_as_nothing_in_both() { + assert_eq!(ask("exit 0"), pair(None, "")); + } + + /// Output that is not UTF-8 is dropped by both. The examination used to + /// keep it with the bad byte replaced. + #[test] + fn output_that_is_not_utf8_is_dropped_by_both() { + assert_eq!(ask("printf '6.8.0-\\377\\n'"), pair(None, "")); + } + + /// A `uname` that answers after 2 s is read by both: the one timeout is + /// the longer of the two old ones, 5 s. The summary used to give up at + /// 1.5 s, so 2 s is past that old limit and leaves 3 s of margin to the + /// new one on a loaded runner. + #[test] + fn an_answer_after_two_seconds_is_read_by_both() { + let started = Instant::now(); + assert_eq!( + ask("sleep 2; echo 6.8.0-slow"), + pair(Some("6.8.0-slow"), "6.8.0-slow") + ); + assert!(started.elapsed() >= std::time::Duration::from_secs(2)); + } + + /// A `uname` that never answers is read as nothing by both, each after + /// the one timeout. `exec` leaves no grandchild holding the pipe. + /// + /// The stand-in would exit successfully with nothing after 30 s, which is + /// also what a read with no timeout would turn into `None`, so the + /// answers alone cannot tell a timeout from no timeout. The elapsed time + /// can: the child reads twice in a row, each bounded by the 5 s limit, so + /// it takes about 10 s, and at least 9 s shows the stand-in really hung. + /// With the limit raised past the stand-in's 30 s it takes 60 s and the + /// upper bound fails. + #[test] + fn a_uname_that_never_answers_reads_as_nothing_in_both_within_the_timeout() { + let started = Instant::now(); + assert_eq!(ask("exec sleep 30"), pair(None, "")); + let elapsed = started.elapsed(); + assert!( + elapsed >= std::time::Duration::from_secs(9), + "two reads of a hung uname returned after {elapsed:?}" + ); + assert!( + elapsed < std::time::Duration::from_secs(25), + "the reads outlived the timeout: {elapsed:?}" + ); + } + } } diff --git a/crates/rocm-core/src/examine/wsl.rs b/crates/rocm-core/src/examine/wsl.rs index 28981417ea..75a52ab4a1 100644 --- a/crates/rocm-core/src/examine/wsl.rs +++ b/crates/rocm-core/src/examine/wsl.rs @@ -515,6 +515,32 @@ mod tests { ); } + /// The consumer of `kernel_release` for the WSL facts. The kernel read is a + /// pair of the host-probing note ("Kernel version"); this pins what + /// `probe_wsl` makes of each string the two reads can hand it, including + /// the empty string a failed read leaves: a WSL 1 style release is version + /// 1, a WSL 2 style one and an empty one are version 2. + #[test] + fn probe_wsl_reads_the_wsl_version_from_the_kernel_release() { + for (release, version) in [ + ("4.4.0-19041-Microsoft", 1), + ("5.15.167.4-microsoft-standard-WSL2", 2), + ("4.19.104-microsoft-standard", 2), + ("", 2), + ] { + let mut e = Examination { + kernel_release: release.to_owned(), + ..Examination::default() + }; + probe_wsl(&mut e); + assert_eq!( + e.wsl.as_ref().map(|wsl| wsl.version), + Some(version), + "{release:?}" + ); + } + } + #[test] fn the_distro_list_decodes_as_utf16_whatever_script_it_is_in() { fn utf16le(text: &str, bom: bool) -> Vec { diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index c1854ac0d8..2e5009dbf9 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -37,9 +37,10 @@ use crate::{ host_path, runtime_is_linux, runtime_is_windows, runtime_os_name, }; use anyhow::Result; +use os_facts::detect_cpu_model; pub(crate) use os_facts::detect_distro_name; +pub(crate) use os_facts::detect_kernel_version; pub use os_facts::detect_system_ram_gib; -use os_facts::{detect_cpu_model, detect_kernel_version}; use serde::{Deserialize, Serialize}; use std::fs; use std::path::{Path, PathBuf}; @@ -194,12 +195,45 @@ pub struct HostGpuSummary { pub therock_family: Option, } +/// The kernel the summary reports: the examination's, when the caller has one and +/// this is Linux (the examination reads it there, and an empty one is a failed +/// read); otherwise whatever `read` gives. `read` is not called in the first case. +fn summary_kernel( + examined: Option<&str>, + linux: bool, + read: impl FnOnce() -> Option, +) -> Option { + match examined { + Some(kernel) if linux => Some(kernel) + .filter(|kernel| !kernel.is_empty()) + .map(str::to_owned), + _ => read(), + } +} + impl ExamineSummary { /// The full host summary, including the Windows display driver version and /// date. Everything that prints the summary, or serializes it for a client /// (`rocm examine`, the MCP and bridge snapshots), wants this one. pub fn gather() -> Result { - Self::gather_scoped(true) + Self::gather_scoped(true, None) + } + + /// [`gather`](Self::gather) for the examination of this same host, which + /// [`Examination::probe_with_summary`](crate::Examination::probe_with_summary) + /// has just made (`rocm examine --json` builds both documents in one run). + /// Crate-private so that no caller can pass an examination of another host, + /// such as one deserialised from a report: its kernel release would be + /// reported as this host's. + /// + /// On Linux the kernel release is the examination's `kernel_release`, a failed + /// read included, instead of a second `uname -r`. One inspection then reads the + /// release once, so `summary.kernel` and `kernel_release` cannot disagree on a + /// transient failure, and a hung `uname` costs one timeout rather than two. + /// Everywhere else (the examination does not read the kernel on Windows) the + /// summary reads it itself, as [`gather`](Self::gather) does. + pub(crate) fn gather_after(examination: &crate::Examination) -> Result { + Self::gather_scoped(true, Some(&examination.kernel_release)) } /// The host summary without the Windows driver version and date, for a @@ -207,10 +241,10 @@ impl ExamineSummary { /// PowerShell video-controller query `gather` adds for them; the GPU /// target, CPU, RAM and every other field are the same. pub fn gather_without_driver_facts() -> Result { - Self::gather_scoped(false) + Self::gather_scoped(false, None) } - fn gather_scoped(driver_facts: bool) -> Result { + fn gather_scoped(driver_facts: bool, examined_kernel: Option<&str>) -> Result { let paths = AppPaths::discover()?; let windows_inventory = if driver_facts { detect_windows_examine_inventory_with_driver_facts() @@ -237,7 +271,7 @@ impl ExamineSummary { Ok(Self { os: runtime_os_name().to_owned(), arch: std::env::consts::ARCH.to_owned(), - kernel: detect_kernel_version(), + kernel: summary_kernel(examined_kernel, runtime_is_linux(), detect_kernel_version), distro: detect_distro_name(), cpu: detect_cpu_model_with_windows_inventory(windows_inventory.as_ref()), system_ram_gib: detect_system_ram_gib_with_windows_inventory( @@ -735,4 +769,30 @@ mod tests { assert!(!rendered.contains(crate::windows_driver::DISPLAY_DRIVER_OLD_FIX_ID)); } } + + /// The summary takes the examination's kernel release on Linux without a + /// second read, and an empty one is a failed read, not a release. Off Linux, + /// or with no examination, it reads for itself. The reads are counted, so a + /// second `uname -r` shows up as a count of 1. + #[test] + fn the_summary_reads_the_kernel_only_when_the_examination_did_not() { + for (examined, linux, reads, expected) in [ + (Some("6.8.0-fake"), true, 0, Some("6.8.0-fake")), + (Some(""), true, 0, None), + (Some("6.8.0-fake"), false, 1, Some("read")), + (None, true, 1, Some("read")), + (None, false, 1, Some("read")), + ] { + let count = std::cell::Cell::new(0); + let kernel = summary_kernel(examined, linux, || { + count.set(count.get() + 1); + Some("read".to_owned()) + }); + assert_eq!( + (kernel.as_deref(), count.get()), + (expected, reads), + "{examined:?} on linux={linux}" + ); + } + } } diff --git a/crates/rocm-core/src/host_gpu/os_facts.rs b/crates/rocm-core/src/host_gpu/os_facts.rs index 0cfab952ff..d78a459f1c 100644 --- a/crates/rocm-core/src/host_gpu/os_facts.rs +++ b/crates/rocm-core/src/host_gpu/os_facts.rs @@ -10,17 +10,49 @@ use super::capture::{ }; use crate::{hardware_root, host_path, runtime_is_linux, runtime_is_windows}; use std::fs; +use std::time::Duration; -pub(super) fn detect_kernel_version() -> Option { - if runtime_is_windows() { +/// How long `uname -r` may take. +/// +/// 5 s, the longer of the two limits the summary (1.5 s) and the examination +/// (5 s) used to apply separately. The release is a few bytes read from the +/// kernel, so a `uname` that takes more than a moment is a stalled host, where +/// waiting longer costs little and a spurious "unknown" costs more: +/// `kernel_release` feeds the WSL version check and the kernel-dependent +/// diagnoses, which read an empty value as "nothing to report". +/// +/// Written out rather than borrowed from another module's constant, so a change +/// to that constant cannot move this limit unnoticed; +/// `the_kernel_release_timeout_is_five_seconds` pins it. +const KERNEL_RELEASE_TIMEOUT: Duration = Duration::from_secs(5); + +/// The running kernel, as the text of `rocm examine` and `summary.kernel` show it +/// and as `kernel_release` in the examination reports it. +/// +/// On Linux (native and WSL), the trimmed output of `uname -r`. That is the one +/// read of the kernel release for both stacks: `Examination`'s `kernel_release` +/// is this value, with `None` shown as empty, and `Examination::probe_with_summary` +/// hands the examination's value to the summary, so one `rocm examine --json` run +/// reads it once. The read is strict: +/// +/// - a non-zero exit gives `None`, whatever `uname` printed; +/// - output that is not UTF-8 gives `None` rather than a string with replacement +/// characters in it; +/// - an empty answer gives `None`; +/// - a `uname` that has not answered after [`KERNEL_RELEASE_TIMEOUT`] gives +/// `None`. +/// +/// On Windows, the trimmed output of `cmd /C ver`, under the shorter optional +/// command timeout; the examination does not read that into `kernel_release`. +pub(crate) fn detect_kernel_version() -> Option { + let output = if runtime_is_windows() { capture_optional_command("cmd", &["/C", "ver"]) - .map(|value| value.trim().to_owned()) - .filter(|value| !value.is_empty()) } else { - capture_optional_command("uname", &["-r"]) - .map(|value| value.trim().to_owned()) - .filter(|value| !value.is_empty()) - } + capture_optional_command_with_timeout("uname", &["-r"], KERNEL_RELEASE_TIMEOUT) + }; + output + .map(|value| value.trim().to_owned()) + .filter(|value| !value.is_empty()) } pub(crate) fn detect_distro_name() -> Option { @@ -221,4 +253,12 @@ mod tests { "Linux (/usr/lib/os-release line 2 is not a plain assignment: \"unset ID\")" ); } + + /// The decision recorded in `docs/host-probing.md`: one 5 s limit for the + /// kernel read. The timeout behaviour itself is exercised, with a stand-in + /// `uname`, by the `kernel_pair` tests in `examine/host_env.rs`. + #[test] + fn the_kernel_release_timeout_is_five_seconds() { + assert_eq!(KERNEL_RELEASE_TIMEOUT, Duration::from_secs(5)); + } } diff --git a/docs/architecture.md b/docs/architecture.md index d66cfbc0da..a54fe54a37 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -196,13 +196,13 @@ Already-extracted subsystem modules, following the same full-domain-extraction p - [`download.rs`](../crates/rocm-core/src/download.rs) — streaming file downloads with resume and retry: `DownloadRequest`/`DownloadOutcome`, `download_file_streaming(_with_progress)` and the simpler `download_file_to_path(_with_progress)`, `Backoff`, and the retry policy (`DOWNLOAD_MAX_ATTEMPTS`). A download streams to a per-process `.part-` sibling, resumes with a `Range` request, and enforces the caller's size limit, expected length and SHA-256 digest before publishing the file. It uses `ureq`, not the loopback client in [`net.rs`](../crates/rocm-core/src/net.rs), and depends only on [`disk_space.rs`](../crates/rocm-core/src/disk_space.rs). The module is private and [`lib.rs`](../crates/rocm-core/src/lib.rs) re-exports its public names. - [`endpoint_key.rs`](../crates/rocm-core/src/endpoint_key.rs) — (ROCMAI-662) the bearer key that guards a managed service's endpoint: the CSPRNG-drawn generator and its header-safety check. Split out of [`service_identity.rs`](../crates/rocm-core/src/service_identity.rs): an id and its endpoint key are made by unrelated mechanisms (readable/timestamp-based vs. CSPRNG) and meet only at the filesystem layer, where the key's sidecar is named after the id — not a reason to keep the two in one module. - [`endpoint_readiness.rs`](../crates/rocm-core/src/endpoint_readiness.rs) — how far along a managed service's OpenAI-compatible endpoint is: `managed_service_endpoint_readiness` and `EndpointReadiness` (unreachable, listing the model, or serving a real single-token chat request), the lower-level `openai_chat_completion_probe`/`openai_chat_completion_status`/`openai_models_endpoint_has_model`, the shared inference-verified latch the engine adapters call (`engine_state_inference_verified`, `merge_json_state_file`) and the model-reference matching behind the model-listing check. It depends on [`net.rs`](../crates/rocm-core/src/net.rs), [`service_record.rs`](../crates/rocm-core/src/service_record.rs) and [`model_catalog.rs`](../crates/rocm-core/src/model_catalog.rs); [`net.rs`](../crates/rocm-core/src/net.rs) and [`service_record.rs`](../crates/rocm-core/src/service_record.rs) do not depend on it. The module is private and [`lib.rs`](../crates/rocm-core/src/lib.rs) re-exports its public names, so callers keep using `rocm_core::` paths. -- [`examine.rs`](../crates/rocm-core/src/examine.rs) — the host examination probe: the `Examination` document and its types, and `Examination::probe`, which runs the probes in order. The probes live in child modules under `examine/`, with `examine.rs` kept as the module root so the [`examine_driver_proptests.rs`](../crates/rocm-core/src/examine_driver_proptests.rs) `#[path]` attribute and the links to it stay valid: [`linux_gpus.rs`](../crates/rocm-core/src/examine/linux_gpus.rs) (Linux GPU enumeration from `lspci`, `rocminfo` and the kernel topology, with the marketing-name and APU classification they share), [`linux_driver.rs`](../crates/rocm-core/src/examine/linux_driver.rs) (kernel modules, device-node access and Secure Boot), [`rocm_install_probe.rs`](../crates/rocm-core/src/examine/rocm_install_probe.rs) (how ROCm was installed and which repository files configure it), [`library_copies.rs`](../crates/rocm-core/src/examine/library_copies.rs) (the loader library-copy search for comgr and the HIP runtime, and `comgr_matches_runtime`, which `diagnose/engine.rs` reads), [`framework.rs`](../crates/rocm-core/src/examine/framework.rs) (the PyTorch and llama.cpp probes), [`host_env.rs`](../crates/rocm-core/src/examine/host_env.rs) (the OS and distro, CPU, tracked environment variables, shared memory, container check and kernel log), [`wsl.rs`](../crates/rocm-core/src/examine/wsl.rs) (the WSL probe, including the remote probe that reads a named distribution from the Windows host, and the distro-floor check) and [`windows.rs`](../crates/rocm-core/src/examine/windows.rs) (the native Windows probes: the GPU query, the HIP SDK, the Adrenalin driver and the MSVC redistributable, plus `parse_dotted_version`, which `diagnose/windows.rs` reads). +- [`examine.rs`](../crates/rocm-core/src/examine.rs) — the host examination probe: the `Examination` document and its types, and `Examination::probe`, which runs the probes in order. The probes live in child modules under `examine/`, with `examine.rs` kept as the module root so the [`examine_driver_proptests.rs`](../crates/rocm-core/src/examine_driver_proptests.rs) `#[path]` attribute and the links to it stay valid: [`linux_gpus.rs`](../crates/rocm-core/src/examine/linux_gpus.rs) (Linux GPU enumeration from `lspci`, `rocminfo` and the kernel topology, with the marketing-name and APU classification they share), [`linux_driver.rs`](../crates/rocm-core/src/examine/linux_driver.rs) (kernel modules, device-node access and Secure Boot), [`rocm_install_probe.rs`](../crates/rocm-core/src/examine/rocm_install_probe.rs) (how ROCm was installed and which repository files configure it), [`library_copies.rs`](../crates/rocm-core/src/examine/library_copies.rs) (the loader library-copy search for comgr and the HIP runtime, and `comgr_matches_runtime`, which `diagnose/engine.rs` reads), [`framework.rs`](../crates/rocm-core/src/examine/framework.rs) (the PyTorch and llama.cpp probes), [`host_env.rs`](../crates/rocm-core/src/examine/host_env.rs) (the OS and distro, with the kernel release read through the summary stack's `detect_kernel_version`, CPU, tracked environment variables, shared memory, container check and kernel log), [`wsl.rs`](../crates/rocm-core/src/examine/wsl.rs) (the WSL probe, including the remote probe that reads a named distribution from the Windows host, and the distro-floor check) and [`windows.rs`](../crates/rocm-core/src/examine/windows.rs) (the native Windows probes: the GPU query, the HIP SDK, the Adrenalin driver and the MSVC redistributable, plus `parse_dotted_version`, which `diagnose/windows.rs` reads). - [`gfx_target.rs`](../crates/rocm-core/src/gfx_target.rs) — gfx-target identification and the TheRock package-family tables, split out of [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) so [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) can read the family mapping without importing the module that probes the host: `extract_first_gfx_token`, `normalize_therock_family`/`known_therock_families`, `gfx_target_has_no_rocm_build`, the AMD PCI-device-id and marketing-name tables behind `gfx_target_from_pnp_device_id`, and the parsers for the Windows display text and the kernel's gfx-target version. The module is private; `lib.rs` re-exports four of the public names (all but `gfx_target_from_pnp_device_id`) and `host_gpu.rs`, as before, re-exports all five. - [`gguf.rs`](../crates/rocm-core/src/gguf.rs) — reads the shape of a GGUF model (layers, KV heads, head width, parameter count, data offset) from its header over any `Read + Seek`, never touching the weights. - [`gguf_fit.rs`](../crates/rocm-core/src/gguf_fit.rs) — pure arithmetic over that header: weights + KV cache at a context size and KV type + compute buffer + runtime reserve against dedicated VRAM, the largest context that fits, and what would make a model that does not fit, fit. - [`gguf_source.rs`](../crates/rocm-core/src/gguf_source.rs) — where a header comes from: a local file, or a Hugging Face file, found by one repository-listing request (and a `HEAD` request for a file the listing does not size) and read by HTTP range requests (`HF_ENDPOINT` selects a mirror; redirects are followed by hand, up to five, and never from https to plain http). The only code in the estimate that reads a file or the network. - [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) — (ROCMAI-85, Phase 7a of ROCMAI-27's sequencing) GPU/host hardware detection — see the module's own doc comment for what it owns. Its leaf concerns are child modules under `host_gpu/`, with `host_gpu.rs` kept as the module root: [`capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs) (running an optional tool under a timeout, trying each spelling [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs) gives for it), [`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) (kernel version, distro name, CPU model and installed RAM), [`linux.rs`](../crates/rocm-core/src/host_gpu/linux.rs) (the sysfs, KFD and DRM probes), [`gpu_visibility.rs`](../crates/rocm-core/src/host_gpu/gpu_visibility.rs) (which AMD GPUs this process may use, from the KFD/DRM counts and the `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` masks, plus `windows_amd_gpu_listing`, the native-Windows adapter bound `rocm serve --gpu ` is checked against), [`windows_inventory.rs`](../crates/rocm-core/src/host_gpu/windows_inventory.rs) (the Windows display-adapter inventory and its parsers, including the AMD-adapter count behind `windows_amd_gpu_listing`), [`wsl.rs`](../crates/rocm-core/src/host_gpu/wsl.rs) (WSL detection, the librocdxg chain and the Windows host driver as seen from the guest), [`render.rs`](../crates/rocm-core/src/host_gpu/render.rs) (the `rocm examine` text of an `ExamineSummary`, as a second `impl` block), [`diagnostics.rs`](../crates/rocm-core/src/host_gpu/diagnostics.rs) (the GPU detection diagnostics text) and [`engine_default.rs`](../crates/rocm-core/src/host_gpu/engine_default.rs) (the default serve engine, and the vLLM preference for the TheRock families that serve through it). -- [`host-probing.md`](host-probing.md) — a design note, not a module: [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) and [`examine.rs`](../crates/rocm-core/src/examine.rs) are two separate host-probing stacks that read some of the same facts twice (kernel version, CPU model, the Windows GPU query, the marketing-name table and the subprocess runners). It proposes which of each pair should survive, how the two differ and the order in which they could converge, and records what each convergence decided; it changes no code itself. The `PATH` lookup is the first pair converged: [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs)'s `which` is now the only copy. It is a snapshot, to be retired when the convergence tickets close. +- [`host-probing.md`](host-probing.md) — a design note, not a module: [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) and [`examine.rs`](../crates/rocm-core/src/examine.rs) are two separate host-probing stacks that read some of the same facts twice (CPU model, the Windows GPU query, the marketing-name table and the subprocess runners). It proposes which of each pair should survive, how the two differ and the order in which they could converge, and records what each convergence decided; it changes no code itself. Two pairs are converged: the `PATH` lookup ([`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs)'s `which` is now the only copy) and the kernel release ([`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs)'s `detect_kernel_version` is now the only `uname -r` reader, which [`host_env.rs`](../crates/rocm-core/src/examine/host_env.rs)'s `probe_os` calls for `kernel_release`, and `rocm examine --json` runs it once: [`examine.rs`](../crates/rocm-core/src/examine.rs)'s `Examination::probe_with_summary` probes the host and hands the examination's kernel release to the summary). It is a snapshot, to be retired when the convergence tickets close. - [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs) — the JSONL store engine behind the automation stores: the sibling `.lock` file (`jsonl_lock_path`, re-exported at the crate root), single-line appends, torn-tail repair, size-based rotation (`JsonlRotation`) and the tail-first readers that span rotated generations. It is generic over the record type; [`automation.rs`](../crates/rocm-core/src/automation.rs) supplies the record types and each store's policy, and nothing in this module depends on it, in production or in its own tests: those drive the engine with a small record type of their own. The module is private; its engine items are `pub(crate)`. - [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) — (ROCMAI-85, Phase 7a) the TheRock managed-runtime-environment surface — see the module's own doc comment for what it owns. - [`model_catalog.rs`](../crates/rocm-core/src/model_catalog.rs) — (ROCMAI-152, Phase 7c) the curated model catalog's types, built-in catalog JSON loading, signed external recipe indexes, and artifact cache-marker bookkeeping — see the module's own doc comment for what it owns. diff --git a/docs/host-probing.md b/docs/host-probing.md index 057f5517e0..f2a7444a55 100644 --- a/docs/host-probing.md +++ b/docs/host-probing.md @@ -6,7 +6,7 @@ SPDX-License-Identifier: MIT # Host probing: which stack owns each fact -Status: proposal, with the first pair delivered. This note changes no code and no behaviour itself, and what it proposes for a pair is decided by the change that converges that pair, which then records the decision here (the PATH lookup, pair 1, has: see "PATH lookup"). It lists the host facts that `rocm-core` reads twice (on the hosts each is read on, see "Which host reads which fact"), says how the two reads differ, and proposes which one should survive and in what order a later series of changes could remove the other. Each pair is meant to be a separate change with its own review. +Status: proposal, with the first two pairs delivered. This note changes no code and no behaviour itself, and what it proposes for a pair is decided by the change that converges that pair, which then records the decision here (the PATH lookup, pair 1, and the kernel release, pair 2, have: see "PATH lookup" and "Kernel version"). It lists the host facts that `rocm-core` reads twice (on the hosts each is read on, see "Which host reads which fact"), says how the two reads differ, and proposes which one should survive and in what order a later series of changes could remove the other. Each pair is meant to be a separate change with its own review. This note is a snapshot of `main` at the time of writing, not maintained documentation. It is to be retired (the file deleted, together with its entry in [architecture.md](architecture.md)) when the convergence tickets for these pairs close, so the inventory does not keep pointing at a stale design record. @@ -18,7 +18,7 @@ For each pair, the proposed survivor and the biggest difference a user can see t | Pair | Proposed survivor | Biggest user-visible difference today | |---|---|---| -| [Kernel version](#kernel-version) | `detect_kernel_version`, called by `probe_os` for `kernel_release` on Linux (the timeout is open) | where `uname` is slow or fails, `kernel:` and `kernel_release` can disagree (UNCONFIRMED); on Windows `kernel_release` is always empty | +| [Kernel version](#kernel-version) | delivered: `detect_kernel_version`, called by `probe_os` for `kernel_release` on Linux; one 5 s limit and a strict read | before the change, where `uname` was slow or failed, `kernel:` and `kernel_release` could disagree (UNCONFIRMED on a real host); on Windows `kernel_release` is still always empty (out of this pair) | | [CPU model](#cpu-model) | one reader in `os_facts` returning model and vendor, called by `probe_cpu_linux` and `probe_cpu_windows` | on WSL the text shows the CPU while `cpu_model` is empty and `cpu_vendor` is `unknown`; the change leaves that as it is (option (a)) | | [Windows GPU query](#windows-gpu-query) | the summary stack's inventory, with `probe_gpus_windows` building `gpus[]` from it and NVIDIA kept in the JSON | an NVIDIA-only host reads "AMD GPU not detected yet" in the text beside `has_nvidia_gpu: true` in the JSON | | [Marketing name to gfx target](#marketing-name-to-gfx-target) | the `AMD_MARKETING_GFX_TARGETS` table, gaining the codenames `classify_amd_marketing_name` knows (finish [#521](https://github.com/ROCm/rocm-cli/pull/521)) | on Windows without a HIP SDK an RX 7900 XTX has an empty `gpus[].gfx_target` while `summary.detected_gfx_target` is `gfx1100` | @@ -46,7 +46,7 @@ Every pair below is read on a subset of the hosts, and the subset decides which | Fact | Native Linux | WSL (probing inside the distro) | Native Windows | Windows host inspecting a WSL distro (`probe_wsl_distro_from_host`) | |---|---|---|---|---| -| Kernel version | both, `uname -r` | both: `probe_os` runs before the WSL branch returns | summary `kernel` from `cmd /C ver`; `kernel_release` stays empty | examination only, from `uname -r` run inside the distro by a shell script (`WSL_REMOTE_PROBE`), a third implementation | +| Kernel version | both, through `detect_kernel_version` (before the change each ran `uname -r` itself) | both: `probe_os` runs before the WSL branch returns | summary `kernel` from `cmd /C ver`; `kernel_release` stays empty | examination only, from `uname -r` run inside the distro by a shell script (`WSL_REMOTE_PROBE`), a third implementation | | CPU model and vendor | both | summary only: the examination returns before `probe_cpu_linux`, so `cpu_model` is empty and `cpu_vendor` is `unknown` | both, plus the inventory row | neither: `cpu_model` empty | | Windows adapter list | not applicable | examination `gpus[]` stays empty (the WSL branch skips the GPU probes); the summary asks the Windows host for the display target only | both | not collected here; the host driver is read through the summary's inventory (`detect_local_windows_host_driver`) | | Marketing name to gfx target | examination only (the `lspci` scan); the summary reads sysfs and KFD, not names | summary only, through the Windows host's display name | both | not collected | @@ -57,7 +57,7 @@ A pair that one host reads with only one stack cannot disagree on that host, but ## Kernel version -Verdict: confirmed duplicate for `uname -r`. `uname -v` in `probe_os` has no twin. +Verdict: confirmed duplicate for `uname -r`, now converged (see "Decided and delivered" below). `uname -v` in `probe_os` has no twin. The comparison that follows is how the two copies differed before that change, kept as the record of what it moved. | | Summary | Examination | |---|---|---| @@ -73,6 +73,32 @@ What a user can see: on a native Linux or WSL host where `uname` is slow or fail Proposal: `detect_kernel_version` survives and `probe_os` calls it for `kernel_release` on Linux, keeping `uname -v` where it is. Calling it on Windows too would populate `kernel_release` there, which is a visible change to the JSON; decide that separately. The timeout is open (1.5 s or 5 s): the characterisation test should pin both before one is chosen. Hosts the test covers: native Linux and WSL (one code path, since `probe_os` runs before the WSL branch returns), with a fake `uname` that succeeds, exits non-zero with stdout, times out, and prints nothing; plus a WSL 1 style and a WSL 2 style string through `probe_wsl`, because `is_wsl1_kernel` reads the field. Windows is covered only by a unit test behind an injectable runner, which pins that `kernel_release` stays empty unless that change is decided. The distro probe's shell `uname -r` has no twin to compare and is out of this pair. +### Decided and delivered + +The change took the proposed survivor: `detect_kernel_version` in [os_facts.rs](../crates/rocm-core/src/host_gpu/os_facts.rs) is the one `uname -r` reader, and `probe_os` in [host_env.rs](../crates/rocm-core/src/examine/host_env.rs) calls it for `kernel_release` on Linux, native and WSL alike, with a failed read left empty. It is also one read per inspection where one command runs both stacks: `rocm examine --json` builds both through `Examination::probe_with_summary`, which probes this host and gathers the summary from that examination with the crate-private `ExamineSummary::gather_after` (so no caller can pair the summary with an examination of another host, such as one deserialised from a report), and the summary takes the examination's `kernel_release` (a failed read included) on Linux instead of running `uname -r` again, so the two fields cannot disagree on a transient failure and a hung `uname` costs one timeout, not two. Callers that gather only the summary (the text form, the MCP and bridge snapshots) still read once for themselves. `uname -v` stays in `probe_os`. Windows is out of this change: the examination still leaves `kernel_release` empty there and the summary still reads `cmd /C ver` under the shorter optional-command timeout; whether `kernel_release` should be filled on Windows stays open, and so does the Windows unit test the proposal mentioned, because the Windows branches did not change. + +The four open questions are decided as follows. + +- Timeout: 5 s, the longer of the two old limits (`KERNEL_RELEASE_TIMEOUT`, written out as five seconds rather than borrowed from `SHORT`, which is also 5 s today, and pinned by `the_kernel_release_timeout_is_five_seconds`). `uname -r` is a few bytes the kernel hands over, so a read that takes seconds is a stalled host, where waiting costs little and a spurious "unknown" costs more: `kernel_release` feeds the WSL version check and the kernel-dependent diagnoses, which read an empty value as nothing to report. The cost is that every caller of `ExamineSummary::gather` and `gather_without_driver_facts` that reads the kernel for itself (the text of `rocm examine`, the MCP and bridge snapshots, the free-form answer, `rocm install driver`, the runtime-install finalizer, and `rocmd`'s host snapshot, MCP and sandbox paths) can wait up to 5 s instead of 1.5 s on a host whose `uname` hangs; none of them is on a path run for every command. `rocm examine --json` waits for one timeout, because it hands the examination's read to the summary. +- Failure: `None` in the summary (`` in the text, null in the JSON) and an empty `kernel_release` in the examination, with no `probe_failures` entry, as before. Adding an entry would change `status` to degraded on a host without a usable `uname`, such as a minimal container, which is a bigger change than the pair warrants. An empty `kernel_release` reads as WSL 2 in `is_wsl1_kernel`, as before. +- Non-zero exit with output: dropped by both. The examination used to keep what a failing `uname` printed; the summary dropped it. A release from a `uname` that failed is not trustworthy. +- Decoding: strict UTF-8 for both. The examination used to decode lossily. A release is ASCII in practice, so the only input that moves is one that is not UTF-8 at all, and a string with replacement characters in it would reach the diagnoses and the WSL check as if it were a release. + +The answers that moved, each with the test that pins it: + +| Difference | Before | Now | Pinned by | +|---|---|---|---| +| `uname` prints a release and exits non-zero | summary: none; examination: the release | none in both | `a_non_zero_exit_with_output_is_dropped_by_both`, scenario `examine-44` | +| `uname` output that is not UTF-8 | summary: none; examination: the release with the bad byte replaced | none in both | `output_that_is_not_utf8_is_dropped_by_both` | +| `uname` that answers after 2 s | summary: none (1.5 s); examination: the release (5 s) | the release in both | `an_answer_after_two_seconds_is_read_by_both` | +| `uname` that never answers | none in both | none in both; the unit test's child reads twice, each bounded by the one 5 s limit | `a_uname_that_never_answers_reads_as_nothing_in_both_within_the_timeout` (it checks the elapsed time, since the answers alone cannot tell a timeout from none) | +| `uname` that prints nothing | none in both | unchanged | `a_silent_uname_reads_as_nothing_in_both` | +| WSL 1 and WSL 2 style releases | read as printed by both | unchanged; `probe_wsl` makes version 1 of the first and version 2 of the second, of an empty one and of the original `-microsoft-standard` form | `both_read_wsl_style_releases_as_printed`, `probe_wsl_reads_the_wsl_version_from_the_kernel_release` | +| A failed read and `probe_failures` | no entry | no entry | the check in the same tests that no case leaves a `probe_failures` entry about the kernel, and scenario `examine-44` | +| Reads in one `rocm examine --json` run | two (`probe_os` and the summary each ran `uname -r`) | one: the summary takes the examination's | `the_summary_reads_the_kernel_only_when_the_examination_did_not` (a counted read), and the last step of scenario `examine-43`, which counts the calls of a stand-in `uname` | + +The scenarios `examine-43` and `examine-44` in [examine.feature](../tests/e2e-cucumber/features/examine.feature) drive the observable answer on the Linux lane, native and WSL: with a stand-in `uname` first on `PATH`, one `rocm examine --json` run holds `summary.kernel` and `kernel_release` level on a specific release and shows from the stand-in's call log that `uname -r` ran once, and, for a `uname` that prints a release and then fails, holds both empty with no kernel `probe_failures` entry. The unit tests plant the same stand-in in a child copy of the test binary with its own `PATH`, because a test thread cannot change `PATH` without disturbing every other test. + ## CPU model Verdict: confirmed duplicate on native Linux and Windows, and on Windows three implementations. On WSL it is not a duplicate: only the summary reads the CPU there (see the table above and the paragraph on WSL below). The three Windows implementations are `detect_cpu_model` (its own PowerShell call), the `CPU` row of `WINDOWS_SYSTEM_INVENTORY_SCRIPT` in [windows_inventory.rs](../crates/rocm-core/src/host_gpu/windows_inventory.rs), and `probe_cpu_windows`. They are not three reads of one run in the usual case: the summary reads the inventory row first and calls `detect_cpu_model` only when no inventory exists, so a host whose inventory answers is read twice in `rocm examine --json` and one whose inventory does not answer is read three times (derived from `detect_cpu_model_with_windows_inventory` in [host_gpu.rs](../crates/rocm-core/src/host_gpu.rs); UNCONFIRMED on a live host). @@ -205,7 +231,7 @@ Proposal: `run_with_timeout` survives as the one bounded executor, with `Command One pair per change. Each begins with a commit that adds a characterisation test and is green against the code as it is: it pins what both copies return for the same input, and where they differ it pins each copy's answer, with a comment naming the pair. The convergence commit then flips the assertions that are meant to change and the PR text lists each one as a user-visible difference. Tests that need a program on `PATH` use the fake-executable helper in [fake_executable.rs](../crates/rocm-core/src/fake_executable.rs); Windows-only logic goes behind an injectable function so a unit test runs on the Windows job, as the inventory already does with `WindowsInventorySources`. 1. PATH lookup (delivered: see "Decided and delivered" under "PATH lookup"). Smallest, and it settles where `tool_path_candidates` lives before the runner work needs it. It is not output-preserving: the differences listed under "PATH lookup" change which tools are reported as present, and its PR text lists each one as intentional, with the characterisation test that pins it. Hosts covered: Linux, which is also WSL's code path, for the empty entry, the non-UTF-8 `PATH` (set through an `OsString` environment value) and the current-directory file; Windows through an injectable resolver for the `PATHEXT` spellings, the System32 candidates and the `;` splitting, since the Linux lane cannot run them. -2. Kernel version. A single command; settles the timeout and the empty-versus-`None` question. Hosts covered as set out under "Kernel version". +2. Kernel version (delivered: see "Decided and delivered" under "Kernel version"). A single command; it settled the timeout and the empty-versus-`None` question. Hosts covered as set out under "Kernel version". 3. CPU model. Adds the model-and-vendor reader; needs the Windows inventory row, which already exists. Its PR says which WSL option under "CPU model" it takes, (a) or (b); a PR that does not is not ready. 4. Marketing name to gfx target, by finishing [#521](https://github.com/ROCm/rocm-cli/pull/521). It must land before the next item, so that `gpus[]` is not rebuilt on top of two name tables. 5. Windows GPU query. The riskiest: it needs an answer on NVIDIA and on adapter names, and a Windows lane. Scenario `examine-windows-driver-version-date-and-age` already runs there; a new scenario should compare the text's `detected_gfx_target` with the JSON `gpus[]` on a named adapter. Native Windows only; no scenario can run this on the Linux or WSL lanes. @@ -221,7 +247,7 @@ No item is output-preserving without qualification, and the PR for each says whi - Whether one `rocm examine --json` run shares answers between the stacks (a single WSL summary and a single `ldconfig -p`, one inventory including its failure) or keeps calling shared code that asks again. - What an absent Windows inventory does once there is one inventory: remember the absence for the run, or drop the per-fact fallbacks. - Whether `gpus[].name` should come from `pnputil` or from WMI. -- Which timeout wins for the kernel and CPU reads, and whether `kernel_release` should be filled on Windows. +- Which timeout wins for the CPU read, and whether `kernel_release` should be filled on Windows (the kernel timeout is decided, see "Kernel version"). - Whether the typed outcome belongs on `CommandCapture` or beside it. - Whether CPU facts should be collected on WSL at all (option (b) under "CPU model"), and if so how the Windows-hosted distro probe obtains them. - Whether the adapters over `run_with_timeout` take a per-call drain grace, so that a probe whose tool is known to leave a late-writing descendant keeps its output, or whether the 200 ms grace is accepted for every probe. diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index f5237db55f..81a3252e6e 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -639,3 +639,35 @@ Feature: GPU detection and system inspection When the user inspects that machine in machine-readable form Then the inspection reports kfd as loaded And the inspection reports amdgpu as loaded + + # `rocm examine --json` reads the kernel release twice: the host summary + # (`summary.kernel`, which the text form prints as `kernel:`) and the examination + # (`kernel_release`, which the WSL version check and the kernel-dependent + # diagnoses read). They used to run `uname -r` separately, with different limits + # and different rules for a failed or malformed answer, and could disagree. They + # share one read now, so one `--json` run holds the two level on a specific + # value, not only on both being filled, and the stand-in counts its calls to show + # that the run asked for the release once: the summary takes the examination's + # read, so a transient failure cannot make the two fields disagree and a hung + # `uname` costs one timeout. + # + # The stand-in `uname` is planted first on PATH, so the scenario describes a + # kernel no runner has. It runs on every Linux lane, native or WSL, because both + # stacks read `uname -r` on both; Windows is out of scope, where the examination + # leaves `kernel_release` empty and the summary reads `cmd /C ver`. + @id:examine-reports-one-kernel-release-in-both-forms @requires-os:linux + Scenario: examine-43 - The summary and the examination report the same kernel release + Given a machine whose uname reports a kernel release + When the user inspects that machine in machine-readable form + Then the summary and the examination report that kernel release + And that run asked uname for the kernel release once + + # A `uname` that fails is not trusted for what it printed first: the examination + # used to keep that release while the summary dropped it. Neither reports a + # kernel now, and a failed read is not a probe failure, so a host without a + # usable `uname` does not read as degraded. + @id:examine-reports-no-kernel-release-from-a-failing-uname @requires-os:linux + Scenario: examine-44 - Neither report keeps a kernel release from a uname that failed + Given a machine whose uname prints a kernel release and then fails + When the user inspects that machine in machine-readable form + Then neither the summary nor the examination reports a kernel release, and neither calls it a failure diff --git a/tests/e2e-cucumber/src/lib.rs b/tests/e2e-cucumber/src/lib.rs index f3969cee22..27fc415a4f 100644 --- a/tests/e2e-cucumber/src/lib.rs +++ b/tests/e2e-cucumber/src/lib.rs @@ -29,6 +29,7 @@ pub mod send_until; pub mod serve_log; pub mod serve_plan; pub mod shared_runtime; +pub mod stand_in_uname; pub mod terminal_size; use std::path::{Path, PathBuf}; diff --git a/tests/e2e-cucumber/src/stand_in_uname.rs b/tests/e2e-cucumber/src/stand_in_uname.rs new file mode 100644 index 0000000000..2be0e2a765 --- /dev/null +++ b/tests/e2e-cucumber/src/stand_in_uname.rs @@ -0,0 +1,72 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! The shell script a scenario plants as a stand-in `uname`. +//! +//! `-v` answers a fixed string, so a probe that also asks for the kernel version +//! has nothing to wait for; `-r` runs a fragment the scenario chooses; and every +//! call appends its arguments to a log, so a scenario can count how often the +//! release was asked for. The log lives under the scenario's isolated root, whose +//! path the harness does not control, so it is quoted for the shell rather than +//! put between bare single quotes. + +use std::path::Path; + +/// `value` as one single-quoted shell word: each embedded single quote closes the +/// quotes, is written escaped and reopens them. (`rocm_core::quoting` has the same +/// quoting, but `rocm-core` is only a dev-dependency of this library.) +fn single_quoted(value: &str) -> String { + format!("'{}'", value.replace('\'', r"'\''")) +} + +/// The text of the stand-in `uname`: log the arguments to `log`, answer `-v`, and +/// otherwise run `release_body` (a shell fragment). +#[must_use] +pub fn stand_in_uname_script(release_body: &str, log: &Path) -> String { + let log = single_quoted(&log.to_string_lossy()); + format!( + "#!/bin/sh\necho \"$*\" >> {log}\nif [ \"$1\" = -v ]; then echo '#1 SMP stand-in'; exit 0; fi\n{release_body}\n" + ) +} + +#[cfg(all(test, unix))] +mod tests { + use super::*; + use std::process::Command; + + /// Plant the script in a directory whose name holds a single quote and a + /// space, run it, and read back what it logged and printed. + fn run_in_awkward_directory(args: &[&str], body: &str) -> (String, String) { + let scratch = tempfile::tempdir().expect("scratch directory"); + let dir = scratch.path().join("it's a dir"); + std::fs::create_dir_all(&dir).expect("create the awkward directory"); + let log = dir.join("uname calls.log"); + let script = dir.join("uname"); + rocm_core::fake_executable::write_fake_executable( + &script, + stand_in_uname_script(body, &log), + ) + .expect("plant the stand-in"); + let output = Command::new(&script).args(args).output().expect("run it"); + assert!(output.status.success(), "{output:?}"); + ( + String::from_utf8_lossy(&output.stdout).into_owned(), + std::fs::read_to_string(&log).expect("the stand-in logged its call"), + ) + } + + #[test] + fn the_log_path_survives_a_single_quote_and_a_space_in_it() { + let (stdout, log) = run_in_awkward_directory(&["-r"], "echo 6.8.0-stand-in"); + assert_eq!(stdout.trim(), "6.8.0-stand-in"); + assert_eq!(log, "-r\n"); + } + + #[test] + fn dash_v_answers_a_fixed_string_and_is_logged_too() { + let (stdout, log) = run_in_awkward_directory(&["-v"], "echo not-reached"); + assert_eq!(stdout.trim(), "#1 SMP stand-in"); + assert_eq!(log, "-v\n"); + } +} diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 0044e897c7..c60f5025c8 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -2703,6 +2703,164 @@ async fn assert_rocminfo_not_installed_in_either_report(world: &mut E2eWorld) { ); } +// ── Both reports read the kernel release from one `uname -r` ─────── + +/// The release the stand-in `uname` reports, chosen so no real host has it. +const STAND_IN_KERNEL_RELEASE: &str = "6.8.0-stand-in-kernel"; + +/// Where the stand-in `uname` logs the arguments of every call, one line each, +/// so a scenario can count the reads. Inside the isolated root. +fn stand_in_uname_log(world: &E2eWorld) -> PathBuf { + world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("uname-calls.log") +} + +/// Plant a `uname` first on the `PATH` of the next `rocm` run. `-r` runs +/// `release_body` (a shell fragment); `-v` answers a fixed string, so the +/// examination's other `uname` read has nothing to wait for. Every call appends +/// its arguments to [`stand_in_uname_log`]. Nothing else on the +/// machine is planted: the stand-in is what `rocm examine` reads on any Linux +/// host, native or WSL, because both stacks run `uname -r` there. +fn plant_stand_in_uname(world: &mut E2eWorld, release_body: &str) { + let bin = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("uname-bin"); + std::fs::create_dir_all(&bin).expect("failed to create the stand-in uname directory"); + let log = stand_in_uname_log(world); + rocm_core::fake_executable::write_fake_executable( + &bin.join("uname"), + e2e_cucumber::stand_in_uname::stand_in_uname_script(release_body, &log), + ) + .expect("failed to write the stand-in uname"); + let inherited = std::env::var_os("PATH").unwrap_or_default(); + let path = std::env::join_paths(std::iter::once(bin).chain(std::env::split_paths(&inherited))) + .expect("PATH entries cannot hold a separator"); + world.command_env.retain(|(key, _)| *key != "PATH"); + world.command_env.push(("PATH", path)); +} + +#[given("a machine whose uname reports a kernel release")] +async fn machine_whose_uname_reports_a_release(world: &mut E2eWorld) { + plant_stand_in_uname(world, &format!("echo {STAND_IN_KERNEL_RELEASE}")); +} + +#[given("a machine whose uname prints a kernel release and then fails")] +async fn machine_whose_uname_prints_a_release_and_fails(world: &mut E2eWorld) { + plant_stand_in_uname(world, &format!("echo {STAND_IN_KERNEL_RELEASE}; exit 3")); +} + +/// The `probe_failures` entries of an `examine --json` document, whatever they +/// say. +fn probe_failures(json: &serde_json::Value) -> Vec { + json.get("probe_failures") + .and_then(serde_json::Value::as_array) + .unwrap_or_else(|| panic!("`examine --json` has no probe_failures array:\n{json:#}")) + .iter() + .filter_map(serde_json::Value::as_str) + .map(str::to_owned) + .collect() +} + +/// The `probe_failures` of the same machine with a `uname` that works, for a +/// baseline: entries this machine legitimately has for other reasons are not +/// the kernel read's, and a failure the failing `uname` adds, however worded, +/// is. +fn probe_failures_with_a_working_uname(world: &E2eWorld) -> Vec { + let bin = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("uname-baseline-bin"); + std::fs::create_dir_all(&bin).expect("failed to create the baseline uname directory"); + rocm_core::fake_executable::write_fake_executable( + &bin.join("uname"), + e2e_cucumber::stand_in_uname::stand_in_uname_script( + &format!("echo {STAND_IN_KERNEL_RELEASE}"), + &world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("uname-baseline-calls.log"), + ), + ) + .expect("failed to write the baseline uname"); + let inherited = std::env::var_os("PATH").unwrap_or_default(); + let path = std::env::join_paths(std::iter::once(bin).chain(std::env::split_paths(&inherited))) + .expect("PATH entries cannot hold a separator"); + let path = path.to_str().expect("PATH is UTF-8").to_owned(); + let (stdout, stderr, rc) = crate::run_rocm_with_env( + world, + &["examine", "--framework", "skip", "--json"], + &[("PATH", path.as_str())], + ); + assert_eq!( + rc, 0, + "the baseline `rocm examine --json` failed:\n{stderr}" + ); + let json: serde_json::Value = + serde_json::from_str(&stdout).expect("the baseline run emitted JSON"); + probe_failures(&json) +} + +#[then("the summary and the examination report that kernel release")] +async fn assert_both_report_the_kernel_release(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "`rocm examine --json` failed"); + let json = parsed_json(world); + assert_eq!( + inspection_str(&json, "/summary/kernel"), + STAND_IN_KERNEL_RELEASE, + "{json:#}" + ); + assert_eq!( + inspection_str(&json, "/kernel_release"), + STAND_IN_KERNEL_RELEASE, + "{json:#}" + ); +} + +/// One `rocm examine --json` run asks `uname` for the release once, for the +/// examination and the summary together: the stand-in logged a single `-r`. A +/// second read would show as a second line, and it is what lets a transient +/// failure make the two fields disagree. +#[then("that run asked uname for the kernel release once")] +async fn assert_uname_was_asked_for_the_release_once(world: &mut E2eWorld) { + let log = std::fs::read_to_string(stand_in_uname_log(world)) + .expect("the stand-in uname was never called"); + let release_reads = log.lines().filter(|line| line.trim() == "-r").count(); + assert_eq!(release_reads, 1, "calls of the stand-in uname:\n{log}"); +} + +#[then( + "neither the summary nor the examination reports a kernel release, and neither calls it a failure" +)] +async fn assert_neither_reports_a_kernel_release(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "`rocm examine --json` failed"); + let json = parsed_json(world); + assert!( + json.pointer("/summary/kernel") + .is_some_and(serde_json::Value::is_null), + "the summary must read no kernel from a uname that failed:\n{json:#}" + ); + assert_eq!(inspection_str(&json, "/kernel_release"), "", "{json:#}"); + // No `probe_failures` entry beyond what the same machine has with a working + // `uname`, so a failure worded in any way cannot slip through. + let baseline = probe_failures_with_a_working_uname(world); + let new_failures: Vec = probe_failures(&json) + .into_iter() + .filter(|failure| !baseline.contains(failure)) + .collect(); + assert_eq!(new_failures, Vec::::new(), "{json:#}"); +} + // ── `rocm serve` on a WSL host whose ROCDXG library the loader cannot find ── #[when("the user serves a model under the GPU-required default on that machine")]