From 1a5329d4925eb84528ad776cad91c5a2f41442f4 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 10 Nov 2026 11:59:07 +0200 Subject: [PATCH 1/2] fix(dash): initialise ADLX once on a resident thread MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On Windows the dashboard's in-process collector initialised and terminated ADLX on every 1s tick, on a different blocking-pool thread each time. That loaded and unloaded amdadlx64.dll (and the d3d9, OPENGL32 and dbghelp it pulls in) repeatedly while the driver's atiadlxx.dll stayed resident. After a while atiadlxx.dll fail-fasts (0xC0000409, FAST_FAIL_FATAL_APP_EXIT), which kills rocm dash with no panic message or stack trace. Initialise ADLX once on a dedicated reader thread that keeps the helper for the process lifetime and answers read requests over a channel. Init is attempted once per process. The init and read steps are injectable so the once-only and same-thread contract is unit-tested on every platform. The reader answers with a Result, so a caller can tell an absent driver from a reader that failed. A failed init answers with no readings and no warning, as before, since that is what a machine with no AMD GPU looks like. A read that panics is caught, reported as a warning and leaves the reader usable; a reader whose thread is gone is reported too. Without that split, one failure would blank temperature, power and clock for the rest of the session and be indistinguishable from having no driver — the reader is built once per process and never replaced. The crash was not reproduced outside the dashboard, so the per-tick init/terminate cycle is the leading cause rather than a proven one. The manual-testing checklist gains a 30 minute long-run check for it. Signed-off-by: Roman Sirokov --- crates/rocm-dash-collectors/src/winperf.rs | 326 ++++++++++++++++++--- docs/manual-testing.md | 8 + 2 files changed, 295 insertions(+), 39 deletions(-) diff --git a/crates/rocm-dash-collectors/src/winperf.rs b/crates/rocm-dash-collectors/src/winperf.rs index 7114f2c1cb..1a1a8fabf1 100644 --- a/crates/rocm-dash-collectors/src/winperf.rs +++ b/crates/rocm-dash-collectors/src/winperf.rs @@ -42,10 +42,12 @@ use tokio::time::timeout; // actually-hung process forever. const RUN_TIMEOUT: Duration = Duration::from_secs(15); -/// ADLX init/read/terminate shouldn't normally take anywhere near this, but a -/// wedged driver call must not freeze CPU/memory updates forever alongside it -/// — same reasoning as `RUN_TIMEOUT`, just a tighter bound since there's no -/// process spawn latency to absorb here. +/// Bounds the whole request to the resident reader — handing the request +/// over, the driver read on that thread, and the answer coming back. None of +/// it should take anywhere near this, but a wedged driver call must not +/// freeze CPU/memory updates forever alongside it — same reasoning as +/// `RUN_TIMEOUT`, just a tighter bound since there's no process spawn latency +/// to absorb here. const ADLX_TIMEOUT: Duration = Duration::from_secs(5); /// `kindkeyvalue[value2]` lines: per-engine utilization, @@ -79,20 +81,24 @@ impl WinPerfCollector { /// `counter_samples_missing_warning`) without failing the whole tick. pub async fn snapshot(&self) -> io::Result<(Vec, GpuSystemInfo, Vec)> { let out = Self::run_script().await?; - // adlx_readings() calls into amdadlx64.dll via FFI, which blocks on - // driver I/O — running it directly here would stall a tokio worker - // thread, so it runs under spawn_blocking. A native call cannot be - // cancelled, so the timeout below only stops *waiting*: a wedged - // call keeps its blocking-pool thread. `run_single_flight` therefore - // refuses to start another call while one is still running, which - // bounds the leak to that one thread instead of one per tick. A - // panic, timeout or still-wedged call surfaces as a warning; a - // driver that is absent or fails to initialise inside - // adlx_readings() degrades to no readings with no warning, since - // "no AMD driver" is an expected state indistinguishable from it. + // adlx_readings() hands its request to the resident ADLX reader + // thread and blocks until that thread answers — the FFI into + // amdadlx64.dll runs there, not here. Blocking at all would stall a + // tokio worker, so it still runs under spawn_blocking. A native call + // cannot be cancelled, so the timeout below only stops *waiting*: a + // wedged driver keeps both this blocking-pool thread and the one + // reader. `run_single_flight` therefore refuses to start another call + // while one is still running, which bounds that to a single + // outstanding request instead of one per tick. + // + // A panic, timeout or still-wedged call surfaces as a warning, as + // does the reader failing or dying (ReaderGone). A driver that is + // absent or fails to initialise answers Ok with no readings and no + // warning, since "no AMD driver" is an expected state. let (adlx, adlx_warning) = match run_single_flight(&ADLX_IN_FLIGHT, ADLX_TIMEOUT, adlx_readings).await { - SingleFlight::Done(readings) => (readings, None), + SingleFlight::Done(Ok(readings)) => (readings, None), + SingleFlight::Done(Err(gone)) => (Vec::new(), Some(gone.warning())), SingleFlight::Panicked => { (Vec::new(), Some("ADLX reading task panicked".to_string())) } @@ -409,44 +415,159 @@ fn apply_adlx(reading: Option<&AdlxReading>) -> (f32, f32, Option) { ) } +/// Owns a value that is built once, on its own thread, and serves every read +/// from that thread for as long as the process lives. +/// +/// ADLX is meant to be initialised once per process. The `adlx` crate's +/// `AdlxHelper` loads `amdadlx64.dll` and calls `ADLXInitialize` in `new()`, +/// and `ADLXTerminate` plus the DLL unload in `Drop`. Doing that every tick, +/// each time on a different blocking-pool thread, loads and unloads +/// `amdadlx64.dll` (and the `d3d9`/`OPENGL32`/`dbghelp` it pulls in) over and +/// over while the driver's `atiadlxx.dll` stays resident. After a while +/// `atiadlxx.dll` fail-fasts (`0xC0000409`, `FAST_FAIL_FATAL_APP_EXIT`) +/// and takes `rocm dash` down with no panic message or stack trace. +/// +/// `init` and `read` are parameters so the "built once, same thread for every +/// read" contract is unit-testable without a driver (see the tests). +#[cfg(any(windows, test))] +struct ResidentReader { + requests: std::sync::mpsc::Sender>, +} + +/// Why the reader could not answer, as distinct from answering with no +/// readings. The difference matters: no readings is what an absent driver +/// looks like and is expected, while these mean the reader itself is in +/// trouble and the user is told. Since the reader is built once per process +/// and never replaced, a failure that reported nothing would blank +/// temperature, power and clock for the rest of the session and look exactly +/// like a machine with no AMD driver. +/// +/// Not gated: `snapshot` matches on it and the non-Windows `adlx_readings` +/// stub returns it, on every platform. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum ReaderGone { + /// The read step panicked. Caught, so the reader survives for the next + /// tick; reported so a driver that panics every read is not silent. + #[cfg_attr(not(any(windows, test)), allow(dead_code))] + ReadPanicked, + /// The thread never started, or exited before answering. + #[cfg_attr(not(any(windows, test)), allow(dead_code))] + ThreadGone, +} + +type ReaderAnswer = Result, ReaderGone>; + +impl ReaderGone { + fn warning(self) -> String { + match self { + Self::ReadPanicked => "ADLX read panicked".to_string(), + Self::ThreadGone => "the ADLX reader thread is gone".to_string(), + } + } +} + +#[cfg(any(windows, test))] +impl ResidentReader { + fn spawn( + init: impl FnOnce() -> Option + Send + 'static, + read: impl Fn(&S) -> Vec + Send + 'static, + ) -> Self { + let (requests, inbox) = std::sync::mpsc::channel::>(); + let spawned = std::thread::Builder::new() + .name("adlx-reader".into()) + .spawn(move || { + // `state` is deliberately held until the request channel closes + // (never, for the process-wide reader) so it is not torn down + // and re-initialised between ticks. + let state = init(); + for reply in inbox { + // A panicking read would otherwise unwind this thread and + // leave every later tick answering from a closed channel, + // with no way to tell that from an absent driver. Catching + // it keeps the one reader alive and names the failure. + let answer = match state.as_ref() { + Some(state) => { + std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| read(state))) + .map_err(|_| ReaderGone::ReadPanicked) + } + None => Ok(Vec::new()), + }; + let _ = reply.send(answer); + } + }); + if let Err(e) = spawned { + // Without the thread `inbox` is dropped, so every `read` below + // sees a closed channel and degrades to no readings. + tracing::warn!("failed to start the ADLX reader thread: {e}"); + } + Self { requests } + } + + /// One reading per GPU, or an empty `Ok` when ADLX is unavailable or + /// failed to initialise — an absent driver is an expected state, not a + /// fault. [`ReaderGone`] is reserved for the reader itself failing. + /// Blocks until the reader answers, so callers run it under + /// `spawn_blocking`. + fn read(&self) -> ReaderAnswer { + let (reply, answer) = std::sync::mpsc::channel(); + if self.requests.send(reply).is_err() { + return Err(ReaderGone::ThreadGone); + } + answer.recv().unwrap_or(Err(ReaderGone::ThreadGone)) + } +} + /// `AdlxHelper::new()` fails cleanly (`Err`, not a panic) when -/// `amdadlx64.dll` isn't found, so a total init failure can just bail to an -/// empty Vec via `.ok()?` — same graceful-degrade shape as the rest of this -/// file. A single GPU's metrics query failing, though, must NOT drop that +/// `amdadlx64.dll` isn't found, so a total init failure leaves the reader +/// answering with an empty Vec — same graceful-degrade shape as the rest of +/// this file. A single GPU's metrics query failing, though, must NOT drop that /// GPU's slot (a `filter_map` here would silently change the reading count, /// which `parse_metrics`'s single-reading guard relies on to decide whether /// an ADLX reading can be trusted at all) — it degrades to a zeroed /// `AdlxReading` instead, keeping one slot per GPU. +/// +/// Initialisation is attempted once per process. A missing driver is a static +/// condition, and retrying a failed `ADLXInitialize` would repeat the DLL +/// load/unload cycle that [`ResidentReader`] exists to avoid. #[cfg(windows)] -fn adlx_readings() -> Vec { +fn adlx_readings() -> ReaderAnswer { + static READER: std::sync::OnceLock = std::sync::OnceLock::new(); + READER + .get_or_init(|| ResidentReader::spawn(init_adlx, read_adlx)) + .read() +} + +#[cfg(windows)] +fn init_adlx() -> Option { // `adlx::helper::AdlxHelper::new()` loads `amdadlx64.dll` by bare name // (`libloading::Library::new("amdadlx64.dll")`, no search-order // hardening, no way to pass flags into the `adlx` crate). Restricting // the process-wide default DLL search path to System32 — this call // affects every subsequent bare-name `LoadLibrary` in the whole process, // not just `adlx`'s — closes the planted-DLL-in-PATH/cwd hijack vector - // for that unflagged load. Must run before the first `AdlxHelper::new()` - // call; `Once` makes repeated per-tick calls to this function a no-op - // after the first. - static HARDEN_DLL_SEARCH_PATH: std::sync::Once = std::sync::Once::new(); + // for that unflagged load. Must run before the `AdlxHelper::new()` call, + // which this function makes exactly once per process. #[allow(unsafe_code)] // Win32 FFI - HARDEN_DLL_SEARCH_PATH.call_once(|| unsafe { - // Failure here (`FALSE`) means the process keeps its default search - // order (which can include the cwd) rather than System32-only; still - // attempt the `AdlxHelper` load afterwards; a stricter search path is - // defense in depth, not a precondition for ADLX to work. - let hardened = windows_sys::Win32::System::LibraryLoader::SetDefaultDllDirectories( + let hardened = unsafe { + windows_sys::Win32::System::LibraryLoader::SetDefaultDllDirectories( windows_sys::Win32::System::LibraryLoader::LOAD_LIBRARY_SEARCH_SYSTEM32, + ) + }; + // Failure here (`FALSE`) means the process keeps its default search + // order (which can include the cwd) rather than System32-only; still + // attempt the `AdlxHelper` load afterwards; a stricter search path is + // defense in depth, not a precondition for ADLX to work. + if hardened == 0 { + tracing::warn!( + "SetDefaultDllDirectories failed; amdadlx64.dll load falls back to the default search order" ); - if hardened == 0 { - tracing::warn!( - "SetDefaultDllDirectories failed; amdadlx64.dll load falls back to the default search order" - ); - } - }); + } + adlx::helper::AdlxHelper::new().ok() +} +#[cfg(windows)] +fn read_adlx(helper: &adlx::helper::AdlxHelper) -> Vec { (|| -> Option> { - let helper = adlx::helper::AdlxHelper::new().ok()?; let system = helper.system(); let gpus = system.gpus().ok()?; let services = system.performance_monitoring_services().ok()?; @@ -479,8 +600,8 @@ fn adlx_readings() -> Vec { } #[cfg(not(windows))] -const fn adlx_readings() -> Vec { - Vec::new() +const fn adlx_readings() -> ReaderAnswer { + Ok(Vec::new()) } struct Adapter { @@ -865,6 +986,133 @@ mod tests { assert_eq!(apply_adlx(Some(&reading)), (65.0, 12.5, Some(1900.0))); } + fn reading(temperature_c: f32) -> AdlxReading { + AdlxReading { + temperature_c, + power_w: 0.0, + clock_mhz: None, + total_vram_mb: None, + } + } + + /// The crash this guards against came from initialising and tearing down + /// ADLX on every tick. Initialisation must run exactly once however many + /// reads follow, and every read must be served by that one thread. + #[test] + fn resident_reader_initialises_once_and_reads_on_one_thread() { + use std::sync::Arc; + use std::sync::atomic::AtomicUsize; + + let inits = Arc::new(AtomicUsize::new(0)); + let inits_in_worker = inits.clone(); + let reader = ResidentReader::spawn( + move || { + inits_in_worker.fetch_add(1, Ordering::SeqCst); + Some(std::thread::current().id()) + }, + |init_thread| { + // Temperature encodes whether this read ran on the thread + // that initialised the state. + let same = *init_thread == std::thread::current().id(); + vec![reading(if same { 1.0 } else { 0.0 })] + }, + ); + + for _ in 0..25 { + let out = reader.read().expect("a live reader answers"); + assert_eq!(out.len(), 1); + assert!((out[0].temperature_c - 1.0).abs() < f32::EPSILON); + } + assert_eq!(inits.load(Ordering::SeqCst), 1); + } + + #[test] + fn resident_reader_degrades_to_empty_when_init_fails_and_does_not_retry() { + use std::sync::Arc; + use std::sync::atomic::AtomicUsize; + + let inits = Arc::new(AtomicUsize::new(0)); + let inits_in_worker = inits.clone(); + let reader = ResidentReader::spawn( + move || { + inits_in_worker.fetch_add(1, Ordering::SeqCst); + None::<()> + }, + |()| vec![reading(1.0)], + ); + + for _ in 0..5 { + // An absent driver answers Ok with nothing, never ReaderGone: + // the dashboard must not warn about a machine that simply has no + // AMD GPU. + assert!( + matches!(reader.read(), Ok(readings) if readings.is_empty()), + "a failed init must read as an absent driver, not a reader fault" + ); + } + assert_eq!(inits.load(Ordering::SeqCst), 1); + } + + #[test] + fn a_panicking_read_is_reported_and_leaves_the_reader_usable() { + use std::sync::Arc; + use std::sync::atomic::AtomicUsize; + + let reads = Arc::new(AtomicUsize::new(0)); + let reads_in_worker = reads.clone(); + // Panics on the first read only, so the second shows the reader is + // still answering rather than having unwound. + let reader = ResidentReader::spawn( + || Some(()), + move |()| { + assert!( + reads_in_worker.fetch_add(1, Ordering::SeqCst) != 0, + "driver blew up" + ); + vec![reading(1.0)] + }, + ); + + // The panic is caught and named, never silently empty: the reader is + // built once per process, so a silent failure would blank + // temperature, power and clock for the whole session and read as a + // machine with no AMD driver. + assert_eq!(reader.read().err(), Some(ReaderGone::ReadPanicked)); + let after = reader.read().expect("the reader survives a caught panic"); + assert_eq!(after.len(), 1); + assert!((after[0].temperature_c - 1.0).abs() < f32::EPSILON); + assert_eq!(reads.load(Ordering::SeqCst), 2); + } + + #[test] + fn every_reader_fault_warns_in_words_that_name_adlx_and_tell_the_faults_apart() { + // `snapshot` pairs each of these with no readings, so the state the + // user lands in is "the ADLX columns are blank, and here is why". + // An absent driver is the other case and deliberately has no warning. + for gone in [ReaderGone::ReadPanicked, ReaderGone::ThreadGone] { + let warning = gone.warning(); + assert!( + warning.to_lowercase().contains("adlx"), + "a warning shown next to blank columns must name the subsystem: {warning}" + ); + } + assert_ne!( + ReaderGone::ReadPanicked.warning(), + ReaderGone::ThreadGone.warning(), + "a panicking driver and a dead reader need different text to be told apart" + ); + } + + #[test] + fn a_reader_whose_thread_is_gone_reports_it_rather_than_answering_empty() { + // A sender whose receiver has been dropped is what a dead or + // never-started reader thread looks like from the caller's side. + let (requests, inbox) = std::sync::mpsc::channel(); + drop(inbox); + let dead = ResidentReader { requests }; + assert_eq!(dead.read().err(), Some(ReaderGone::ThreadGone)); + } + #[tokio::test] async fn single_flight_returns_value_and_clears_flag() { static FLAG: AtomicBool = AtomicBool::new(false); diff --git a/docs/manual-testing.md b/docs/manual-testing.md index a2eb7dd454..efc444fd31 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -1007,6 +1007,14 @@ Expected result: - Hiding `amdadlx64.dll` (for example on a non-AMD GPU host) leaves utilization and VRAM usage working and temperature/power/clock at their unmeasured values, with no crash. +- Leave the dashboard open for at least 30 minutes on an AMD host with the + driver's `amdadlx64.dll` present. It must keep running and keep updating + temperature, power and clock. ADLX is initialised once for the whole + session; initialising it on every tick made the driver's `atiadlxx.dll` + abort the process (exit code `0xC0000409`) with no message. If the + dashboard still exits silently, Event Viewer > Windows Logs > Application + shows an `Application Error` entry for `rocm.exe` naming the faulting + module and offset. - No other output of this section needs a daemon: `rocm dash` must not report a failed daemon connection. From a3bbda02efced9c7a800715cb8fc6a0aa6625ee2 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 12 Nov 2026 10:53:53 +0200 Subject: [PATCH 2/2] fix(dash): catch an ADLX init panic, and test the snapshot mapping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups on the resident ADLX reader. Both findings were real. `read` was wrapped in `catch_unwind` and `init` was not, and the asymmetry mattered more on the init side. A panic escaping `init` unwound the reader thread before its request loop started, so `inbox` dropped and every later tick answered `ThreadGone` — the one verdict that cannot be told apart from "the thread never started", for a cause that is neither. Because the reader is built once per process, that verdict was permanent, where the previous per-tick design would simply have retried init on the next tick. Init is now caught the same way, with its own `InitPanicked` variant rather than being folded into `ThreadGone`: the two are different faults and the remedy differs. This does not restore per-tick retrying — that is the trade this design makes deliberately — but it keeps the thread answering and names the real cause. Verified by mutation: dropping the `catch_unwind` turns the new test red with the reader thread unwinding, which is the reported defect exactly. The `SingleFlight::Done(Err(_))` arm added by this PR was the one piece of new wiring nothing executed: `snapshot` calls `run_script()` unconditionally, so no test can drive it. The match is pure and total, so it is now `adlx_outcome`, tested over every arm — including the one that must stay silent, since an absent driver is an expected state and not a fault to warn about. Mutation-checked by making the fault arm silent. Signed-off-by: Roman Sirokov --- crates/rocm-dash-collectors/src/winperf.rs | 170 ++++++++++++++++++--- 1 file changed, 152 insertions(+), 18 deletions(-) diff --git a/crates/rocm-dash-collectors/src/winperf.rs b/crates/rocm-dash-collectors/src/winperf.rs index 1a1a8fabf1..36d03aceb6 100644 --- a/crates/rocm-dash-collectors/src/winperf.rs +++ b/crates/rocm-dash-collectors/src/winperf.rs @@ -96,21 +96,7 @@ impl WinPerfCollector { // absent or fails to initialise answers Ok with no readings and no // warning, since "no AMD driver" is an expected state. let (adlx, adlx_warning) = - match run_single_flight(&ADLX_IN_FLIGHT, ADLX_TIMEOUT, adlx_readings).await { - SingleFlight::Done(Ok(readings)) => (readings, None), - SingleFlight::Done(Err(gone)) => (Vec::new(), Some(gone.warning())), - SingleFlight::Panicked => { - (Vec::new(), Some("ADLX reading task panicked".to_string())) - } - SingleFlight::TimedOut => ( - Vec::new(), - Some(format!("ADLX query timed out after {ADLX_TIMEOUT:?}")), - ), - SingleFlight::StillRunning => ( - Vec::new(), - Some("ADLX query skipped: a previous call is still running".to_string()), - ), - }; + adlx_outcome(run_single_flight(&ADLX_IN_FLIGHT, ADLX_TIMEOUT, adlx_readings).await); let metrics = parse_metrics(&out, &adlx); let warnings = counter_samples_missing_warning(&out, &metrics) .into_iter() @@ -450,6 +436,20 @@ enum ReaderGone { /// tick; reported so a driver that panics every read is not silent. #[cfg_attr(not(any(windows, test)), allow(dead_code))] ReadPanicked, + /// The init step panicked. Caught for the same reason as a read panic, and + /// kept apart from it because they are not the same fault: a read panic is + /// per-tick and the next read may succeed, while this one is terminal — + /// the state was never built and this design never rebuilds it. + /// + /// Before it was caught, an init panic unwound the reader thread outright, + /// so every later tick met a closed channel and reported + /// [`Self::ThreadGone`] — indistinguishable from the thread never having + /// started. Catching it keeps the thread alive to answer and names the real + /// cause. It does **not** restore the pre-resident-reader behaviour of + /// retrying init on the next tick; that is the deliberate trade this + /// design makes, and the reason the distinct variant is worth having. + #[cfg_attr(not(any(windows, test)), allow(dead_code))] + InitPanicked, /// The thread never started, or exited before answering. #[cfg_attr(not(any(windows, test)), allow(dead_code))] ThreadGone, @@ -457,10 +457,42 @@ enum ReaderGone { type ReaderAnswer = Result, ReaderGone>; +/// What one ADLX attempt contributes to a snapshot: the readings to use, and +/// the warning to show the user, if any. +/// +/// Split out of [`WinPerfCollector::snapshot`] purely so it can be tested. +/// `snapshot` itself calls `Self::run_script()` unconditionally, which shells +/// out to PowerShell, so no test can drive it — the same impracticality +/// `adlx_readings`'s own FFI chain is documented with. That left the mapping +/// below, including the `Done(Err(_))` arm this change introduced, as the one +/// piece of new wiring nothing exercised. It is pure and total, so testing it +/// here costs nothing and covers every arm. +/// +/// The asymmetry worth keeping: a driver that is absent or failed to +/// initialise answers `Ok` with no readings and **no** warning, because "no AMD +/// driver" is an expected state rather than a fault. Everything else is a +/// fault, and says so. +fn adlx_outcome(outcome: SingleFlight) -> (Vec, Option) { + match outcome { + SingleFlight::Done(Ok(readings)) => (readings, None), + SingleFlight::Done(Err(gone)) => (Vec::new(), Some(gone.warning())), + SingleFlight::Panicked => (Vec::new(), Some("ADLX reading task panicked".to_string())), + SingleFlight::TimedOut => ( + Vec::new(), + Some(format!("ADLX query timed out after {ADLX_TIMEOUT:?}")), + ), + SingleFlight::StillRunning => ( + Vec::new(), + Some("ADLX query skipped: a previous call is still running".to_string()), + ), + } +} + impl ReaderGone { fn warning(self) -> String { match self { Self::ReadPanicked => "ADLX read panicked".to_string(), + Self::InitPanicked => "ADLX initialisation panicked".to_string(), Self::ThreadGone => "the ADLX reader thread is gone".to_string(), } } @@ -479,18 +511,28 @@ impl ResidentReader { // `state` is deliberately held until the request channel closes // (never, for the process-wide reader) so it is not torn down // and re-initialised between ticks. - let state = init(); + // + // Init is caught for the same reason the read below is, and the + // asymmetry mattered more here: a panic escaping this line + // unwound the thread before the request loop started, so + // `inbox` dropped and every later tick reported `ThreadGone` — + // the one state that cannot be told apart from "the thread + // never started". Catching keeps the reader answering, with a + // variant that names what actually happened. + let state = std::panic::catch_unwind(std::panic::AssertUnwindSafe(init)) + .map_err(|_| ReaderGone::InitPanicked); for reply in inbox { // A panicking read would otherwise unwind this thread and // leave every later tick answering from a closed channel, // with no way to tell that from an absent driver. Catching // it keeps the one reader alive and names the failure. let answer = match state.as_ref() { - Some(state) => { + Ok(Some(state)) => { std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| read(state))) .map_err(|_| ReaderGone::ReadPanicked) } - None => Ok(Vec::new()), + Ok(None) => Ok(Vec::new()), + Err(gone) => Err(*gone), }; let _ = reply.send(answer); } @@ -1053,6 +1095,98 @@ mod tests { assert_eq!(inits.load(Ordering::SeqCst), 1); } + /// The asymmetry raised in review on #1020: `read` was wrapped in + /// `catch_unwind` and `init` was not. + /// + /// A panic escaping `init` unwound the reader thread before the request + /// loop began, so `inbox` dropped and every later tick answered + /// `ThreadGone` — the one verdict indistinguishable from "the thread never + /// started", for a cause that is neither. Worse, the reader is built once + /// per process, so that verdict was permanent. + /// + /// Both halves are asserted, and the second is what makes this + /// discriminating: the answer must name the init panic, **and** the reader + /// must still be answering on later calls rather than having died. Before + /// the fix the first assertion fails (`ThreadGone`, not `InitPanicked`). + #[test] + fn a_panicking_init_is_named_and_leaves_the_reader_answering() { + let reader = ResidentReader::spawn( + || -> Option<()> { panic!("ADLX init blew up") }, + |()| vec![reading(1.0)], + ); + + for _ in 0..5 { + // `matches!`, not `assert_eq!`: `AdlxReading` carries no `Debug` + // or `PartialEq`, and adding them to a production struct to satisfy + // a test would be the tail wagging the dog. + assert!( + matches!(reader.read(), Err(ReaderGone::InitPanicked)), + "an init panic must be named, not reported as a thread that never started" + ); + } + assert_eq!( + ReaderGone::InitPanicked.warning(), + "ADLX initialisation panicked" + ); + } + + /// The `snapshot` mapping, which nothing reached: `snapshot` shells out to + /// PowerShell unconditionally, so the `Done(Err(_))` arm this change added + /// was wired up and never executed by a test. See `adlx_outcome`. + /// + /// Every arm is covered, including the one that must stay silent: an absent + /// driver answers `Ok` with no readings and raises no warning, because a + /// machine with no AMD GPU is not a fault to report. + #[test] + fn the_adlx_outcome_mapping_warns_for_faults_and_stays_silent_for_an_absent_driver() { + let (readings, warning) = adlx_outcome(SingleFlight::Done(Ok(vec![reading(42.0)]))); + assert_eq!(readings.len(), 1); + assert!(warning.is_none(), "a good read raises nothing"); + + let (readings, warning) = adlx_outcome(SingleFlight::Done(Ok(Vec::new()))); + assert!(readings.is_empty()); + assert!( + warning.is_none(), + "an absent driver is an expected state, not a warning" + ); + + for gone in [ + ReaderGone::ReadPanicked, + ReaderGone::InitPanicked, + ReaderGone::ThreadGone, + ] { + let (readings, warning) = adlx_outcome(SingleFlight::Done(Err(gone))); + assert!(readings.is_empty()); + assert_eq!( + warning.as_deref(), + Some(gone.warning().as_str()), + "a reader fault must surface its own wording: {gone:?}" + ); + } + + for (outcome, expected) in [ + (SingleFlight::Panicked, "ADLX reading task panicked"), + ( + SingleFlight::StillRunning, + "a previous call is still running", + ), + ] { + let (readings, warning) = adlx_outcome(outcome); + assert!(readings.is_empty()); + assert!( + warning.as_deref().is_some_and(|w| w.contains(expected)), + "expected {expected:?}, got {warning:?}" + ); + } + + let (readings, warning) = adlx_outcome(SingleFlight::TimedOut); + assert!(readings.is_empty()); + assert!( + warning.as_deref().is_some_and(|w| w.contains("timed out")), + "{warning:?}" + ); + } + #[test] fn a_panicking_read_is_reported_and_leaves_the_reader_usable() { use std::sync::Arc;