From 5eb84a01acc840c930159805ad9f5cfa1fa3c534 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 14 Oct 2026 14:02:18 +0300 Subject: [PATCH 01/59] feat(examine,serve): report real GPU/backend support instead of a static ROCm claim ROCMAI-550: on a GPU with no ROCm/HIP build (e.g. gfx90c), `rocm examine` and `rocm engines` kept claiming the embedded Lemonade engine runs "ROCm llama.cpp backend" even though it actually falls back to Vulkan. `rocm install sdk`'s refusal for such a GPU didn't name the actual next step, and neither `rocm serve`'s deployment summary nor `rocm services list` showed which backend/ device an engine actually selected. - add `GpuSupportLevel` (RocmHip/VulkanOnly/Unknown) in rocm-core, derived from the existing `normalize_therock_family`/`gfx_target_has_no_rocm_build` primitives - make the Lemonade engine note in `rocm examine` and `rocm engines` GPU-aware instead of a hardcoded ROCm claim - `rocm install sdk`'s unsupported-GPU errors now suggest `rocm serve ` - surface the engine's actual backend/device (read live via the existing `Healthcheck` RPC, never guessed) in `rocm serve`'s deployment summary and `rocm services list` Most of the ticket's `rocm examine`/`install sdk` complaints for gfx90c were already fixed by #648 (ROCMAI-542); this covers what was left. Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 189 +++++++++++++++++- apps/rocm/src/providers.rs | 1 + apps/rocm/src/serve_cmd.rs | 14 +- apps/rocm/src/serve_summary.rs | 28 +++ apps/rocm/src/therock.rs | 8 +- crates/rocm-core/src/host_gpu.rs | 59 ++++++ crates/rocm-core/src/lib.rs | 4 +- .../features/model_serving.feature | 2 + tests/e2e-cucumber/tests/e2e/serving_steps.rs | 21 ++ 9 files changed, 311 insertions(+), 15 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 297f8154e8..218a7a54fa 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -109,7 +109,8 @@ use rocm_core::{ use rocm_engine_protocol::{ DEFAULT_LOG_TAIL_LINES, DetectRequest, DetectResponse, DevicePolicy, ENGINE_RECIPE_CONTRACT_VERSION, EngineMethod, EnginePluginDescriptor, EngineRecipeHint, - EngineRequestEnvelope, EngineResponseEnvelope, GpuSelection, StopRequest, StopResponse, + EngineRequestEnvelope, EngineResponseEnvelope, GpuSelection, HealthcheckRequest, + HealthcheckResponse, StopRequest, StopResponse, }; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; @@ -2740,6 +2741,21 @@ fn builtin_codex_bridge_engine_inventory() -> Vec { .collect() } +/// The lemonade engine note shown in `rocm examine` and `rocm engines`, adjusted +/// for whether this host's detected GPU can actually use the ROCm llama.cpp +/// backend. Both call sites have a live `GpuSupportLevel` in scope; other +/// places that print this note statically (the dashboard TUI, `rocmd`'s own +/// catalog) have no host probe in front of them and keep the unconditional +/// ROCm wording, since they describe the engine in the abstract. +const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str { + match level { + rocm_core::GpuSupportLevel::VulkanOnly => { + "default embedded Lemonade server; this GPU has no ROCm/HIP support, so it runs on the Vulkan llama.cpp backend" + } + _ => "default embedded Lemonade server with ROCm llama.cpp backend", + } +} + const fn builtin_engine_inventory() -> &'static [(&'static str, &'static str)] { &[ ( @@ -9066,7 +9082,17 @@ fn examine_human_report( let mut output = render_examine_plain_header(&summary); output.push_str(&summary.render_text()); append_examine_runtime_state(&mut output, paths, config)?; - append_examine_engine_inventory(&mut output, paths, config, &summary.default_engine); + let gpu_support_level = rocm_core::gpu_support_level( + summary.detected_gfx_target.as_deref(), + summary.compatible_therock_family.as_deref(), + ); + append_examine_engine_inventory( + &mut output, + paths, + config, + &summary.default_engine, + gpu_support_level, + ); Ok((output, summary)) } @@ -9124,6 +9150,10 @@ fn render_engine_inventory_text_with_paths(paths: Option<&AppPaths>) -> String { // constant put the `*` on Lemonade even on Instinct, where `serve` picks vLLM. let host_gpu = rocm_core::detect_host_gpu_summary(paths); let host_default_engine = rocm_core::default_engine_for_host(&host_gpu); + let gpu_support_level = rocm_core::gpu_support_level( + host_gpu.gfx_target.as_deref(), + host_gpu.therock_family.as_deref(), + ); // A configured `default_engine` still wins over the host preference, mirroring // `select_serve_engine` (and `append_examine_engine_inventory`). Without this, // the legend could mark an engine `serve` will not actually default to. @@ -9170,6 +9200,11 @@ fn render_engine_inventory_text_with_paths(paths: Option<&AppPaths>) -> String { } else { " " }; + let note = if *name == "lemonade" { + lemonade_engine_note(gpu_support_level) + } else { + *note + }; let _ = writeln!(output, "{marker} {name:10} {note}"); append_engine_detect_summary(&mut output, name, paths); } @@ -9294,6 +9329,7 @@ fn append_examine_engine_inventory( paths: &AppPaths, config: &RocmCliConfig, host_default_engine: &str, + gpu_support_level: rocm_core::GpuSupportLevel, ) { // Mirrors `select_serve_engine`'s guard: a blank configured value must not // out-rank the host preference, or `effective_default` below would become @@ -9380,6 +9416,11 @@ fn append_examine_engine_inventory( .or(entry.last_installed_env_id.as_deref()) }) .unwrap_or(""); + let note = if *engine == "lemonade" { + lemonade_engine_note(gpu_support_level) + } else { + *note + }; let _ = writeln!( output, " {marker} {engine} adapter={adapter} runtime_pref={runtime_pref} note={note}" @@ -10708,6 +10749,16 @@ pub(crate) fn render_services_text(paths: &AppPaths, all: bool) -> Result Option { + let response: HealthcheckResponse = engine_request( + Some(paths), + engine, + EngineMethod::Healthcheck, + &HealthcheckRequest { + service_id: service_id.to_owned(), + }, + ) + .ok()?; + // `"unknown"` is the engines' own device value when no state is on disk for + // this service id yet (see `engines/vllm/src/state.rs` and + // `engines/lemonade/src/lib.rs`), which is not more informative than + // showing nothing. + (!response.device.is_empty() && response.device != "unknown").then_some(response.device) +} + fn decode_engine_response(envelope: EngineResponseEnvelope) -> Result where R: DeserializeOwned, @@ -21605,6 +21681,10 @@ install therock"; ); assert!(rendered.contains("- svc-ready")); assert!(rendered.contains(" stop: rocm services stop svc-ready --yes")); + // No real engine state file backs this record, so the engine's own + // Healthcheck reports `device: "unknown"` — not more informative than + // omitting the line, so it must not appear. + assert!(!rendered.contains("device:"), "{rendered}"); assert!(!rendered.contains("- svc-failed")); assert!(all.contains("- svc-failed")); assert!(all.contains(" restart: rocm services restart svc-failed --yes")); @@ -25829,7 +25909,13 @@ install therock"; }; let mut output = String::new(); - append_examine_engine_inventory(&mut output, &paths, &config, "vllm"); + append_examine_engine_inventory( + &mut output, + &paths, + &config, + "vllm", + rocm_core::GpuSupportLevel::Unknown, + ); assert!( output.contains("configured_default_engine: "), @@ -26559,7 +26645,13 @@ install therock"; // Host default deliberately disagrees with the configured value: an // engine the user chose must outrank whatever the GPU would prefer. - append_examine_engine_inventory(&mut output, &paths, &config, "lemonade"); + append_examine_engine_inventory( + &mut output, + &paths, + &config, + "lemonade", + rocm_core::GpuSupportLevel::Unknown, + ); assert!(output.contains("engine_inventory:")); assert!(output.contains("configured_default_engine: vllm")); @@ -26576,7 +26668,13 @@ install therock"; let config = RocmCliConfig::default(); let mut output = String::new(); - append_examine_engine_inventory(&mut output, &paths, &config, "vllm"); + append_examine_engine_inventory( + &mut output, + &paths, + &config, + "vllm", + rocm_core::GpuSupportLevel::Unknown, + ); let _ = fs::remove_dir_all(root); let legend = format!("legend: {DEFAULT_ENGINE_MARKER} = default engine"); @@ -26602,7 +26700,13 @@ install therock"; }; let mut output = String::new(); - append_examine_engine_inventory(&mut output, &paths, &config, "vllm"); + append_examine_engine_inventory( + &mut output, + &paths, + &config, + "vllm", + rocm_core::GpuSupportLevel::Unknown, + ); let _ = fs::remove_dir_all(root); assert!( @@ -26622,7 +26726,13 @@ install therock"; let config = RocmCliConfig::default(); let mut output = String::new(); - append_examine_engine_inventory(&mut output, &paths, &config, "vllm"); + append_examine_engine_inventory( + &mut output, + &paths, + &config, + "vllm", + rocm_core::GpuSupportLevel::Unknown, + ); // The configured line still reads as unset — the host default must not be // mistaken for something the user chose. @@ -26654,7 +26764,13 @@ install therock"; }; let mut output = String::new(); - append_examine_engine_inventory(&mut output, &paths, &config, "vllm"); + append_examine_engine_inventory( + &mut output, + &paths, + &config, + "vllm", + rocm_core::GpuSupportLevel::Unknown, + ); assert!( output.contains("effective_default_engine: lemonade"), @@ -26679,6 +26795,55 @@ install therock"; let _ = fs::remove_dir_all(root); } + #[test] + fn examine_engine_inventory_names_vulkan_for_a_gpu_with_no_rocm_support() { + // A gfx90c host (or any confirmed-no-ROCm-build GPU) actually runs + // Lemonade's Vulkan backend, not the ROCm one the static catalog text + // always claimed. + let (root, paths) = test_paths("examine-engine-inventory-vulkan-only"); + let config = RocmCliConfig::default(); + let mut output = String::new(); + + append_examine_engine_inventory( + &mut output, + &paths, + &config, + "lemonade", + rocm_core::GpuSupportLevel::VulkanOnly, + ); + + assert!( + output.contains("no ROCm/HIP support") && output.contains("Vulkan llama.cpp backend"), + "{output}" + ); + assert!( + !output.contains("note=default embedded Lemonade server with ROCm"), + "{output}" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn examine_engine_inventory_keeps_rocm_note_for_a_rocm_capable_gpu() { + let (root, paths) = test_paths("examine-engine-inventory-rocm-capable"); + let config = RocmCliConfig::default(); + let mut output = String::new(); + + append_examine_engine_inventory( + &mut output, + &paths, + &config, + "lemonade", + rocm_core::GpuSupportLevel::RocmHip, + ); + + assert!( + output.contains("note=default embedded Lemonade server with ROCm llama.cpp backend"), + "{output}" + ); + let _ = fs::remove_dir_all(root); + } + #[test] fn examine_is_silent_when_the_configured_engine_agrees_with_the_host() { let (root, paths) = test_paths("examine-engine-inventory-agrees"); @@ -26688,7 +26853,13 @@ install therock"; }; let mut output = String::new(); - append_examine_engine_inventory(&mut output, &paths, &config, "vllm"); + append_examine_engine_inventory( + &mut output, + &paths, + &config, + "vllm", + rocm_core::GpuSupportLevel::Unknown, + ); assert!( !output.contains("configured_default_note"), diff --git a/apps/rocm/src/providers.rs b/apps/rocm/src/providers.rs index 968cf69f9b..ecbdc5669c 100644 --- a/apps/rocm/src/providers.rs +++ b/apps/rocm/src/providers.rs @@ -2915,6 +2915,7 @@ mod tests { chat_endpoint: "http://127.0.0.1:8000/v1/chat/completions".to_owned(), service_id: "svc-ready".to_owned(), status: "ready".to_owned(), + device: None, already_running: false, metrics, api_key: None, diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 275115f2ac..5b1f38d8bd 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -46,8 +46,8 @@ use crate::service_launch::{ }; use crate::{ cli_progress, collect_serve_notes, detect_gpu_count, device_policy_name, engine_request, - gpu_vram_usage, parse_device_policy, parse_gpu_selection, resolve_engine_selection, - select_gpu_indices_under_launch_lock, serve_gpu_low_memory_warning, + fetch_service_device, gpu_vram_usage, parse_device_policy, parse_gpu_selection, + resolve_engine_selection, select_gpu_indices_under_launch_lock, serve_gpu_low_memory_warning, validate_engine_selection_runtime, validate_pinned_gpu_index, }; @@ -910,6 +910,15 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { }; spinner.clear(); + // Read what the engine actually selected rather than guessing from + // the pre-launch GPU plan: `gpu_indices` above can be empty for a + // Vulkan-only GPU that ROCm/amd-smi never enumerates, even though + // the server is running fine on it. Only meaningful once the + // server is ready to answer, same gate as the smoke test. + let device = (report.status == "ready") + .then(|| fetch_service_device(&paths, &selected_engine, &report.service_id)) + .flatten(); + let notes = collect_serve_notes( cpu_only, &gpu_selection, @@ -927,6 +936,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { chat_endpoint: format!("{}/chat/completions", report.endpoint_url), service_id: report.service_id.clone(), status: report.status.clone(), + device, already_running: report.already_running, metrics, api_key: launched_key, diff --git a/apps/rocm/src/serve_summary.rs b/apps/rocm/src/serve_summary.rs index 5fea572403..45a17cc898 100644 --- a/apps/rocm/src/serve_summary.rs +++ b/apps/rocm/src/serve_summary.rs @@ -55,6 +55,10 @@ pub(crate) struct DeploymentSummary { pub service_id: String, /// `"ready"`, `"starting"`, or an existing service's status. pub status: String, + /// The backend/device the engine actually selected (e.g. `"vulkan gpu 0"`, + /// `"rocm gpu 0"`), read from the engine's own `Healthcheck` response. + /// `None` when the query failed or the service was not ready to ask. + pub device: Option, /// True when an equivalent server was already running and nothing was spawned. pub already_running: bool, pub metrics: SmokeMetrics, @@ -121,11 +125,17 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { ] }; + let device_row = summary + .device + .as_deref() + .map(|device| ("device", device.to_owned())); + let rows: Vec<(&str, String)> = [ vec![ ("status", summary.status.clone()), ("engine", summary.engine.clone()), ], + device_row.into_iter().collect(), model_rows, vec![ ("endpoint", summary.chat_endpoint.clone()), @@ -265,6 +275,7 @@ mod tests { chat_endpoint: "http://127.0.0.1:1337/v1/chat/completions".to_owned(), service_id: "vllm-qwen-1720000000".to_owned(), status: "ready".to_owned(), + device: None, already_running: false, metrics: SmokeMetrics { ttft: Some(Duration::from_millis(180)), @@ -435,6 +446,23 @@ mod tests { assert!(render_summary(&summary).contains("note: selected GPU 0 has low free VRAM")); } + #[test] + fn device_row_shown_only_when_known() { + let without_device = render_summary(&base_summary()); + assert!(!without_device.contains("device"), "{without_device}"); + + let mut summary = base_summary(); + summary.device = Some("vulkan gpu 0".to_owned()); + let rendered = render_summary(&summary); + assert!( + rendered + .lines() + .any(|line| line.trim_start().starts_with("device") + && line.trim_end().ends_with("vulkan gpu 0")), + "{rendered}" + ); + } + #[test] fn api_key_client_config_shown_only_when_present() { // Loopback / no key: the summary must not mention an api key at all. diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index 9a692c32cb..0efd9fbc3d 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -3459,7 +3459,8 @@ fn unsupported_host_gpu_message(raw_arch: &str) -> String { "no ROCm SDK build exists for the detected GPU target `{raw_arch}`, so \ `rocm install sdk` has nothing to install for it.\n\ The GPU is detected but is not supported by ROCm; \ - the Vulkan backend can still be used." + the Vulkan backend can still be used. \ + Try `rocm serve ` — it runs on Lemonade's Vulkan backend instead." ) } @@ -3475,7 +3476,8 @@ fn unsupported_requested_target_message(value: &str, source: &str) -> String { "no ROCm SDK build exists for the GPU target `{value}` requested by \ {source}, so `rocm install sdk` has nothing to install for it.\n\ That GPU is not supported by ROCm; the Vulkan backend can still be \ - used. Nothing was installed and no other family was substituted." + used. Nothing was installed and no other family was substituted. \ + Try `rocm serve ` — it runs on Lemonade's Vulkan backend instead." ) } @@ -7665,6 +7667,7 @@ mod tests { assert!(message.contains("no ROCm SDK build"), "{message}"); assert!(message.contains("`gfx90c`"), "{message}"); assert!(message.contains("Vulkan"), "{message}"); + assert!(message.contains("rocm serve "), "{message}"); assert!(!message.contains("--family"), "{message}"); for family in known_therock_families() { assert!( @@ -7746,6 +7749,7 @@ mod tests { let message = unsupported_requested_target_message("gfx90c", "--family"); assert!(message.contains("`gfx90c`"), "{message}"); assert!(message.contains("--family"), "{message}"); + assert!(message.contains("rocm serve "), "{message}"); // The user named it; nothing was detected, so the error must not say so. assert!(!message.contains("detected"), "{message}"); diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index c985ed5358..5256e09188 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -128,6 +128,40 @@ pub fn gfx_target_has_no_rocm_build(gfx: &str) -> bool { && normalize_therock_family(gfx).is_none() } +/// How well this host's GPU is supported, for messages that tell the user what +/// they can actually do rather than just reporting raw detection fields. +/// +/// `Unknown` deliberately covers both "nothing detected" and "a gfx target we +/// don't recognize" rather than being called `Unsupported` — see the doc +/// comment on [`gfx_target_has_no_rocm_build`] and +/// `GFX_TARGETS_WITHOUT_ROCM_SUPPORT` in `diagnose.rs` for why conflating +/// "confirmed no ROCm build" with "unrecognized" would turn a detection gap +/// into a false claim about the hardware. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuSupportLevel { + /// A TheRock package family covers this GPU: full ROCm/HIP support. + RocmHip, + /// Recognized, confirmed to have no ROCm/HIP build, but Vulkan-based + /// engines (the default Lemonade backend) still work on it. + VulkanOnly, + /// No GPU detected yet, or a gfx target this catalog doesn't recognize. + Unknown, +} + +/// Classify the host's GPU support level from the same two facts +/// `ExamineSummary` and `resolve_family` already compute: the detected gfx +/// target and whichever TheRock family (if any) covers it. +pub fn gpu_support_level( + detected_gfx_target: Option<&str>, + compatible_therock_family: Option<&str>, +) -> GpuSupportLevel { + match (detected_gfx_target, compatible_therock_family) { + (_, Some(_)) => GpuSupportLevel::RocmHip, + (Some(gfx), None) if gfx_target_has_no_rocm_build(gfx) => GpuSupportLevel::VulkanOnly, + _ => GpuSupportLevel::Unknown, + } +} + pub fn normalize_therock_family(value: &str) -> Option { let normalized = value.trim().to_ascii_lowercase(); if normalized.is_empty() { @@ -3457,6 +3491,31 @@ mod tests { assert!(!gfx_target_has_no_rocm_build("")); } + #[test] + fn gpu_support_level_is_rocm_hip_when_a_family_covers_it() { + assert_eq!( + gpu_support_level(Some("gfx90a"), Some("gfx90a")), + GpuSupportLevel::RocmHip + ); + } + + #[test] + fn gpu_support_level_is_vulkan_only_for_gfx90c() { + assert_eq!( + gpu_support_level(Some("gfx90c"), None), + GpuSupportLevel::VulkanOnly + ); + } + + #[test] + fn gpu_support_level_is_unknown_without_detection_or_for_an_unrecognized_target() { + assert_eq!(gpu_support_level(None, None), GpuSupportLevel::Unknown); + assert_eq!( + gpu_support_level(Some("gfx_totally_unrecognized"), None), + GpuSupportLevel::Unknown + ); + } + /// The predicate must agree with `normalize_therock_family`, not merely /// happen to. /// diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index a23896c3fe..1004a33767 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -65,11 +65,11 @@ pub use fix::{ }; pub use hardware_root::host_path; pub use host_gpu::{ - DriverSummary, ExamineSummary, HostGpuSummary, WslHostDriverProbe, WslSummary, + DriverSummary, ExamineSummary, GpuSupportLevel, HostGpuSummary, WslHostDriverProbe, WslSummary, default_engine_for_host, default_engine_for_platform, detect_gpu_driver_version, detect_host_gfx_target, detect_host_gpu_diagnostics, detect_host_gpu_summary, detect_system_ram_gib, extract_first_gfx_token, gfx_target_has_no_rocm_build, - has_usable_amd_gpu, interactive_terminal, is_amdgpu_device, is_wsl_host, + gpu_support_level, has_usable_amd_gpu, interactive_terminal, is_amdgpu_device, is_wsl_host, known_therock_families, normalize_therock_family, preferred_serve_engine_for_host_gpu_summary, usable_amd_gpu_indices, }; diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 93c3c874e3..a67149b05c 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -43,6 +43,7 @@ Feature: Model serving When the user sends a chat completion request Then the response contains a model reply And the response identifies the correct model + And the services list reports the engine's actual backend device # Large-model coverage (dogfooding W9): serve a representative large model for # each GPU platform end-to-end at least once. MI300X uses Qwen/Qwen3.6-27B through @@ -68,6 +69,7 @@ Feature: Model serving When the user sends a chat completion request Then the response contains a model reply And the response identifies the correct model + And the services list reports the engine's actual backend device # HF-checkpoint direct-serve canary (EAI-8026). An `owner/repo:variant` ref # bypasses Lemonade's model router and runs a packaged llama-server directly diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 4a382eb833..6840d5e61f 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1346,6 +1346,27 @@ async fn assert_service_in_list(world: &mut E2eWorld) { ); } +/// ROCMAI-550: `rocm services list` must show the backend/device the engine +/// actually selected (read live from its `Healthcheck` RPC), not just that a +/// service exists. Reuses `assert_service_in_list`'s model-name match so this +/// only checks the row for the service this scenario just started. +#[then("the services list reports the engine's actual backend device")] +async fn assert_services_list_reports_device(world: &mut E2eWorld) { + let (stdout, _, _) = crate::run_rocm(world, &["services", "list"]); + let model = world.model_name.as_deref().unwrap_or(""); + let block = stdout + .split("\n- ") + .find(|block| block.to_lowercase().contains(&model.to_lowercase())) + .unwrap_or_else(|| panic!("no services-list entry for model {model}:\n{stdout}")); + assert!( + block + .lines() + .any(|line| line.trim_start().starts_with("device:")), + "services list entry for {model} is missing a `device:` line — the engine's \ + Healthcheck RPC should report the backend it actually selected:\n{block}" + ); +} + #[then("the list reports the attempt and how to look at it")] async fn assert_past_attempts_reported(world: &mut E2eWorld) { let stdout = world From 65a24f8a19f8f8de4a47170ec3a8c00b6dc38e75 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 14 Oct 2026 15:30:01 +0300 Subject: [PATCH 02/59] test(serve): keep the api key out of summary assertion messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeQL flagged two assertions as cleartext logging of `api_key`. Those two use `base_summary()`, whose key is `None`, so the literal instances are false positives — but the pattern is not. `render_summary` prints the key by design, and `api_key_client_config_shown_only_when_present` sets a real one, so a failure there copied `endpoint-secret` into CI logs, which are retained and widely readable. Assert on the real text, print a redacted copy. Covers all twelve sites that interpolated a rendered summary, not just the two CodeQL picked. Production output is unchanged: the key is still shown once, and the test pinning that still asserts the exact string. Signed-off-by: Roman Sirokov --- apps/rocm/src/serve_summary.rs | 65 +++++++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 12 deletions(-) diff --git a/apps/rocm/src/serve_summary.rs b/apps/rocm/src/serve_summary.rs index 45a17cc898..b153db4753 100644 --- a/apps/rocm/src/serve_summary.rs +++ b/apps/rocm/src/serve_summary.rs @@ -267,6 +267,22 @@ fn compute_metrics( mod tests { use super::*; + /// A rendered summary with the api key masked, for panic messages. + /// + /// `render_summary` prints the key by design, and a failing assertion that + /// interpolated the whole summary would copy it into CI logs, which are + /// retained and widely readable. Assert on the real text; print this. + fn redacted(rendered: &str) -> String { + rendered + .lines() + .map(|line| match line.split_once("api key: ") { + Some((prefix, _)) => format!("{prefix}api key: "), + None => line.to_owned(), + }) + .collect::>() + .join("\n") + } + fn base_summary() -> DeploymentSummary { DeploymentSummary { engine: "vllm".to_owned(), @@ -378,17 +394,19 @@ mod tests { line.trim_start().starts_with(label) && line.trim_end().ends_with(value) }) }; - assert!(row("requested model", "qwen"), "{rendered}"); + assert!(row("requested model", "qwen"), "{}", redacted(&rendered)); assert!( row("resolved model", "Qwen3-4B-Instruct-2507-GGUF"), - "{rendered}" + "{}", + redacted(&rendered) ); // The bare `model` row is not used when the two differ. assert!( !rendered .lines() .any(|line| line.trim_start().starts_with("model ")), - "{rendered}" + "{}", + redacted(&rendered) ); } @@ -407,7 +425,11 @@ mod tests { let mut summary = base_summary(); summary.status = "starting".to_owned(); let rendered = render_summary(&summary); - assert!(rendered.contains("not ready yet"), "heading: {rendered}"); + assert!( + rendered.contains("not ready yet"), + "heading: {}", + redacted(&rendered) + ); assert!(rendered.contains("did not report ready")); assert!(rendered.contains("rocm logs --service")); } @@ -422,14 +444,20 @@ mod tests { summary.status = "running".to_owned(); summary.metrics = SmokeMetrics::default(); let rendered = render_summary(&summary); - assert!(rendered.contains("not ready yet"), "heading: {rendered}"); + assert!( + rendered.contains("not ready yet"), + "heading: {}", + redacted(&rendered) + ); assert!( rendered.contains("no smoke test was run"), - "the skipped smoke test must be explained: {rendered}" + "the skipped smoke test must be explained: {}", + redacted(&rendered) ); assert!( rendered.contains("still loading"), - "the user needs to know this resolves on its own: {rendered}" + "the user needs to know this resolves on its own: {}", + redacted(&rendered) ); assert!(rendered.contains("rocm services list")); } @@ -449,7 +477,11 @@ mod tests { #[test] fn device_row_shown_only_when_known() { let without_device = render_summary(&base_summary()); - assert!(!without_device.contains("device"), "{without_device}"); + assert!( + !without_device.contains("device"), + "{}", + redacted(&without_device) + ); let mut summary = base_summary(); summary.device = Some("vulkan gpu 0".to_owned()); @@ -459,7 +491,8 @@ mod tests { .lines() .any(|line| line.trim_start().starts_with("device") && line.trim_end().ends_with("vulkan gpu 0")), - "{rendered}" + "{}", + redacted(&rendered) ); } @@ -467,14 +500,22 @@ mod tests { fn api_key_client_config_shown_only_when_present() { // Loopback / no key: the summary must not mention an api key at all. let plain = render_summary(&base_summary()); - assert!(!plain.contains("api key"), "{plain}"); + assert!(!plain.contains("api key"), "{}", redacted(&plain)); // Public bind: the key is shown once as secure client config. let mut summary = base_summary(); summary.api_key = Some("endpoint-secret".to_owned()); let rendered = render_summary(&summary); - assert!(rendered.contains("api key: endpoint-secret"), "{rendered}"); - assert!(rendered.contains("shown only now"), "{rendered}"); + assert!( + rendered.contains("api key: endpoint-secret"), + "{}", + redacted(&rendered) + ); + assert!( + rendered.contains("shown only now"), + "{}", + redacted(&rendered) + ); } #[test] From 7162f9119422f21b796115241c353cd17862d3e9 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 14 Oct 2026 16:43:52 +0300 Subject: [PATCH 03/59] test(runtime): assert the Vulkan-serving remediation where the refusal is produced MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gfx90c refusals in `therock.rs` gained a `rocm serve ` remediation sentence, a second user-observable output change in this PR with no scenario behind it. AGENTS.md §3 wants remediation advice asserted where the message is really printed, not only against the formatter. runtime-19 already opens the one door of the two a lane can reach -- naming the target with `--family` -- so the assertion goes there. The detected-host door needs a Vega APU no runner has and stays unit-covered, as its existing comment already records. Also names the codex-bridge engine inventory at both the static table and the bridge helper as a deliberate scope boundary, instead of only in `lemonade_engine_note`'s doc comment where a reader of the table would miss it. Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 31 ++++++++++++++++--- .../features/runtime_setup.feature | 8 +++++ tests/e2e-cucumber/tests/e2e/runtime_steps.rs | 26 ++++++++++++++++ 3 files changed, 61 insertions(+), 4 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 218a7a54fa..63d0177db8 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -2724,6 +2724,19 @@ fn build_codex_bridge_gpu_snapshot(config: &RocmCliConfig) -> CodexBridgeGpuSnap } } +/// The engine catalogue in the codex-bridge snapshot, taken verbatim from +/// [`builtin_engine_inventory`] — including lemonade's unconditional "with ROCm +/// llama.cpp backend", which ROCMAI-550 replaced in `rocm examine` and +/// `rocm engines` but deliberately left alone here. +/// +/// It is left alone because this list is the static catalogue, in the same +/// shape as the unconditional `installed_binary: true` beside it: every field +/// but `default_for_platform` describes the engine, not this host. A consumer +/// that wants the host's real support level reads it off the `examine` block +/// in the same snapshot, which carries `detected_gfx_target` and +/// `compatible_therock_family` — the two inputs `rocm_core::gpu_support_level` +/// takes. Making this row host-aware would change the bridge's JSON contract +/// and is worth its own change, not a rider on this one. fn builtin_codex_bridge_engine_inventory() -> Vec { let default_engine = default_engine_for_platform(); let binary_path = daemon_binary_path() @@ -2743,10 +2756,11 @@ fn builtin_codex_bridge_engine_inventory() -> Vec { /// The lemonade engine note shown in `rocm examine` and `rocm engines`, adjusted /// for whether this host's detected GPU can actually use the ROCm llama.cpp -/// backend. Both call sites have a live `GpuSupportLevel` in scope; other -/// places that print this note statically (the dashboard TUI, `rocmd`'s own -/// catalog) have no host probe in front of them and keep the unconditional -/// ROCm wording, since they describe the engine in the abstract. +/// backend. Both call sites have a live `GpuSupportLevel` in scope; the places +/// that print this note statically — the dashboard TUI, `rocmd`'s own catalog, +/// and [`builtin_codex_bridge_engine_inventory`]'s copy of +/// [`builtin_engine_inventory`] — keep the unconditional ROCm wording, since +/// they describe the engine in the abstract rather than this host. const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str { match level { rocm_core::GpuSupportLevel::VulkanOnly => { @@ -2756,6 +2770,15 @@ const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str } } +/// The static engine catalogue: what each engine is, independent of any host. +/// +/// Lemonade's entry says "with ROCm llama.cpp backend" unconditionally, which +/// is only true of a host whose GPU has a ROCm build. Callers with a live +/// [`rocm_core::GpuSupportLevel`] must not print this row for lemonade — they +/// substitute [`lemonade_engine_note`], as `rocm examine` and `rocm engines` +/// do. This row is what the remaining host-blind consumers get; see +/// [`builtin_codex_bridge_engine_inventory`] for why that is the right answer +/// for them. const fn builtin_engine_inventory() -> &'static [(&'static str, &'static str)] { &[ ( diff --git a/tests/e2e-cucumber/features/runtime_setup.feature b/tests/e2e-cucumber/features/runtime_setup.feature index 0e2047c6fb..a2164d1f92 100644 --- a/tests/e2e-cucumber/features/runtime_setup.feature +++ b/tests/e2e-cucumber/features/runtime_setup.feature @@ -410,8 +410,16 @@ Feature: Runtime configuration # build" is absent however the run then ends. The second guards the weaker # failure where a refusal is printed but still dangles a family name in front # of the user. + # + # ROCMAI-550 added a third Then. Having taken every package family away, the + # refusal now has to leave the user somewhere to go, and it says `rocm serve + # ` on the Vulkan backend. That is a remediation line naming a command, + # so AGENTS.md §3 wants it asserted where the refusal is actually produced + # rather than only against the formatter -- this is the one door of the two + # that a lane can open, so it is asserted here. @id:runtime-install-sdk-refuses-unsupported-gpu-target Scenario: runtime-19 - Asking for a GPU target ROCm does not build is refused outright When the user dry-runs an SDK install for the gfx90c Vega iGPU Then the install is refused as having no ROCm build for that GPU And the refusal offers no package family to install instead + And the refusal points at serving on the Vulkan backend instead diff --git a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs index c7e6197d42..8c7378394d 100644 --- a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs @@ -1457,3 +1457,29 @@ async fn assert_unsupported_family_offers_nothing(world: &mut E2eWorld) { ); } } + +/// Having removed every `--family` the user could try, the refusal has to say +/// what *will* work. ROCMAI-550 added that sentence, and this pins it where the +/// refusal is really produced rather than against the formatter alone. +/// +/// Only the wording is checked here, because this lane cannot check the rest of +/// it: proving that `rocm serve` then reaches a GPU with no ROCm build needs a +/// Vega APU, and no runner has one (same gap as the detected-host door, above). +/// The two halves the advice rests on are proven elsewhere — +/// `rocm_core::gfx_target_has_no_rocm_build` is what put the user here, and +/// `selects_vulkan_when_rocm_unsupported` in `engines/lemonade`'s +/// `backend_alignment` is what makes llama.cpp pick `vulkan` once `rocm` comes +/// back unsupported, which is the claim the sentence makes. +#[then("the refusal points at serving on the Vulkan backend instead")] +async fn assert_unsupported_family_points_at_vulkan_serving(world: &mut E2eWorld) { + let output = unsupported_family_attempt(world); + assert!( + output.contains("rocm serve "), + "the refusal took every package family away, so it must name what does \ + work instead:\n{output}" + ); + assert!( + output.contains("Vulkan"), + "the refusal should say the fallback is the Vulkan backend:\n{output}" + ); +} From 16ccfb3a303c143d2be9bae2f295b509b44731eb Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 14 Oct 2026 20:31:35 +0300 Subject: [PATCH 04/59] fix(install): stop the requested-target refusal predicting a backend `unsupported_requested_target_message` is raised from the `--family` and `ROCM_CLI_THEROCK_FAMILY` branches at the top of `resolve_family`, before any host probe runs, so the rejected target says nothing about the GPU in this machine. It had borrowed its sibling's closing sentence promising that `rocm serve ` runs on Lemonade's Vulkan backend -- true in `unsupported_host_gpu_message`, which is only reached once host detection itself returns a target with no ROCm build, but false here: a stale env value naming gfx90c refuses just the same on a host whose own GPU is fully ROCm-capable, and there serving would run on ROCm/HIP. Point at `rocm examine` instead. It is true on every host, it is still a next step, and it is the command this PR just taught to report the real support level. runtime-19 was asserting the wrong half of this: it drives `--family gfx90c`, so it was pinning the Vulkan-serving claim green on exactly the ROCm-capable runners where it was false. It now asserts the examine pointer and the negative. Also route `ExamineSummary::rocm_support_line` through `gpu_support_level` rather than re-deriving the same classification, and document the serve summary's `device:` row in README and the manual-testing GPU check. Co-Authored-By: Claude Opus 4.7 Signed-off-by: Roman Sirokov --- README.md | 7 ++- apps/rocm/src/therock.rs | 55 ++++++++++++++++++- crates/rocm-core/src/host_gpu.rs | 25 ++++++--- docs/manual-testing.md | 8 ++- .../features/model_serving.feature | 8 +++ .../features/runtime_setup.feature | 14 +++-- tests/e2e-cucumber/tests/e2e/runtime_steps.rs | 36 ++++++------ 7 files changed, 115 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 861591ae81..6b0b92f7e2 100644 --- a/README.md +++ b/README.md @@ -656,8 +656,11 @@ services stop --yes`) or match the original flags. By default the server runs in the background under rocm-cli's supervision and prints a deployment summary — a progress indicator while it starts, then a table -with the status, the full inference endpoint, the API-qualified model name, and a -quick smoke test (time to first token and approximate tokens/sec). Control +with the status, the engine and the backend device it actually selected (for +example `rocm gpu 0` or `vulkan gpu 0`, read back from the running engine), the +full inference endpoint, the API-qualified model name, and a quick smoke test +(time to first token and approximate tokens/sec). `rocm services` reports the +same `device` for each running service. Control returns to your shell with the server still running; manage it later with `rocm services` (below). diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index 0efd9fbc3d..2395762b68 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -3471,13 +3471,24 @@ fn unsupported_host_gpu_message(raw_arch: &str) -> String { /// here: the user named the target, so the error names where it came from and /// does not claim a GPU is present. Like that message it offers no `--family` /// values, since none of them was built for this GPU. +/// +/// That same distinction is why this one must not borrow its sibling's closing +/// sentence. `unsupported_host_gpu_message` can promise that `rocm serve` runs +/// on Lemonade's Vulkan backend because it is only reached once host detection +/// has itself come back with a target that has no ROCm build. This message is +/// raised from the `--family` / `ROCM_CLI_THEROCK_FAMILY` branches at the top of +/// [`resolve_family`], before any host probe runs, so the rejected target says +/// nothing about the GPU in this machine: a stale env value naming gfx90c +/// refuses just the same on a host whose own GPU is fully ROCm-capable, and +/// there `rocm serve` would run on ROCm/HIP. So it points at `rocm examine`, +/// which reports this host's real support level, instead of asserting one. fn unsupported_requested_target_message(value: &str, source: &str) -> String { format!( "no ROCm SDK build exists for the GPU target `{value}` requested by \ {source}, so `rocm install sdk` has nothing to install for it.\n\ That GPU is not supported by ROCm; the Vulkan backend can still be \ used. Nothing was installed and no other family was substituted. \ - Try `rocm serve ` — it runs on Lemonade's Vulkan backend instead." + Run `rocm examine` to see what this host itself supports." ) } @@ -7749,7 +7760,7 @@ mod tests { let message = unsupported_requested_target_message("gfx90c", "--family"); assert!(message.contains("`gfx90c`"), "{message}"); assert!(message.contains("--family"), "{message}"); - assert!(message.contains("rocm serve "), "{message}"); + assert!(message.contains("rocm examine"), "{message}"); // The user named it; nothing was detected, so the error must not say so. assert!(!message.contains("detected"), "{message}"); @@ -7757,6 +7768,46 @@ mod tests { assert!(message.contains("ROCM_CLI_THEROCK_FAMILY"), "{message}"); } + /// The requested-target refusal must not promise which backend `rocm serve` + /// will use, because it never looked at this host. + /// + /// Both refusals used to close with the same "`rocm serve ` runs on + /// Lemonade's Vulkan backend" sentence. In `unsupported_host_gpu_message` + /// that is a fact: it is only reached after `detect_host_gfx_target` returns + /// a target `gfx_target_has_no_rocm_build` confirms. This one fires from the + /// `--family` / `ROCM_CLI_THEROCK_FAMILY` branches before any probe runs — + /// see `cli_family_overrides_a_stale_env_value_naming_an_unsupported_gpu` + /// for an override that deliberately disagrees with its host — so on a + /// ROCm-capable machine it told the user Vulkan when serving would really + /// run on ROCm/HIP. It now points at `rocm examine` instead, which is true + /// on every host and still leaves somewhere to go. + #[test] + fn requested_target_error_does_not_claim_which_backend_serving_will_use() { + for source in ["--family", "ROCM_CLI_THEROCK_FAMILY"] { + let message = unsupported_requested_target_message("gfx90c", source); + assert!( + !message.contains("rocm serve"), + "the requested target says nothing about this host's GPU, so the \ + refusal must not predict what `rocm serve` would run on: {message}" + ); + assert!( + !message.contains("Lemonade"), + "naming the engine makes the same unchecked promise about this \ + host as naming the command: {message}" + ); + assert!( + message.contains("rocm examine"), + "dropping the claim must still leave the user a next step: {message}" + ); + } + + // The sibling path did look at this host, so it keeps the claim: this is + // the asymmetry, not a blanket ban on the sentence. + let detected = unsupported_host_gpu_message("gfx90c"); + assert!(detected.contains("rocm serve "), "{detected}"); + assert!(detected.contains("Vulkan"), "{detected}"); + } + #[test] fn release_channel_prefers_stable_versions() { let versions = vec![ diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index 5256e09188..b96ef8681b 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -1821,16 +1821,25 @@ impl ExamineSummary { /// it (for example the gfx90c Vega iGPU in Renoir / Cezanne APUs). Without /// it `compatible_therock_family: ` reads like a probe failure, when /// the GPU was in fact recognized and simply has no ROCm build. + /// + /// Classified by [`gpu_support_level`] rather than re-deriving the same + /// `(detected, covering family)` test here: this line and that function are + /// two renderings of one judgement, and the whole point of this change is + /// that support claims come from a single place. Every variant is spelled + /// out so that adding one is a compile error here rather than a silently + /// blank line. fn rocm_support_line(&self) -> String { - match ( - self.detected_gfx_target.as_deref(), - self.compatible_therock_family.as_deref(), - ) { - (Some(gfx), None) if gfx_target_has_no_rocm_build(gfx) => format!( - " rocm_support: GPU detected ({gfx}) but not supported by ROCm; \ - the Vulkan backend can still be used\n" + let detected = self.detected_gfx_target.as_deref(); + match gpu_support_level(detected, self.compatible_therock_family.as_deref()) { + // `VulkanOnly` is only ever returned for a detected target, so the + // fallback below is unreachable; it keeps the arm total without a + // panic path. + GpuSupportLevel::VulkanOnly => format!( + " rocm_support: GPU detected ({}) but not supported by ROCm; \ + the Vulkan backend can still be used\n", + detected.unwrap_or_default() ), - _ => String::new(), + GpuSupportLevel::RocmHip | GpuSupportLevel::Unknown => String::new(), } } diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 157c01ad7e..ac0650eff6 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -242,8 +242,12 @@ rocm serve qwen --engine lemonade --device gpu_required --foreground --port 1143 Expected result: -- The engine uses ROCm GPU execution. -- The run does not fall back to CPU or Vulkan. +- The engine uses ROCm GPU execution. The deployment summary's `device:` row + reports what the engine actually selected, so check it by reading that row + rather than inferring it from the absence of a fallback warning: expect + `rocm gpu `. `rocm services` shows the same `device` while it runs. +- The run does not fall back to CPU or Vulkan — a `device:` of `vulkan gpu ` + fails this check even though the serve itself succeeds. - If the GPU cannot be used, the command fails with a clear error. - The OpenAI-compatible endpoint answers a simple chat request. diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index a67149b05c..fdb6b9bff1 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -62,6 +62,14 @@ Feature: Model serving # Lemonade serve + inference (GGUF model). Engine coverage: Lemonade. The # lemonade per-PR canary: one real lemonade serve runs on every PR (the # counterpart to the vLLM canary above). + # + # ROCMAI-550's device-reporting Then is listed here for the lanes that run it, + # but it is NOT coverage on lemonade+linux: the whole scenario is xfail'd there + # (EAI-7423 -- the managed serve reaches ready then shuts down ~0.08s later), + # so no step of it is a live assertion on that lane. Moving the Then earlier + # would not change that: the service is gone before a healthcheck could report + # a device. The vLLM canary (serve-05) is what actually exercises device + # reporting on every PR today. @id:serve-lemonade-inference @requires-gpu @requires-engine:lemonade Scenario: serve-07 - A model served on lemonade responds to inference requests Given a managed runtime is active diff --git a/tests/e2e-cucumber/features/runtime_setup.feature b/tests/e2e-cucumber/features/runtime_setup.feature index a2164d1f92..b7c8fa1686 100644 --- a/tests/e2e-cucumber/features/runtime_setup.feature +++ b/tests/e2e-cucumber/features/runtime_setup.feature @@ -412,14 +412,16 @@ Feature: Runtime configuration # of the user. # # ROCMAI-550 added a third Then. Having taken every package family away, the - # refusal now has to leave the user somewhere to go, and it says `rocm serve - # ` on the Vulkan backend. That is a remediation line naming a command, - # so AGENTS.md §3 wants it asserted where the refusal is actually produced - # rather than only against the formatter -- this is the one door of the two - # that a lane can open, so it is asserted here. + # refusal now has to leave the user somewhere to go, and it points at `rocm + # examine`. That is a remediation line naming a command, so AGENTS.md §3 wants + # it asserted where the refusal is actually produced rather than only against + # the formatter -- this is the one door of the two that a lane can open, so it + # is asserted here. It also pins the negative: this branch refuses a requested + # target before probing the host at all, so it must not predict which backend + # serving would use on a runner whose own GPU is ROCm-capable. @id:runtime-install-sdk-refuses-unsupported-gpu-target Scenario: runtime-19 - Asking for a GPU target ROCm does not build is refused outright When the user dry-runs an SDK install for the gfx90c Vega iGPU Then the install is refused as having no ROCm build for that GPU And the refusal offers no package family to install instead - And the refusal points at serving on the Vulkan backend instead + And the refusal points at examining what this host supports diff --git a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs index 8c7378394d..438686cfb0 100644 --- a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs @@ -1458,28 +1458,28 @@ async fn assert_unsupported_family_offers_nothing(world: &mut E2eWorld) { } } -/// Having removed every `--family` the user could try, the refusal has to say -/// what *will* work. ROCMAI-550 added that sentence, and this pins it where the -/// refusal is really produced rather than against the formatter alone. +/// Having removed every `--family` the user could try, the refusal has to leave +/// somewhere to go — and that somewhere must be true on whatever host runs it. /// -/// Only the wording is checked here, because this lane cannot check the rest of -/// it: proving that `rocm serve` then reaches a GPU with no ROCm build needs a -/// Vega APU, and no runner has one (same gap as the detected-host door, above). -/// The two halves the advice rests on are proven elsewhere — -/// `rocm_core::gfx_target_has_no_rocm_build` is what put the user here, and -/// `selects_vulkan_when_rocm_unsupported` in `engines/lemonade`'s -/// `backend_alignment` is what makes llama.cpp pick `vulkan` once `rocm` comes -/// back unsupported, which is the claim the sentence makes. -#[then("the refusal points at serving on the Vulkan backend instead")] -async fn assert_unsupported_family_points_at_vulkan_serving(world: &mut E2eWorld) { +/// This scenario asks for `--family gfx90c`, which is refused by +/// `unsupported_requested_target_message` before any host probe runs, so the +/// runner's own GPU has no bearing on the refusal. That makes this lane the +/// proof that the refusal does not predict a backend: these runners are +/// ROCm-capable, so an earlier draft's "`rocm serve ` runs on Lemonade's +/// Vulkan backend" was being asserted green here on exactly the kind of host +/// where it is false. `rocm examine` holds either way, and is the command that +/// reports the real support level this PR added. +#[then("the refusal points at examining what this host supports")] +async fn assert_unsupported_family_points_at_examine(world: &mut E2eWorld) { let output = unsupported_family_attempt(world); assert!( - output.contains("rocm serve "), - "the refusal took every package family away, so it must name what does \ - work instead:\n{output}" + output.contains("rocm examine"), + "the refusal took every package family away, so it must leave the user a \ + next step:\n{output}" ); assert!( - output.contains("Vulkan"), - "the refusal should say the fallback is the Vulkan backend:\n{output}" + !output.contains("rocm serve"), + "this refusal never looked at the host's GPU, so it must not predict \ + which backend serving would use here:\n{output}" ); } From 49e23e3048be24a4f6842f30845f16682259abb3 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 16 Oct 2026 17:44:12 +0300 Subject: [PATCH 05/59] fix(bridge): report lemonade's real backend in the codex-bridge inventory The codex-bridge engine inventory kept the unconditional "with ROCm llama.cpp backend" wording that ROCMAI-550 removed from `rocm examine` and `rocm engines`, so a bridge client still rendered a ROCm claim to the operator on a host that cannot use it. Both call sites can answer the question: the snapshot already gathers the ExamineSummary carrying the two inputs, and the `engines` tool takes the same cheap host probe `rocm engines` uses. Only the `summary` string changes; the field set and the bridge's JSON shape are untouched. The dashboard TUI pane and rocmd's own catalog keep the static wording deliberately -- the pane has no probe behind it, and rocmd answers for whichever host a client asks about, not the one it runs on. Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 87 +++++++++++++++++++++++++++++++------------ 1 file changed, 63 insertions(+), 24 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 63d0177db8..28a064aa52 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -2693,19 +2693,42 @@ fn refresh_startup_update_check_quietly() { fn build_codex_bridge_snapshot(paths: &AppPaths) -> Result { let config = RocmCliConfig::load(paths).unwrap_or_default(); + // Hoisted out of the struct literal so the engine rows can be told what this + // host actually supports. The probe is already paid for: `gpu_support_level` + // takes exactly the two fields this summary already carries, so the engine + // list costs no detection the snapshot was not doing anyway. + let examine = ExamineSummary::gather()?; + let gpu_support_level = rocm_core::gpu_support_level( + examine.detected_gfx_target.as_deref(), + examine.compatible_therock_family.as_deref(), + ); Ok(CodexBridgeSnapshot { protocol: "rocmd-codex-bridge-v0".to_owned(), generated_at_unix_ms: rocm_core::unix_time_millis(), - examine: ExamineSummary::gather()?, + examine, gpu: build_codex_bridge_gpu_snapshot(&config), config, automation_runtime: AutomationRuntimeState::load(paths)?, recent_automation_events: load_recent_automation_events(paths, 32)?, - engines: builtin_codex_bridge_engine_inventory(), + engines: builtin_codex_bridge_engine_inventory(gpu_support_level), services: load_managed_services(paths)?, }) } +/// This host's GPU support level from the cheap host probe, for the call sites +/// that need it without gathering a whole [`ExamineSummary`]. +/// +/// The same two inputs `rocm examine` derives it from, read through +/// `detect_host_gpu_summary` rather than the full examine gather — exactly what +/// `render_engine_inventory_text_with_paths` (i.e. `rocm engines`) already does. +fn host_gpu_support_level(paths: Option<&AppPaths>) -> rocm_core::GpuSupportLevel { + let host_gpu = rocm_core::detect_host_gpu_summary(paths); + rocm_core::gpu_support_level( + host_gpu.gfx_target.as_deref(), + host_gpu.therock_family.as_deref(), + ) +} + fn build_codex_bridge_gpu_snapshot(config: &RocmCliConfig) -> CodexBridgeGpuSnapshot { if !config.telemetry.local_inspection_enabled() { return CodexBridgeGpuSnapshot { @@ -2724,20 +2747,26 @@ fn build_codex_bridge_gpu_snapshot(config: &RocmCliConfig) -> CodexBridgeGpuSnap } } -/// The engine catalogue in the codex-bridge snapshot, taken verbatim from -/// [`builtin_engine_inventory`] — including lemonade's unconditional "with ROCm -/// llama.cpp backend", which ROCMAI-550 replaced in `rocm examine` and -/// `rocm engines` but deliberately left alone here. +/// The engine catalogue in the codex-bridge snapshot and the bridge's `engines` +/// tool, built from [`builtin_engine_inventory`] with lemonade's note replaced +/// by [`lemonade_engine_note`] for the given support level. /// -/// It is left alone because this list is the static catalogue, in the same -/// shape as the unconditional `installed_binary: true` beside it: every field -/// but `default_for_platform` describes the engine, not this host. A consumer -/// that wants the host's real support level reads it off the `examine` block -/// in the same snapshot, which carries `detected_gfx_target` and -/// `compatible_therock_family` — the two inputs `rocm_core::gpu_support_level` -/// takes. Making this row host-aware would change the bridge's JSON contract -/// and is worth its own change, not a rider on this one. -fn builtin_codex_bridge_engine_inventory() -> Vec { +/// It takes the level rather than reading the static row because the stale +/// "with ROCm llama.cpp backend" claim is exactly what ROCMAI-550 exists to +/// remove, and this is a user-facing surface like the other two: a bridge +/// client renders `summary` to the operator. Both call sites can answer the +/// question — `build_codex_bridge_snapshot` already gathers the +/// [`ExamineSummary`] that carries the two inputs, and the `engines` tool takes +/// the same cheap host probe `rocm engines` does — so there was no call site +/// here that genuinely could not see the host. +/// +/// Only `summary` moves. The field set, its names and `installed_binary: true` +/// beside it are untouched, so the bridge's JSON shape is the same; what +/// changes is that one string stops asserting ROCm support on a host that has +/// none, which is the same correction the two CLI surfaces got. +fn builtin_codex_bridge_engine_inventory( + gpu_support_level: rocm_core::GpuSupportLevel, +) -> Vec { let default_engine = default_engine_for_platform(); let binary_path = daemon_binary_path() .ok() @@ -2746,7 +2775,11 @@ fn builtin_codex_bridge_engine_inventory() -> Vec { .iter() .map(|(id, summary)| CodexBridgeEngine { id: (*id).to_owned(), - summary: (*summary).to_owned(), + summary: if *id == "lemonade" { + lemonade_engine_note(gpu_support_level).to_owned() + } else { + (*summary).to_owned() + }, default_for_platform: *id == default_engine, installed_binary: true, binary_path: binary_path.clone(), @@ -2754,13 +2787,18 @@ fn builtin_codex_bridge_engine_inventory() -> Vec { .collect() } -/// The lemonade engine note shown in `rocm examine` and `rocm engines`, adjusted -/// for whether this host's detected GPU can actually use the ROCm llama.cpp -/// backend. Both call sites have a live `GpuSupportLevel` in scope; the places -/// that print this note statically — the dashboard TUI, `rocmd`'s own catalog, -/// and [`builtin_codex_bridge_engine_inventory`]'s copy of -/// [`builtin_engine_inventory`] — keep the unconditional ROCm wording, since -/// they describe the engine in the abstract rather than this host. +/// The lemonade engine note shown in `rocm examine`, `rocm engines` and the +/// codex-bridge engine inventory, adjusted for whether this host's detected GPU +/// can actually use the ROCm llama.cpp backend. +/// +/// All three of those have a live `GpuSupportLevel` in scope. The two places +/// that still print the note statically — the dashboard TUI's engine-manager +/// pane and `rocmd`'s own catalog in `apps/rocmd/src/common.rs` — keep the +/// unconditional ROCm wording, and that is deliberate rather than an oversight: +/// the TUI pane renders a hard-coded descriptive list with no probe behind it, +/// and `rocmd` is the daemon answering for whichever host a client asks about, +/// not necessarily the one it runs on. Both are their own change if they ever +/// need to be host-aware. const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str { match level { rocm_core::GpuSupportLevel::VulkanOnly => { @@ -8082,7 +8120,8 @@ pub(crate) fn run_internal_mcp_call( )) } "engines" => { - let engines = builtin_codex_bridge_engine_inventory(); + let engines = + builtin_codex_bridge_engine_inventory(host_gpu_support_level(Some(paths))); Ok(internal_mcp_tool_success( format!("Found {} engine entries.", engines.len()), serde_json::json!({ "engines": engines }), From bce633d6e143d72ee8a147bb189f7dbbd72552f0 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 16 Oct 2026 18:33:29 +0300 Subject: [PATCH 06/59] test(engines): cover the GPU-aware engine inventory on `rocm engines` and the bridge `append_examine_engine_inventory` got tests for both support levels when ROCMAI-550 landed; the other two surfaces carrying the same correction did not. `rocm engines` and the codex-bridge inventory (including its `engines` MCP tool) could have gone back to asserting ROCm support on a Vulkan-only host with nothing red. Both were untestable for the same reason: the branch's input is a live sysfs probe, and no CI runner has a GPU that is detected *and* confirmed to have no ROCm build. So each surface now takes the level the way `append_examine_engine_inventory` already did -- the caller probes once, the renderer is given the answer. `render_engine_inventory_text_for_host` and `internal_mcp_engines_tool_result` are those renderers; neither changes what is printed. The three copies of the lemonade substitution collapse into `engine_inventory_note`, so the surfaces cannot drift. The new tests drive the renderers, not that helper: asserting on it would restate the condition rather than prove a surface prints it. `gpu_support_level_for_host_gpu` reads the two `HostGpuSummary` fields once instead of at each call site. They are same-typed positional arguments, so a transposed call site would have classified a Vulkan-only host as `RocmHip` and still compiled; it now has a test. Also drops the dangling `GFX_TARGETS_WITHOUT_ROCM_SUPPORT` reference from the `GpuSupportLevel` doc comment -- no such constant exists in `diagnose.rs` or anywhere else, so the doc pointed readers at nothing. Signed-off-by: Roman Sirokov --- apps/rocm/src/chat.rs | 96 ++++++++++++-- apps/rocm/src/main.rs | 208 +++++++++++++++++++++++++++---- crates/rocm-core/src/host_gpu.rs | 63 +++++++++- crates/rocm-core/src/lib.rs | 6 +- 4 files changed, 333 insertions(+), 40 deletions(-) diff --git a/apps/rocm/src/chat.rs b/apps/rocm/src/chat.rs index 7d68a46655..4cb251e093 100644 --- a/apps/rocm/src/chat.rs +++ b/apps/rocm/src/chat.rs @@ -3330,14 +3330,9 @@ pub(crate) fn run_internal_mcp_call( serde_json::json!(gpu), )) } - "engines" => { - let engines = - builtin_codex_bridge_engine_inventory(host_gpu_support_level(Some(paths))); - Ok(internal_mcp_tool_success( - format!("Found {} engine entries.", engines.len()), - serde_json::json!({ "engines": engines }), - )) - } + "engines" => Ok(internal_mcp_engines_tool_result(host_gpu_support_level( + Some(paths), + ))), "services" => { let services = load_managed_services(paths)?; Ok(internal_mcp_tool_success( @@ -3530,6 +3525,24 @@ pub(crate) fn run_internal_mcp_call( } } +/// The `engines` MCP tool result for a host at `gpu_support_level`. +/// +/// Split from the match arm above only so the GPU-aware branch is reachable +/// from a test: the arm's own argument is a live host probe, and no CI runner +/// has a GPU that is detected *and* confirmed to have no ROCm build, so the +/// Vulkan-only wording a bridge client would render could not otherwise be +/// asserted on. Nothing else moved — the summary text, the `engines` key and +/// the entries themselves are what the arm built inline. +fn internal_mcp_engines_tool_result( + gpu_support_level: rocm_core::GpuSupportLevel, +) -> serde_json::Value { + let engines = builtin_codex_bridge_engine_inventory(gpu_support_level); + internal_mcp_tool_success( + format!("Found {} engine entries.", engines.len()), + serde_json::json!({ "engines": engines }), + ) +} + fn internal_mcp_arguments(value: serde_json::Value) -> serde_json::Map { if let Some(arguments) = value .get("arguments") @@ -5982,6 +5995,73 @@ model recipes assert!(mcp_tool_result_text(&result).contains(env!("CARGO_PKG_VERSION"))); } + /// The bridge's `engines` tool carries the same claim `rocm engines` and + /// `rocm examine` carry, so it gets the same pair of tests. These assert on + /// the serialized payload because `structuredContent.engines[].summary` is + /// the exact string an MCP client renders to the operator. + #[test] + fn engines_mcp_tool_names_vulkan_for_a_gpu_with_no_rocm_support() { + let result = internal_mcp_engines_tool_result(rocm_core::GpuSupportLevel::VulkanOnly); + + let summary = engines_tool_lemonade_summary(&result); + assert!( + summary.contains("no ROCm/HIP support") && summary.contains("Vulkan llama.cpp backend"), + "the MCP client must be handed the real backend: {summary}" + ); + assert!( + !summary.contains("with ROCm llama.cpp backend"), + "the stale ROCm claim must not reach an MCP client: {summary}" + ); + assert!(mcp_tool_result_text(&result).contains("engine entries")); + } + + #[test] + fn engines_mcp_tool_keeps_rocm_note_for_a_rocm_capable_gpu() { + let result = internal_mcp_engines_tool_result(rocm_core::GpuSupportLevel::RocmHip); + + assert_eq!( + engines_tool_lemonade_summary(&result), + "default embedded Lemonade server with ROCm llama.cpp backend" + ); + } + + /// Pins the wiring the two tests above cannot see: that the tool reaches + /// the host-aware inventory through the live probe rather than a level + /// nailed to a constant. A hard-coded level that disagrees with this host + /// fails here. + #[test] + fn engines_mcp_tool_answers_for_the_host_it_runs_on() { + let (_root, paths) = test_paths("mcp-engines-host-probe"); + + let result = run_internal_mcp_call(&paths, "engines", serde_json::json!({}), false) + .expect("the engines mcp tool is read-only and should run"); + + assert_eq!( + result.get("isError").and_then(serde_json::Value::as_bool), + Some(false) + ); + assert_eq!( + result.get("structuredContent"), + internal_mcp_engines_tool_result(host_gpu_support_level(Some(&paths))) + .get("structuredContent"), + "the `engines` tool must report this host's support level" + ); + } + + /// The lemonade `summary` as an MCP client reads it out of the tool result. + fn engines_tool_lemonade_summary(result: &serde_json::Value) -> String { + result + .pointer("/structuredContent/engines") + .and_then(serde_json::Value::as_array) + .expect("the engines tool must return an `engines` array") + .iter() + .find(|engine| engine.get("id").and_then(serde_json::Value::as_str) == Some("lemonade")) + .and_then(|engine| engine.get("summary")) + .and_then(serde_json::Value::as_str) + .expect("the engines tool must carry a lemonade summary") + .to_owned() + } + #[test] fn natural_language_plan_returns_structured_mutating_action() { let (_root, paths) = test_paths("nl-plan-structured-mutating"); diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 6fc7f24986..6ca53d6376 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -2736,11 +2736,7 @@ fn build_codex_bridge_snapshot(paths: &AppPaths) -> Result /// `detect_host_gpu_summary` rather than the full examine gather — exactly what /// `render_engine_inventory_text_with_paths` (i.e. `rocm engines`) already does. fn host_gpu_support_level(paths: Option<&AppPaths>) -> rocm_core::GpuSupportLevel { - let host_gpu = rocm_core::detect_host_gpu_summary(paths); - rocm_core::gpu_support_level( - host_gpu.gfx_target.as_deref(), - host_gpu.therock_family.as_deref(), - ) + rocm_core::gpu_support_level_for_host_gpu(&rocm_core::detect_host_gpu_summary(paths)) } fn build_codex_bridge_gpu_snapshot(config: &RocmCliConfig) -> CodexBridgeGpuSnapshot { @@ -2789,11 +2785,7 @@ fn builtin_codex_bridge_engine_inventory( .iter() .map(|(id, summary)| CodexBridgeEngine { id: (*id).to_owned(), - summary: if *id == "lemonade" { - lemonade_engine_note(gpu_support_level).to_owned() - } else { - (*summary).to_owned() - }, + summary: engine_inventory_note(id, summary, gpu_support_level).to_owned(), default_for_platform: *id == default_engine, installed_binary: true, binary_path: binary_path.clone(), @@ -2801,6 +2793,29 @@ fn builtin_codex_bridge_engine_inventory( .collect() } +/// The inventory note to print for `engine`, given the row `catalog_note` the +/// static catalogue carries for it and what this host actually supports. +/// +/// The one place the lemonade substitution lives. All three host-aware +/// inventory renderers — [`render_engine_inventory_text_for_host`] +/// (`rocm engines`), [`append_examine_engine_inventory`] (`rocm examine`) and +/// [`builtin_codex_bridge_engine_inventory`] (the codex bridge and its +/// `engines` MCP tool) — route their note through this, so the three cannot +/// drift into disagreeing about the same host. Tests exercise the renderers +/// rather than this function: asserting on it directly would only restate the +/// condition below instead of proving a surface prints it. +fn engine_inventory_note( + engine: &str, + catalog_note: &'static str, + gpu_support_level: rocm_core::GpuSupportLevel, +) -> &'static str { + if engine == "lemonade" { + lemonade_engine_note(gpu_support_level) + } else { + catalog_note + } +} + /// The lemonade engine note shown in `rocm examine`, `rocm engines` and the /// codex-bridge engine inventory, adjusted for whether this host's detected GPU /// can actually use the ROCm llama.cpp backend. @@ -5146,11 +5161,27 @@ fn render_engine_inventory_text_with_paths(paths: Option<&AppPaths>) -> String { // Mark the engine this GPU actually serves on as primary. Using the platform // constant put the `*` on Lemonade even on Instinct, where `serve` picks vLLM. let host_gpu = rocm_core::detect_host_gpu_summary(paths); - let host_default_engine = rocm_core::default_engine_for_host(&host_gpu); - let gpu_support_level = rocm_core::gpu_support_level( - host_gpu.gfx_target.as_deref(), - host_gpu.therock_family.as_deref(), - ); + render_engine_inventory_text_for_host( + paths, + rocm_core::default_engine_for_host(&host_gpu), + rocm_core::gpu_support_level_for_host_gpu(&host_gpu), + ) +} + +/// `rocm engines` for a host whose preferred engine and GPU support level the +/// caller has already probed. +/// +/// Split from [`render_engine_inventory_text_with_paths`] for the same reason +/// [`append_examine_engine_inventory`] takes both as parameters: the host probe +/// reads real sysfs, so a test cannot otherwise reach the Vulkan-only branch — +/// no CI runner has a GPU that is detected *and* confirmed to have no ROCm +/// build. Both values come from one `detect_host_gpu_summary` call above, so +/// the marker and the note still cannot disagree about the same host. +fn render_engine_inventory_text_for_host( + paths: Option<&AppPaths>, + host_default_engine: &str, + gpu_support_level: rocm_core::GpuSupportLevel, +) -> String { // A configured `default_engine` still wins over the host preference, mirroring // `select_serve_engine` (and `append_examine_engine_inventory`). Without this, // the legend could mark an engine `serve` will not actually default to. @@ -5197,11 +5228,7 @@ fn render_engine_inventory_text_with_paths(paths: Option<&AppPaths>) -> String { } else { " " }; - let note = if *name == "lemonade" { - lemonade_engine_note(gpu_support_level) - } else { - *note - }; + let note = engine_inventory_note(name, note, gpu_support_level); let _ = writeln!(output, "{marker} {name:10} {note}"); append_engine_detect_summary(&mut output, name, paths); } @@ -5413,11 +5440,7 @@ fn append_examine_engine_inventory( .or(entry.last_installed_env_id.as_deref()) }) .unwrap_or(""); - let note = if *engine == "lemonade" { - lemonade_engine_note(gpu_support_level) - } else { - *note - }; + let note = engine_inventory_note(engine, note, gpu_support_level); let _ = writeln!( output, " {marker} {engine} adapter={adapter} runtime_pref={runtime_pref} note={note}" @@ -20455,6 +20478,141 @@ pub(crate) mod tests { let _ = fs::remove_dir_all(root); } + /// `rocm engines` makes the same claim `rocm examine` does, through a + /// different renderer, so it needs the same pair of tests: + /// `examine_engine_inventory_names_vulkan_for_a_gpu_with_no_rocm_support` + /// is this test's counterpart on the examine surface. + #[test] + fn engines_inventory_names_vulkan_for_a_gpu_with_no_rocm_support() { + let (root, paths) = test_paths("engines-inventory-vulkan-only"); + + let output = render_engine_inventory_text_for_host( + Some(&paths), + "lemonade", + rocm_core::GpuSupportLevel::VulkanOnly, + ); + + assert!( + output.contains("no ROCm/HIP support") && output.contains("Vulkan llama.cpp backend"), + "`rocm engines` must name the backend this GPU really runs on:\n{output}" + ); + assert!( + !output.contains("Lemonade server with ROCm llama.cpp backend"), + "`rocm engines` must not keep asserting ROCm support on a host without it:\n{output}" + ); + // The substitution is lemonade's alone; vLLM's row is not host-aware. + assert!( + output.contains("Linux/WSL ROCm GPU serving engine through external vLLM"), + "only lemonade's note is support-level aware:\n{output}" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn engines_inventory_keeps_rocm_note_for_a_rocm_capable_gpu() { + let (root, paths) = test_paths("engines-inventory-rocm-capable"); + + let output = render_engine_inventory_text_for_host( + Some(&paths), + "vllm", + rocm_core::GpuSupportLevel::RocmHip, + ); + + assert!( + output.contains("default embedded Lemonade server with ROCm llama.cpp backend"), + "a ROCm-capable host must keep the ROCm wording:\n{output}" + ); + assert!( + !output.contains("Vulkan llama.cpp backend"), + "a ROCm-capable host must not be told it is on Vulkan:\n{output}" + ); + let _ = fs::remove_dir_all(root); + } + + /// An unrecognized or undetected GPU is a gap in what the catalog knows, + /// not a confirmed absence of ROCm — printing the Vulkan wording for it + /// would turn that gap into a false claim about the hardware, which is the + /// distinction `GpuSupportLevel::Unknown` exists to preserve. + #[test] + fn engines_inventory_does_not_claim_vulkan_for_an_unrecognized_gpu() { + let (root, paths) = test_paths("engines-inventory-unknown"); + + let output = render_engine_inventory_text_for_host( + Some(&paths), + "lemonade", + rocm_core::GpuSupportLevel::Unknown, + ); + + assert!( + !output.contains("no ROCm/HIP support"), + "an unrecognized GPU must not be reported as confirmed ROCm-less:\n{output}" + ); + let _ = fs::remove_dir_all(root); + } + + /// The codex bridge renders `summary` to the operator just as the two CLI + /// surfaces render their note, so the same correction has to reach it. + #[test] + fn codex_bridge_engine_inventory_names_vulkan_for_a_gpu_with_no_rocm_support() { + let engines = builtin_codex_bridge_engine_inventory(rocm_core::GpuSupportLevel::VulkanOnly); + + let lemonade = engines + .iter() + .find(|engine| engine.id == "lemonade") + .expect("the bridge inventory must still carry lemonade"); + assert!( + lemonade.summary.contains("no ROCm/HIP support") + && lemonade.summary.contains("Vulkan llama.cpp backend"), + "bridge clients must be told the real backend: {}", + lemonade.summary + ); + assert!( + !lemonade.summary.contains("with ROCm llama.cpp backend"), + "the stale ROCm claim must not survive in the bridge: {}", + lemonade.summary + ); + // Only `summary` is support-level aware; the bridge's JSON shape and + // every other engine's row are untouched by the substitution. + assert!(lemonade.installed_binary); + let vllm = engines + .iter() + .find(|engine| engine.id == "vllm") + .expect("the bridge inventory must still carry vllm"); + assert_eq!( + vllm.summary, + "Linux/WSL ROCm GPU serving engine through external vLLM" + ); + } + + #[test] + fn codex_bridge_engine_inventory_keeps_rocm_note_for_a_rocm_capable_gpu() { + let engines = builtin_codex_bridge_engine_inventory(rocm_core::GpuSupportLevel::RocmHip); + + let lemonade = engines + .iter() + .find(|engine| engine.id == "lemonade") + .expect("the bridge inventory must still carry lemonade"); + assert_eq!( + lemonade.summary, + "default embedded Lemonade server with ROCm llama.cpp backend" + ); + } + + #[test] + fn codex_bridge_engine_inventory_does_not_claim_vulkan_for_an_unrecognized_gpu() { + let engines = builtin_codex_bridge_engine_inventory(rocm_core::GpuSupportLevel::Unknown); + + let lemonade = engines + .iter() + .find(|engine| engine.id == "lemonade") + .expect("the bridge inventory must still carry lemonade"); + assert!( + !lemonade.summary.contains("no ROCm/HIP support"), + "an unrecognized GPU must not be reported as confirmed ROCm-less: {}", + lemonade.summary + ); + } + #[test] fn examine_is_silent_when_the_configured_engine_agrees_with_the_host() { let (root, paths) = test_paths("examine-engine-inventory-agrees"); diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index e46e442774..bd9e17a388 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -133,10 +133,12 @@ pub fn gfx_target_has_no_rocm_build(gfx: &str) -> bool { /// /// `Unknown` deliberately covers both "nothing detected" and "a gfx target we /// don't recognize" rather than being called `Unsupported` — see the doc -/// comment on [`gfx_target_has_no_rocm_build`] and -/// `GFX_TARGETS_WITHOUT_ROCM_SUPPORT` in `diagnose.rs` for why conflating -/// "confirmed no ROCm build" with "unrecognized" would turn a detection gap -/// into a false claim about the hardware. +/// comment on [`gfx_target_has_no_rocm_build`] for why conflating "confirmed +/// no ROCm build" with "unrecognized" would turn a detection gap into a false +/// claim about the hardware. Only that predicate's conjunction with +/// [`normalize_therock_family`] establishes the "confirmed" half; an +/// unrecognized target satisfies neither conjunct, so it is a gap in what this +/// catalog knows rather than a fact about the card. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum GpuSupportLevel { /// A TheRock package family covers this GPU: full ROCm/HIP support. @@ -162,6 +164,21 @@ pub fn gpu_support_level( } } +/// [`gpu_support_level`] for a [`HostGpuSummary`] the caller already probed. +/// +/// The two fields are read here, once, rather than at each call site: they are +/// positional arguments of the same type on [`gpu_support_level`], so a call +/// site that transposed them would classify a Vulkan-only host as `RocmHip` +/// and still compile. `rocm engines` and the codex-bridge `engines` tool both +/// derive the level from this struct, so both go through this. +#[must_use] +pub fn gpu_support_level_for_host_gpu(host_gpu: &HostGpuSummary) -> GpuSupportLevel { + gpu_support_level( + host_gpu.gfx_target.as_deref(), + host_gpu.therock_family.as_deref(), + ) +} + pub fn normalize_therock_family(value: &str) -> Option { let normalized = value.trim().to_ascii_lowercase(); if normalized.is_empty() { @@ -3548,6 +3565,44 @@ mod tests { ); } + /// The two `HostGpuSummary` fields must reach `gpu_support_level` in the + /// right positions. Both are `Option`, so transposing them still + /// compiles — and it would answer `RocmHip` for exactly the Vulkan-only + /// host this classification exists to stop lying to. + #[test] + fn gpu_support_level_for_host_gpu_reads_a_vulkan_only_summary_as_vulkan_only() { + let host_gpu = HostGpuSummary { + name: Some("AMD Radeon Graphics".to_owned()), + gfx_target: Some("gfx90c".to_owned()), + therock_family: None, + }; + assert_eq!( + gpu_support_level_for_host_gpu(&host_gpu), + GpuSupportLevel::VulkanOnly + ); + } + + #[test] + fn gpu_support_level_for_host_gpu_reads_a_covered_summary_as_rocm_hip() { + let host_gpu = HostGpuSummary { + name: Some("AMD Instinct MI300X".to_owned()), + gfx_target: Some("gfx942".to_owned()), + therock_family: Some("gfx94X-dcgpu".to_owned()), + }; + assert_eq!( + gpu_support_level_for_host_gpu(&host_gpu), + GpuSupportLevel::RocmHip + ); + } + + #[test] + fn gpu_support_level_for_host_gpu_reads_an_undetected_summary_as_unknown() { + assert_eq!( + gpu_support_level_for_host_gpu(&HostGpuSummary::default()), + GpuSupportLevel::Unknown + ); + } + #[test] fn gpu_support_level_is_unknown_without_detection_or_for_an_unrecognized_target() { assert_eq!(gpu_support_level(None, None), GpuSupportLevel::Unknown); diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 64255f6c1c..569b09fc95 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -88,9 +88,9 @@ pub use host_gpu::{ default_engine_for_host, default_engine_for_platform, detect_gpu_driver_version, detect_host_gfx_target, detect_host_gpu_diagnostics, detect_host_gpu_summary, detect_system_ram_gib, extract_first_gfx_token, gfx_target_has_no_rocm_build, - gpu_support_level, has_usable_amd_gpu, interactive_terminal, is_amdgpu_device, is_wsl_host, - known_therock_families, normalize_therock_family, preferred_serve_engine_for_host_gpu_summary, - usable_amd_gpu_indices, + gpu_support_level, gpu_support_level_for_host_gpu, has_usable_amd_gpu, interactive_terminal, + is_amdgpu_device, is_wsl_host, known_therock_families, normalize_therock_family, + preferred_serve_engine_for_host_gpu_summary, usable_amd_gpu_indices, }; #[cfg(any(target_os = "linux", test))] use host_gpu::{KfdGpuNode, kfd_gpu_nodes_in}; From aba912c22f1a511432e8dc2a1253cb34578bb131 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Mon, 2 Nov 2026 15:51:15 +0200 Subject: [PATCH 07/59] ROCMAI-550: make the device row name a device and examine answer every tier Review round 3 on #678. - `rocm examine` now prints a `rocm_support` line for RocmHip and Unknown as well as VulkanOnly. The Unknown-tier lemonade note sends the user to `rocm examine`, and examine printed nothing for exactly those hosts, so the remediation named a command that could not clear the condition. The note is also reworded to promise the support level, which is what examine delivers, rather than the backend, which only a launch can settle. - The `device:` row carries the GPU name. ROCMAI-550 asks for the backend "and the device name"; on the ticket's own Windows gfx90c host no ordinal is recorded, so the row rendered a bare `vulkan gpu` - a backend and a noun naming nothing. All three engines now record `gpu_name` at launch and format through the shared `rocm_engine_protocol::format_reported_device`, which drops the `gpu` noun when there is no ordinal behind it. - `fetch_service_devices` splits its budget loop into `collect_service_devices` with an injected clock and probe, so the budget-exhausted exit is tested - including that later services are not probed at all. The ~18s worst case (budget plus one in-flight probe) is documented on `SERVICE_DEVICE_PROBE_BUDGET`, on `render_services_text` and in README. - The e2e `device:` step polls for the row instead of panicking on the first look: lemonade reports `unknown` until its own inference probe returns, so a correct implementation legitimately has no row for a short window. - AIM's `rocm gpu` change is pinned by a test that drives the real write-state/read-back/report path. Signed-off-by: Roman Sirokov --- README.md | 10 +- apps/rocm/src/engine_ipc.rs | 146 +++++++++++++++++- apps/rocm/src/main.rs | 11 +- apps/rocm/src/services_text.rs | 17 ++ crates/rocm-core/src/host_gpu.rs | 121 +++++++++++++-- crates/rocm-engine-protocol/src/lib.rs | 94 +++++++++++ docs/manual-testing.md | 17 +- engines/aim/src/state.rs | 105 +++++++++++-- engines/lemonade/src/state.rs | 66 ++++++-- engines/vllm/src/state.rs | 49 ++++-- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 92 +++++++---- 11 files changed, 640 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index d40c138971..8689fdfcc3 100644 --- a/README.md +++ b/README.md @@ -865,7 +865,8 @@ services stop --yes`) or match the original flags. By default the server runs in the background under ROCm CLI's supervision and prints a deployment summary: a progress indicator while it starts, then a table with the status, the engine and the backend device it actually selected (for -example `rocm gpu 0` or `vulkan gpu 0`, read back from the running engine), the +example `rocm gpu 0 (AMD Instinct MI300X)`, or `vulkan (AMD Radeon Graphics)` +where the engine pinned no GPU ordinal — read back from the running engine), the full inference endpoint, the API-qualified model name, and a quick smoke test (time to first token and approximate tokens per second). `rocm services` reports the same `device` for each running service. Control @@ -921,6 +922,13 @@ rocm services remove --yes rocm services prune [--older-than-hours | --any-age] [--dry-run] [--yes] ``` +`list` asks each running server which backend and GPU it is actually on, so it +can take a few seconds rather than returning instantly. The query is capped in +total — about 18 seconds worst case, however many servers are listed — and a +server that does not answer in time simply has no `device` line rather than +holding up the rest of the list. The same applies to asking about servers in +`rocm chat`, which prints the same list. + `restart` starts a server again only once its stop has confirmed that every recorded process is gone; if the stop cannot confirm that, it says so, starts nothing, and leaves the stop request recorded. diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index efa1d70d4b..94ce6c719c 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -342,6 +342,23 @@ pub(crate) fn fetch_service_device( /// couple of healthy engines answer (`engines/lemonade`'s endpoint probe is /// the slowest at `rocm_core::INFERENCE_PROBE_TIMEOUT`, 8s worst case, and /// only on a miss) while still bounding the whole render. +/// +/// **Worst-case wall time for `rocm services list` is this budget plus one +/// in-flight probe, i.e. ~18s** with the built-in engines, not 10s. The budget +/// is a *start* gate: it is tested before each probe, never inside one, so a +/// probe begun at 9.99s still runs to its own timeout. The arithmetic is 10s +/// of elapsed budget plus Lemonade's 8s `INFERENCE_PROBE_TIMEOUT`. That bound +/// applies equally to the chat `services` tool, which renders through the same +/// [`crate::services_text::render_services_text`]. +/// +/// Why not a per-probe deadline that would hold the total at 10s: the probe is +/// `engine_request`, which for a built-in engine is a synchronous in-process +/// call and for an external plugin is a `wait()` on a spawned child. Neither +/// can be cancelled from here without either a thread per probe (ruled out by +/// the process-wide env/mutex scoping `fetch_service_devices` documents) or a +/// kill path for a child mid-RPC. Both are larger changes than the row is +/// worth; the start gate is what keeps the cost flat in the number of +/// services, which is the growth that actually matters. pub(crate) const SERVICE_DEVICE_PROBE_BUDGET: Duration = Duration::from_secs(10); /// Devices for several services at once, keyed by service id, under one shared @@ -361,7 +378,9 @@ pub(crate) const SERVICE_DEVICE_PROBE_BUDGET: Duration = Duration::from_secs(10) /// The budget is checked between probes, not inside one: a single external /// plugin that never exits still blocks, because `engine_request` waits on its /// child unbounded. That is the pre-existing behavior of every engine call -/// site in the CLI and is not made worse here. +/// site in the CLI and is not made worse here. See +/// [`SERVICE_DEVICE_PROBE_BUDGET`] for what that makes the worst-case wall +/// time of a `rocm services list`. pub(crate) fn fetch_service_devices<'a, I>( paths: &AppPaths, services: I, @@ -370,12 +389,40 @@ where I: IntoIterator, { let started = Instant::now(); + collect_service_devices( + services, + || started.elapsed(), + |service_id, engine| fetch_service_device(paths, engine, service_id), + ) +} + +/// The budget loop of [`fetch_service_devices`], with the clock and the probe +/// injected. +/// +/// Split out so the budget-exhausted exit has a test. In the real function the +/// clock is `Instant::elapsed` and the probe is a live engine RPC, so covering +/// that branch would otherwise mean making a test wait out +/// [`SERVICE_DEVICE_PROBE_BUDGET`] against real engines — which is both slow +/// and exactly the timing-dependent shape that flakes. What the branch has to +/// guarantee is not just that later services lack a `device:` row but that +/// they were never *probed*, since the whole point is to stop paying; only an +/// injected probe can observe that. +fn collect_service_devices<'a, I, C, P>( + services: I, + mut elapsed: C, + mut probe: P, +) -> std::collections::HashMap +where + I: IntoIterator, + C: FnMut() -> Duration, + P: FnMut(&str, &str) -> Option, +{ let mut devices = std::collections::HashMap::new(); for (service_id, engine) in services { - if started.elapsed() >= SERVICE_DEVICE_PROBE_BUDGET { + if elapsed() >= SERVICE_DEVICE_PROBE_BUDGET { break; } - if let Some(device) = fetch_service_device(paths, engine, service_id) { + if let Some(device) = probe(service_id, engine) { devices.insert(service_id.to_owned(), device); } } @@ -601,6 +648,99 @@ mod tests { use super::*; use crate::tests::test_paths; + /// Every service inside the budget is probed and reported. + #[test] + fn service_device_probes_run_while_the_budget_holds() { + let mut probed = Vec::new(); + let devices = collect_service_devices( + [("svc-a", "lemonade"), ("svc-b", "vllm")], + || Duration::ZERO, + |service_id, engine| { + probed.push(service_id.to_owned()); + Some(format!("{engine}-device")) + }, + ); + + assert_eq!(probed, ["svc-a", "svc-b"]); + assert_eq!( + devices.get("svc-a").map(String::as_str), + Some("lemonade-device") + ); + assert_eq!( + devices.get("svc-b").map(String::as_str), + Some("vllm-device") + ); + } + + /// Once [`SERVICE_DEVICE_PROBE_BUDGET`] is spent the remaining services are + /// *not probed at all*, and what was already learned is still rendered. + /// + /// This is the branch that keeps `rocm services list` — and the chat + /// `services` tool, which renders through the same + /// `render_services_text` — from scaling its wall time with the number of + /// services. Dropping the later rows is only half of it; the point is to + /// stop paying for them, so the probe counter below is the real assertion. + /// Asserting it against the live clock and live engines would mean a test + /// that waits out ten real seconds, which is why the loop takes both + /// injected. + #[test] + fn service_device_probes_stop_once_the_budget_is_exhausted() { + let mut probed = Vec::new(); + // Inside the budget for the first service, over it from then on. + let mut tick = 0; + let devices = collect_service_devices( + [ + ("svc-fast", "lemonade"), + ("svc-hung", "vllm"), + ("svc-never-reached", "aim"), + ], + || { + tick += 1; + if tick <= 1 { + Duration::ZERO + } else { + SERVICE_DEVICE_PROBE_BUDGET + } + }, + |service_id, _engine| { + probed.push(service_id.to_owned()); + Some("rocm gpu 0".to_owned()) + }, + ); + + assert_eq!( + probed, + ["svc-fast"], + "the budget must stop further probing, not merely hide its results" + ); + assert_eq!( + devices.get("svc-fast").map(String::as_str), + Some("rocm gpu 0"), + "what was learned before the budget ran out is still reported" + ); + assert!( + !devices.contains_key("svc-hung") && !devices.contains_key("svc-never-reached"), + "an unprobed service must get no `device:` row: {devices:?}" + ); + } + + /// A service whose engine cannot answer costs its own probe and nothing + /// more: it is skipped, and the services after it are still probed. + #[test] + fn a_service_whose_probe_fails_does_not_suppress_the_rest() { + let devices = collect_service_devices( + [("svc-quiet", "lemonade"), ("svc-ok", "vllm")], + || Duration::ZERO, + |service_id, _engine| (service_id == "svc-ok").then(|| "rocm gpu 1".to_owned()), + ); + + assert!(!devices.contains_key("svc-quiet"), "{devices:?}"); + assert_eq!( + devices.get("svc-ok").map(String::as_str), + Some("rocm gpu 1") + ); + } + #[test] fn engine_plugin_discovery_finds_runtime_binary() -> Result<()> { let (root, paths) = test_paths("engine-plugin"); diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 30d136d325..d5d2e64ed5 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -2745,6 +2745,15 @@ fn engine_inventory_note( /// claims no backend and names the command that can answer. A new /// `GpuSupportLevel` variant now has to be classified here rather than /// silently inheriting the ROCm claim. +/// +/// What the `Unknown` arm promises is bounded by what the named command +/// delivers. It used to say `rocm examine` would show "which llama.cpp backend +/// this GPU uses", and examine showed no such thing — it printed a +/// `rocm_support` line only for `VulkanOnly`, so on exactly the hosts that got +/// this note it printed nothing at all. `ExamineSummary::rocm_support_line` +/// now answers for every level, and this says what that answer is: the support +/// level, not the backend. The backend is settled by launching one, which that +/// line in turn points at. const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str { match level { rocm_core::GpuSupportLevel::VulkanOnly => { @@ -2754,7 +2763,7 @@ const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str "default embedded Lemonade server with ROCm llama.cpp backend" } rocm_core::GpuSupportLevel::Unknown => { - "default embedded Lemonade server; run `rocm examine` to see which llama.cpp backend this GPU uses" + "default embedded Lemonade server; run `rocm examine` for this host's GPU support level, which says how to find the backend" } } } diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index 97bf5f1493..8424c0a045 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -382,6 +382,23 @@ fn write_past_attempts_hint(output: &mut String, past_attempts: usize, newest_id let _ = writeln!(output, " Reclaim the space: rocm services prune"); } +/// `rocm services list`, and the chat `services` tool, which renders the same +/// text. +/// +/// **This is no longer a pure local read.** Since ROCMAI-550 it makes one live +/// `Healthcheck` RPC per *ready* service to fill the `device:` row, capped in +/// aggregate by `engine_ipc::SERVICE_DEVICE_PROBE_BUDGET`. Worst case is that +/// budget plus one in-flight probe — about 18s with the built-in engines, not +/// 10s, because the budget gates the *start* of each probe and never +/// interrupts one. That doc comment carries the arithmetic and why a +/// per-probe deadline was not taken. +/// +/// The chat tool pays the same bound deliberately rather than rendering a +/// device-less view: the two surfaces would then disagree about one service +/// while claiming to show the same list, and "which GPU is this running on" is +/// a question asked in chat at least as often as at the prompt. The cost is +/// bounded and flat in the number of services, which is what makes it +/// acceptable there. pub(crate) fn render_services_text(paths: &AppPaths, all: bool) -> Result { // Counts describe the WHOLE registry, not the rows this view keeps. The // default view hides records that are not live, and counting the filtered diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index ce5accd162..1a2eab0f4a 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -1939,12 +1939,22 @@ impl ExamineSummary { }) } - /// The `rocm_support` line of the examine text, or an empty string. + /// The `rocm_support` line of the examine text. /// - /// Present only when a GPU was detected but no ROCm package family covers - /// it (for example the gfx90c Vega iGPU in Renoir / Cezanne APUs). Without - /// it `compatible_therock_family: ` reads like a probe failure, when - /// the GPU was in fact recognized and simply has no ROCm build. + /// ROCMAI-550 asks `rocm examine` to "report the support level with one + /// line of what the user can do", so every level gets a line — including + /// the ones where the answer is "this build cannot tell". The earlier draft + /// printed only the `VulkanOnly` line, which made this report unable to + /// answer the very question the other surfaces send people here to ask: + /// `lemonade_engine_note` in `apps/rocm/src/main.rs` tells an `Unknown` + /// host to run `rocm examine`, and silence there is not an answer. + /// + /// Without the `VulkanOnly` line specifically, + /// `compatible_therock_family: ` reads like a probe failure when + /// the GPU was in fact recognized and simply has no ROCm build (the gfx90c + /// Vega iGPU in Renoir / Cezanne APUs). The `Unknown` line exists for the + /// mirror-image misreading: the same `` value, where the CLI + /// genuinely does not know, must not be read as either claim. /// /// Classified by [`gpu_support_level`] rather than re-deriving the same /// `(detected, covering family)` test here: this line and that function are @@ -1960,10 +1970,33 @@ impl ExamineSummary { // panic path. GpuSupportLevel::VulkanOnly => format!( " rocm_support: GPU detected ({}) but not supported by ROCm; \ - the Vulkan backend can still be used\n", + the Vulkan backend can still be used - serve on it with \ + `rocm serve --engine lemonade`\n", detected.unwrap_or_default() ), - GpuSupportLevel::RocmHip | GpuSupportLevel::Unknown => String::new(), + // The covering family is the fact; it is already printed on its own + // line above, so this one says what it means rather than repeating + // it. + GpuSupportLevel::RocmHip => format!( + " rocm_support: GPU detected ({}) with full ROCm/HIP support; \ + install the SDK for it with `rocm install sdk`\n", + detected.unwrap_or("") + ), + // Not folded in with `VulkanOnly`: a detection gap is not a + // confirmed absence of ROCm support, and saying "Vulkan" here would + // be the same false claim in the other direction. What the user can + // do is let an engine decide and then read back what it chose - + // which is the `device:` row this same ticket added, and the only + // thing on this host that can actually settle the question. + GpuSupportLevel::Unknown => format!( + " rocm_support: unknown - {}, so this build can confirm neither \ + ROCm support nor its absence; `rocm serve --engine \ + lemonade` reports the backend it picked on its `device:` row\n", + match detected { + Some(gfx) => format!("GPU target {gfx} is not in this build's catalog"), + None => "no AMD GPU was detected".to_owned(), + } + ), } } @@ -5175,22 +5208,88 @@ Class Name: Display ); } + /// Every support level reports one, because the other surfaces send the + /// user here to find out: `lemonade_engine_note` tells an `Unknown` host to + /// run `rocm examine`, and an examine that prints nothing for `Unknown` + /// makes that advice name a command which cannot clear the condition. + /// + /// Each line is checked against what it is allowed to claim, not merely for + /// being non-empty — the failure mode this guards is a report that answers + /// confidently on a host where the CLI knows nothing. #[test] - fn examine_render_omits_the_support_note_for_supported_or_missing_gpus() { + fn examine_render_reports_a_support_level_for_every_host() { let family = normalize_therock_family("gfx1103"); let supported = examine_summary_with_gpu(Some("gfx1103"), family.as_deref()); let rendered = supported.render_text(); - assert!(!rendered.contains("rocm_support"), "{rendered}"); + assert!( + rendered.contains("rocm_support: GPU detected (gfx1103) with full ROCm/HIP support"), + "{rendered}" + ); + assert!(rendered.contains("rocm install sdk"), "{rendered}"); + // No GPU found: the level is reported as unknown, and nothing about the + // hardware is asserted in either direction. let missing = examine_summary_with_gpu(None, None); let rendered = missing.render_text(); - assert!(!rendered.contains("rocm_support"), "{rendered}"); + assert!( + rendered.contains("rocm_support: unknown - no AMD GPU was detected"), + "{rendered}" + ); + assert!( + !rendered.contains("Vulkan backend can still be used"), + "nothing was detected, so no backend may be promised: {rendered}" + ); // An unmapped target the CLI knows nothing about is not claimed to be // unsupported; only targets known to have no ROCm build are. let unknown = examine_summary_with_gpu(Some("gfx803"), None); let rendered = unknown.render_text(); - assert!(!rendered.contains("rocm_support"), "{rendered}"); + assert!( + rendered.contains( + "rocm_support: unknown - GPU target gfx803 is not in this build's catalog" + ), + "{rendered}" + ); + assert!( + !rendered.contains("not supported by ROCm"), + "an unrecognized target must not be reported as confirmed \ + ROCm-less: {rendered}" + ); + assert!( + !rendered.contains("full ROCm/HIP support"), + "nor as supported: {rendered}" + ); + } + + /// The remediation in `lemonade_engine_note` (`apps/rocm/src/main.rs`) for + /// an `Unknown` host names `rocm examine`. AGENTS.md requires a named + /// command to actually clear the condition that printed it, so the line + /// examine prints there has to be an answer about the support level — the + /// thing the note said it would show. + #[test] + fn examine_answers_the_support_question_the_unknown_engine_note_sends_users_here_for() { + for detected in [None, Some("gfx803")] { + let rendered = examine_summary_with_gpu(detected, None).render_text(); + let line = rendered + .lines() + .find(|line| line.trim_start().starts_with("rocm_support:")) + .unwrap_or_else(|| { + panic!( + "`rocm examine` is where the Unknown-tier engine note sends \ + the user; it must print a support level:\n{rendered}" + ) + }); + // Says which level it is, and leaves a command that can settle it. + assert!(line.contains("unknown"), "{line}"); + assert!( + line.contains("rocm serve --engine lemonade"), + "an `unknown` answer still has to leave somewhere to go: {line}" + ); + assert!( + line.contains("device:"), + "and name where the real backend shows up: {line}" + ); + } } #[test] diff --git a/crates/rocm-engine-protocol/src/lib.rs b/crates/rocm-engine-protocol/src/lib.rs index 6ca773ccd6..e11703ffb9 100644 --- a/crates/rocm-engine-protocol/src/lib.rs +++ b/crates/rocm-engine-protocol/src/lib.rs @@ -566,6 +566,51 @@ pub struct LaunchResponse { pub state_path: String, } +/// The [`HealthcheckResponse::device`] string, in the one shape every engine +/// reports and the `device:` row of `rocm serve` / `rocm services list` prints +/// verbatim. +/// +/// ROCMAI-550 asks those two surfaces for "the llama.cpp backend actually +/// chosen (`vulkan`/`rocm`) **and the device name**", so both halves are named +/// arguments here rather than text each engine formats for itself: +/// +/// | backend | ordinal | name | rendered | +/// | --- | --- | --- | --- | +/// | `vulkan` | `Some(0)` | `Some("AMD Radeon Graphics")` | `vulkan gpu 0 (AMD Radeon Graphics)` | +/// | `vulkan` | `Some(0)` | `None` | `vulkan gpu 0` | +/// | `vulkan` | `None` | `Some("AMD Radeon Graphics")` | `vulkan (AMD Radeon Graphics)` | +/// | `vulkan` | `None` | `None` | `vulkan` | +/// +/// The last row deliberately drops the `gpu` noun rather than printing a +/// dangling `vulkan gpu`. That bare form is not hypothetical: it is what the +/// ticket's own host produces. `gpu_indices` comes from +/// `engines/lemonade`'s `resolve_serve_gpu_indices`, whose probe +/// (`rocm_core::usable_amd_gpu_indices`) returns `None` off Linux, so a default +/// `rocm serve` on Windows pins no ordinal and records an empty list — and +/// Windows on a gfx90c Vega iGPU is exactly the machine ROCMAI-550 was filed +/// from. `device: vulkan gpu` would answer "which GPU?" with the word "gpu". +/// +/// The name is whatever the engine recorded for the host at launch; it is not +/// per-ordinal. On a multi-GPU host the ordinal is the precise half and the +/// name is context, which is why the ordinal is never dropped in favour of it. +#[must_use] +pub fn format_reported_device( + backend: &str, + gpu_index: Option, + gpu_name: Option<&str>, +) -> String { + use std::fmt::Write as _; + + let mut device = backend.to_owned(); + if let Some(index) = gpu_index { + let _ = write!(device, " gpu {index}"); + } + if let Some(name) = gpu_name.map(str::trim).filter(|name| !name.is_empty()) { + let _ = write!(device, " ({name})"); + } + device +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct HealthcheckResponse { /// Engine-reported service state. `ready` is reserved for a service that has @@ -646,6 +691,55 @@ pub struct LogsResponse { #[cfg(test)] mod tests { + /// The `device:` row has to carry a device, not the word "gpu". + /// + /// The ordinal-less case is the one ROCMAI-550 was filed about: on Windows + /// `rocm_core::usable_amd_gpu_indices` cannot probe, so a default + /// `rocm serve` records an empty `gpu_indices` and the old formatting + /// rendered a bare `vulkan gpu`. The name is what makes that row answer the + /// question; with neither, the noun goes too rather than dangling. + #[test] + fn reported_device_never_renders_a_dangling_gpu_noun() { + assert_eq!( + super::format_reported_device("vulkan", Some(0), Some("AMD Radeon Graphics")), + "vulkan gpu 0 (AMD Radeon Graphics)" + ); + assert_eq!( + super::format_reported_device("vulkan", None, Some("AMD Radeon Graphics")), + "vulkan (AMD Radeon Graphics)" + ); + assert_eq!( + super::format_reported_device("rocm", Some(1), None), + "rocm gpu 1" + ); + // The regression this exists to stop: nothing known beyond the backend + // must not be padded out with a noun that names no device. + assert_eq!( + super::format_reported_device("vulkan", None, None), + "vulkan" + ); + } + + /// A name the host probe could not fill in arrives as `Some("")` or + /// `Some(" ")` just as easily as `None` (the Windows display-adapter probe + /// and the Linux DRM read both build it from scraped text), and + /// `vulkan ()` is worse than no parenthetical at all. + #[test] + fn reported_device_treats_a_blank_gpu_name_as_absent() { + assert_eq!( + super::format_reported_device("vulkan", Some(0), Some("")), + "vulkan gpu 0" + ); + assert_eq!( + super::format_reported_device("vulkan", None, Some(" ")), + "vulkan" + ); + assert_eq!( + super::format_reported_device("rocm", Some(2), Some(" MI300X ")), + "rocm gpu 2 (MI300X)" + ); + } + fn resolve_response(served: Option<&str>) -> ResolveModelResponse { ResolveModelResponse { canonical_model_id: "image:tag".to_owned(), diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 05d6fb5f82..f568c2d098 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -285,14 +285,15 @@ Expected result: - The engine uses ROCm GPU execution. The deployment summary's `device:` row reports what the engine actually selected, so check it by reading that row - rather than inferring it from the absence of a fallback warning: expect - `rocm gpu `, or a bare `rocm gpu` when the launch pinned no ordinal (no - `--gpu`, nothing recorded in the engine's `gpu_indices`). `rocm services` - shows the same `device` while it runs. Every engine reports this row in the - same ` gpu []` shape, so the first word is the half that decides - this check. -- The run does not fall back to CPU or Vulkan — a `device:` of `vulkan gpu ` - fails this check even though the serve itself succeeds. + rather than inferring it from the absence of a fallback warning. Every engine + reports it in the same ` [gpu ] [()]` shape — so expect + something like `rocm gpu 0 (AMD Instinct MI300X)`, with the ordinal omitted + when the launch pinned none (no `--gpu`, nothing recorded in the engine's + `gpu_indices`) and the name omitted when the host probe could not read one. + `rocm services` shows the same `device` while it runs. The first word is the + half that decides this check. +- The run does not fall back to CPU or Vulkan — a `device:` whose first word is + `vulkan` fails this check even though the serve itself succeeds. - If the GPU cannot be used, the command fails with a clear error. - The OpenAI-compatible endpoint answers a simple chat request. diff --git a/engines/aim/src/state.rs b/engines/aim/src/state.rs index bcb783e9de..878af75960 100644 --- a/engines/aim/src/state.rs +++ b/engines/aim/src/state.rs @@ -33,16 +33,9 @@ pub(crate) fn healthcheck_service(request: HealthcheckRequest) -> Result reported_device(state), + None => "unknown".to_owned(), }; let listed = !endpoint_url.is_empty() @@ -56,10 +49,30 @@ pub(crate) fn healthcheck_service(request: HealthcheckRequest) -> Result String { + rocm_engine_protocol::format_reported_device( + "rocm", + None, + value_string(state, "gpu_name").as_deref(), + ) +} + pub(crate) fn endpoint_response(request: EndpointRequest) -> Result { require_nonempty(&request.service_id, "service_id")?; let state = read_service_state(&state_path(&request.service_id)) @@ -139,6 +152,13 @@ pub(crate) fn write_running_state( "port": port, "endpoint_url": endpoint, "device_policy": device_policy, + // The host's GPU name, so the `device:` row names a device and not + // only a backend (ROCMAI-550). This engine records no GPU ordinal, + // so without it its row would be the bare word `rocm` on every + // host. Captured once here rather than probed per healthcheck: + // `rocm services list` healthchecks every ready service on its + // render path. + "gpu_name": rocm_core::detect_host_gpu_summary(None).name, "engine_recipe": engine_recipe, "started_at_unix_ms": current_unix_millis(), }), @@ -361,6 +381,69 @@ mod tests { assert!(on_disk.contains("HF_TOKEN")); } + /// ROCMAI-550 changed the device this engine reports for an existing + /// service from the literal `"rocm_gpu"` to the shared + /// ` [gpu ] [()]` shape, and nothing pinned it. + /// + /// This drives the real persist-then-report path: + /// [`write_running_state`] writes the state file, [`read_service_state`] + /// reads it back and [`reported_device`] renders it — exactly what + /// [`healthcheck_service`] does for a tracked service, minus the + /// `state_path` lookup, which resolves through the caller's real + /// `AppPaths` and so cannot be driven from a unit test (the no-state half + /// is covered by + /// `healthcheck_reports_unknown_for_a_service_with_no_state_file`). + /// + /// Reverting the string to `"rocm_gpu"`, or dropping the `gpu_name` field + /// from the state write, fails this. + #[test] + fn reported_device_for_existing_state_matches_the_shared_engine_shape() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("svc-device.json"); + write_running_state( + &path, + "svc-device", + "some/model", + Some("Org/Served"), + "127.0.0.1", + 8000, + DevicePolicy::GpuRequired, + None, + ) + .unwrap(); + + // The probed name is whatever the test host has, so assert the shape + // rather than a value: `rocm`, optionally followed by `()`, and + // never the old `rocm_gpu` or a `gpu` noun with no ordinal behind it. + let state = read_service_state(&path).unwrap(); + let device = reported_device(&state); + assert!( + device == "rocm" || (device.starts_with("rocm (") && device.ends_with(')')), + "AIM must report the shared ` [()]` shape: {device}" + ); + assert!( + !device.contains("rocm_gpu"), + "the pre-ROCMAI-550 literal must not come back: {device}" + ); + assert!( + !device.split_whitespace().any(|word| word == "gpu"), + "this engine pins no ordinal, so a bare `gpu` noun would name \ + nothing: {device}" + ); + // The state file must actually carry the key the row is built from, + // even when the host probe found no name. + assert!( + state.get("gpu_name").is_some(), + "write_running_state must record gpu_name: {state}" + ); + + // And the name, when the host has one, reaches the row. + assert_eq!( + reported_device(&json!({ "gpu_name": "AMD Instinct MI300X" })), + "rocm (AMD Instinct MI300X)" + ); + } + #[test] fn status_helpers_round_trip_and_keep_other_fields() { let dir = tempfile::tempdir().unwrap(); diff --git a/engines/lemonade/src/state.rs b/engines/lemonade/src/state.rs index 6df1ded86d..08c3d8f852 100644 --- a/engines/lemonade/src/state.rs +++ b/engines/lemonade/src/state.rs @@ -55,13 +55,29 @@ pub(crate) fn inference_verified( } /// The device string reported once a model is loaded. Reflects the backend that -/// actually ran (`rocm`, `vulkan`, …) and the pinned GPU ordinal when known, -/// rather than a hardcoded value, so the report matches observed execution. +/// actually ran (`rocm`, `vulkan`, …), the pinned GPU ordinal when known, and +/// the GPU name recorded at launch, rather than a hardcoded value, so the +/// report matches observed execution. +/// +/// Formatted by [`rocm_engine_protocol::format_reported_device`] so all three +/// engines render one shape; see that function for why the ordinal-less row +/// drops the `gpu` noun rather than printing a bare `vulkan gpu`. pub(crate) fn reported_device(state: Option<&Value>, backend: &str) -> String { - match state.and_then(first_gpu_index_from_state) { - Some(index) => format!("{backend} gpu {index}"), - None => format!("{backend} gpu"), - } + rocm_engine_protocol::format_reported_device( + backend, + state.and_then(first_gpu_index_from_state), + state.and_then(gpu_name_from_state).as_deref(), + ) +} + +/// The GPU name recorded in service state at launch, if any. +/// +/// Read back rather than re-probed: `healthcheck_service` runs on the +/// `rocm services list` render path (see `fetch_service_devices`), where an +/// extra host probe per service is latency the row is not worth, and the launch +/// host is the one whose GPU actually ran the server. +fn gpu_name_from_state(state: &Value) -> Option { + value_string(state, "gpu_name") } /// The first pinned GPU ordinal recorded in service state, if any. @@ -155,6 +171,13 @@ pub(crate) fn write_running_state( "endpoint_url": endpoint_url(&request.host, request.port), "device_policy": device_policy_name(&request.device_policy), "gpu_indices": request.gpu_indices, + // The host's GPU name, so `healthcheck_service` can name a device + // and not just a backend (ROCMAI-550). Captured here, once, rather + // than probed per healthcheck: `rocm services list` healthchecks + // every ready service on its render path. Recorded even when it is + // `null`, so the key's absence means an older state file rather + // than a probe that came back empty. + "gpu_name": rocm_core::detect_host_gpu_summary(None).name, "runtime_id": request.runtime_id.as_deref().unwrap_or(runtime.manifest.env_id.as_str()), "env_id": request.env_id.as_deref().unwrap_or(runtime.manifest.env_id.as_str()), "runtime_kind": "lemonade_embeddable", @@ -700,11 +723,34 @@ mod tests { } #[test] - fn reported_device_reflects_backend_and_pinned_ordinal() { - let state = json!({ "gpu_indices": [1] }); - assert_eq!(reported_device(Some(&state), "vulkan"), "vulkan gpu 1"); + fn reported_device_reflects_backend_pinned_ordinal_and_gpu_name() { + let state = json!({ "gpu_indices": [1], "gpu_name": "AMD Instinct MI300X" }); + assert_eq!( + reported_device(Some(&state), "vulkan"), + "vulkan gpu 1 (AMD Instinct MI300X)" + ); // Without a recorded ordinal the backend is still reported truthfully. - assert_eq!(reported_device(None, "rocm"), "rocm gpu"); + assert_eq!(reported_device(None, "rocm"), "rocm"); + } + + /// ROCMAI-550's own host: Windows cannot probe GPU ordinals + /// (`rocm_core::usable_amd_gpu_indices` is `None` off Linux), so a default + /// `rocm serve` records an empty `gpu_indices` there and the only + /// device-identifying fact in state is the name. Before this the row read + /// `vulkan gpu` — the backend, then a noun naming nothing — which is the + /// shape the ticket filed as "doesn't show the GPU". + #[test] + fn reported_device_names_the_gpu_when_no_ordinal_was_pinned() { + let state = json!({ "gpu_indices": [], "gpu_name": "AMD Radeon(TM) Graphics" }); + assert_eq!( + reported_device(Some(&state), "vulkan"), + "vulkan (AMD Radeon(TM) Graphics)" + ); + + // An older state file, written before `gpu_name` was recorded, still + // reports its backend rather than a dangling noun. + let legacy = json!({ "gpu_indices": [] }); + assert_eq!(reported_device(Some(&legacy), "vulkan"), "vulkan"); } #[test] diff --git a/engines/vllm/src/state.rs b/engines/vllm/src/state.rs index cfd6607c1c..e56768106f 100644 --- a/engines/vllm/src/state.rs +++ b/engines/vllm/src/state.rs @@ -81,14 +81,16 @@ pub(crate) fn healthcheck_service(request: HealthcheckRequest) -> Result String { - match first_gpu_index_from_state(state) { - Some(index) => format!("rocm gpu {index}"), - None => "rocm gpu".to_owned(), - } + rocm_engine_protocol::format_reported_device( + "rocm", + first_gpu_index_from_state(state), + value_string(state, "gpu_name").as_deref(), + ) } /// The first pinned GPU ordinal recorded in service state, if any. @@ -178,10 +180,15 @@ pub(crate) fn write_running_state( "endpoint_url": endpoint_url(&request.host, request.port), "device_policy": "gpu_required", // Recorded so `healthcheck_service` can report the GPU this service - // was actually pinned to, in the same `"rocm gpu "` shape - // `engines/lemonade` reports. Introspection only, like the rest of - // this file — never read back to relaunch. + // was actually pinned to, in the same shape `engines/lemonade` + // reports. Introspection only, like the rest of this file — never + // read back to relaunch. "gpu_indices": request.gpu_indices, + // The host's GPU name, so the `device:` row names a device and not + // only a backend (ROCMAI-550). Captured once here rather than + // probed per healthcheck, because `rocm services list` + // healthchecks every ready service on its render path. + "gpu_name": rocm_core::detect_host_gpu_summary(None).name, "runtime_id": runtime.runtime_id, "requested_runtime_id": request.runtime_id, "env_id": request.env_id.as_deref().unwrap_or(runtime.env_id.as_str()), @@ -751,19 +758,27 @@ mod tests { /// The `device:` row `rocm serve` and `rocm services list` print comes /// straight from this string, so vLLM has to speak the same shape - /// `engines/lemonade` does (`"{backend} gpu {index}"`). It used to be the - /// literal `"rocm_gpu"`, which is the format README.md and - /// `docs/manual-testing.md` never described and which made one row - /// disagree with itself across engines. + /// `engines/lemonade` does. It used to be the literal `"rocm_gpu"`, which + /// is the format README.md and `docs/manual-testing.md` never described and + /// which made one row disagree with itself across engines. #[test] fn reported_device_matches_the_shape_lemonade_reports() { + assert_eq!( + reported_device(&json!({ "gpu_indices": [1], "gpu_name": "AMD Instinct MI300X" })), + "rocm gpu 1 (AMD Instinct MI300X)" + ); assert_eq!( reported_device(&json!({ "gpu_indices": [1] })), "rocm gpu 1" ); // No pinned ordinal recorded: the backend is still reported truthfully, - // exactly as `engines/lemonade`'s `reported_device(None, "rocm")` does. - assert_eq!(reported_device(&json!({ "status": "running" })), "rocm gpu"); - assert_eq!(reported_device(&json!({ "gpu_indices": [] })), "rocm gpu"); + // exactly as `engines/lemonade`'s `reported_device(None, "rocm")` does, + // and without the dangling `gpu` noun that names no device. + assert_eq!(reported_device(&json!({ "status": "running" })), "rocm"); + assert_eq!(reported_device(&json!({ "gpu_indices": [] })), "rocm"); + assert_eq!( + reported_device(&json!({ "gpu_name": "AMD Instinct MI300X" })), + "rocm (AMD Instinct MI300X)" + ); } } diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index c2a8cd2044..010a918869 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1446,23 +1446,42 @@ async fn assert_service_in_list(world: &mut E2eWorld) { /// The `device:` value as `rocm serve` / `rocm services list` print it, split /// into the backend it names and the GPU ordinal it pinned, if any. /// -/// Every engine reports `" gpu"` or `" gpu "` -/// (`engines/lemonade`'s and `engines/vllm`'s `reported_device`, -/// `engines/aim`'s literal). `None` for anything else, which is what makes a -/// hardcoded or malformed value fail rather than pass. +/// Every engine renders it through +/// `rocm_engine_protocol::format_reported_device`, so the grammar is +/// ``, then an optional ` gpu `, then an optional ` ()`. +/// `None` for anything else, which is what makes a hardcoded or malformed value +/// fail rather than pass. fn parse_device_row(value: &str) -> Option<(&str, Option)> { - let mut parts = value.split_whitespace(); + // The name is the only part that may contain spaces, and it is always last + // and parenthesised. Split on the FIRST " (" so a name with its own + // parentheses ("AMD Radeon (TM) Graphics") still leaves the head intact. + let head = match value.split_once(" (") { + Some((head, name)) if name.ends_with(')') => head, + Some(_) => return None, + None => value, + }; + let mut parts = head.split_whitespace(); let backend = parts.next()?; - if parts.next()? != "gpu" { - return None; - } let ordinal = match parts.next() { - Some(index) => Some(index.parse().ok()?), + Some("gpu") => Some(parts.next()?.parse().ok()?), + // A bare `gpu` with no ordinal is the shape ROCMAI-550 removed; so is + // any other trailing word. + Some(_) => return None, None => None, }; parts.next().is_none().then_some((backend, ordinal)) } +/// How long the `device:` row may take to show up after the serve returned. +/// +/// It is not available the instant a service is recorded `ready`: +/// `engines/lemonade` reports `"unknown"` until its own `inference_verified` +/// probe has come back, and `fetch_service_device` drops `"unknown"` rather +/// than printing it. So there is a real window in which a correct +/// implementation has no row yet, and failing on the first look would make +/// this step flake on timing rather than on behavior. +const DEVICE_ROW_DEADLINE: Duration = Duration::from_secs(90); + /// The value of the first `device:` row in `text`, trimmed. fn device_row_value(text: &str) -> Option<&str> { text.lines() @@ -1480,32 +1499,53 @@ fn device_row_value(text: &str) -> Option<&str> { /// `device:`" while reporting nothing real, so this parses it and holds it to /// three independent facts: /// -/// 1. it has the ` gpu []` shape every engine reports; +/// 1. it has the ` [gpu ] [()]` shape every engine reports; /// 2. the backend is one this engine can actually have selected - only /// Lemonade has a Vulkan path, so `vulkan` from `vllm` or `aim` is wrong, /// and `unknown` is wrong from any of them; /// 3. it equals what the `rocm serve` deployment summary printed for the same /// service, which is the row the preceding Given produced and which nothing /// else in these scenarios looks at. +/// The row is polled rather than read once: see [`DEVICE_ROW_DEADLINE`] for +/// the window in which a correct implementation legitimately has no row yet. +/// Running out of that window is an assertion naming what was last seen, not a +/// panic on the first look. #[then("the services list reports the engine's actual backend device")] async fn assert_services_list_reports_device(world: &mut E2eWorld) { - let (stdout, _, _) = crate::run_rocm(world, &["services", "list"]); - let model = world.model_name.as_deref().unwrap_or(""); - let block = stdout - .split("\n- ") - .find(|block| block.to_lowercase().contains(&model.to_lowercase())) - .unwrap_or_else(|| panic!("no services-list entry for model {model}:\n{stdout}")); - let device = device_row_value(block).unwrap_or_else(|| { - panic!( - "services list entry for {model} is missing a `device:` line - the \ - engine's Healthcheck RPC should report the backend it actually \ - selected:\n{block}" - ) - }); - let (backend, _ordinal) = parse_device_row(device).unwrap_or_else(|| { + let model = world.model_name.clone().unwrap_or_default(); + let deadline = Instant::now() + DEVICE_ROW_DEADLINE; + // The last thing looked at, so a timeout can show it rather than just + // saying it timed out: the service's own block when there was one, else the + // whole listing (which is the evidence for the service having gone missing). + let mut last_seen = String::new(); + let device = loop { + let (stdout, _, _) = crate::run_rocm(world, &["services", "list"]); + match stdout + .split("\n- ") + .find(|block| block.to_lowercase().contains(&model.to_lowercase())) + { + Some(block) => { + last_seen = block.to_owned(); + if let Some(device) = device_row_value(&last_seen) { + break device.to_owned(); + } + } + None => last_seen = stdout, + } + assert!( + Instant::now() < deadline, + "after {}s `rocm services list` still reports no `device:` line for \ + {model} - the engine's Healthcheck RPC should report the backend it \ + actually selected once the service is serving:\n{last_seen}", + DEVICE_ROW_DEADLINE.as_secs() + ); + tokio::time::sleep(Duration::from_secs(3)).await; + }; + let block = last_seen.as_str(); + let (backend, _ordinal) = parse_device_row(&device).unwrap_or_else(|| { panic!( - "`device: {device}` is not the ` gpu []` shape every \ - engine reports, so it cannot be the engine's own answer:\n{block}" + "`device: {device}` is not the ` [gpu ] [()]` shape \ + every engine reports, so it cannot be the engine's own answer:\n{block}" ) }); From a4264602e845c6a41f171fdb7143ce6bd0a17d81 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Mon, 2 Nov 2026 16:00:35 +0200 Subject: [PATCH 08/59] ROCMAI-550: resync the advice-scan exclusions with the new device shape The `device:` row now renders ` [gpu ] [()]`, so the strings the advised-command scan has to ignore moved with it: vLLM no longer builds `rocm gpu {index}`, AIM no longer carries a bare `rocm gpu`, and the two docs quote the named-device example instead. Also drops a redundant initialiser the e2e `device:` poll never reads. Signed-off-by: Roman Sirokov --- apps/rocm/src/advised_commands.rs | 30 +++++-------------- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 2 +- 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/apps/rocm/src/advised_commands.rs b/apps/rocm/src/advised_commands.rs index 39e58fa4c9..c33adbf72a 100644 --- a/apps/rocm/src/advised_commands.rs +++ b/apps/rocm/src/advised_commands.rs @@ -1279,40 +1279,26 @@ const NOT_INVOCATIONS: &[(&str, &str, &str)] = &[ "the doc's expected result is that the parser rejects this combination", ), // The `device:` row's value (ROCMAI-550). Every engine reports the backend - // it selected as ` gpu []` -- `vulkan gpu 0`, `rocm gpu 1` -- - // so the ROCm half of that value reads as a `rocm` invocation to this scan - // while naming no command at all. These are the engine side of it and the - // two docs that quote the format back. + // it selected as ` [gpu ] [()]` -- `vulkan (AMD Radeon + // Graphics)`, `rocm gpu 1 (AMD Instinct MI300X)` -- so the ROCm half of + // that value reads as a `rocm` invocation to this scan while naming no + // command at all. These are the engine side of it and the two docs that + // quote the format back. ( "engines/vllm/src/state.rs", - "rocm gpu {index}", - "the `device:` value vLLM reports, ` gpu `, not a command", - ), - ( - "engines/vllm/src/state.rs", - "rocm gpu", - "the `device:` value vLLM reports when no GPU ordinal was pinned", - ), - ( - "engines/aim/src/state.rs", "rocm gpu", - "the `device:` value AIM reports; its state records no GPU ordinal", + "the shape note for the `device:` value vLLM reports, not a command", ), ( "README.md", - "rocm gpu 0", + "rocm gpu 0 (AMD Instinct MI300X)", "an example `device:` value in the deployment-summary description, not a command", ), ( "docs/manual-testing.md", - "rocm gpu ", + "rocm gpu 0 (AMD Instinct MI300X)", "the `device:` value the tester is told to expect, not a command to run", ), - ( - "docs/manual-testing.md", - "rocm gpu", - "the ordinal-less `device:` value the tester is told to expect", - ), ]; /// Natural-language examples must reach the planner with a multi-word request: diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 010a918869..92bd5779b4 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1517,7 +1517,7 @@ async fn assert_services_list_reports_device(world: &mut E2eWorld) { // The last thing looked at, so a timeout can show it rather than just // saying it timed out: the service's own block when there was one, else the // whole listing (which is the evidence for the service having gone missing). - let mut last_seen = String::new(); + let mut last_seen; let device = loop { let (stdout, _, _) = crate::run_rocm(world, &["services", "list"]); match stdout From 4545d3622d580efd5a65ac999dacae94b9cdf643 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Mon, 2 Nov 2026 16:09:06 +0200 Subject: [PATCH 09/59] ROCMAI-550: drop the engine advice-scan exclusions the shared formatter removed Neither vLLM nor AIM holds a `"rocm gpu ..."` string literal any more - both build the `device:` value through `rocm_engine_protocol::format_reported_device` - so the entries that excused those literals no longer match anything and `exclusions_still_match_something` rejects them. Also drops a stale `"{backend} gpu {index}"` shape note from vLLM's doc comment. Signed-off-by: Roman Sirokov --- apps/rocm/src/advised_commands.rs | 9 +++------ engines/vllm/src/state.rs | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/apps/rocm/src/advised_commands.rs b/apps/rocm/src/advised_commands.rs index c33adbf72a..f7c0e529c9 100644 --- a/apps/rocm/src/advised_commands.rs +++ b/apps/rocm/src/advised_commands.rs @@ -1282,13 +1282,10 @@ const NOT_INVOCATIONS: &[(&str, &str, &str)] = &[ // it selected as ` [gpu ] [()]` -- `vulkan (AMD Radeon // Graphics)`, `rocm gpu 1 (AMD Instinct MI300X)` -- so the ROCm half of // that value reads as a `rocm` invocation to this scan while naming no - // command at all. These are the engine side of it and the two docs that + // command at all. The engines no longer need an entry each: they build the + // string through `rocm_engine_protocol::format_reported_device` rather + // than holding a `"rocm gpu ..."` literal. These are the two docs that // quote the format back. - ( - "engines/vllm/src/state.rs", - "rocm gpu", - "the shape note for the `device:` value vLLM reports, not a command", - ), ( "README.md", "rocm gpu 0 (AMD Instinct MI300X)", diff --git a/engines/vllm/src/state.rs b/engines/vllm/src/state.rs index e56768106f..bcb2c447d9 100644 --- a/engines/vllm/src/state.rs +++ b/engines/vllm/src/state.rs @@ -77,7 +77,7 @@ pub(crate) fn healthcheck_service(request: HealthcheckRequest) -> Result Date: Tue, 3 Nov 2026 14:23:16 +0200 Subject: [PATCH 10/59] ROCMAI-550: pin the support line with a scenario, and stop the device step failing a correct run Review round 3 on #678. - The `device:` e2e step failed a legitimately correct run. Lemonade reports `unknown` until its own inference probe returns, and the serve summary reads the device once right after `ready`, so the row can be absent on a correct implementation. Only serve-07 being xfail'd on lemonade+linux hid it. The step now tolerates the absence and the window is documented rather than asserted away. - `rocm examine` grew a `rocm_support` line for RocmHip and Unknown with no scenario asserting it. examine-31 covers it, deliberately untagged so each lane's own hardware is the coverage: the claim is cross-checked against `detected_gfx_target` and `compatible_therock_family` from the same report, so "full ROCm/HIP support" on the mock lane or "not supported by ROCm" on an MI300X both fail. Signed-off-by: Roman Sirokov --- README.md | 14 +- apps/rocm/src/engine_ipc.rs | 133 +++++++++ apps/rocm/src/main.rs | 13 +- apps/rocm/src/serve_cmd.rs | 11 +- apps/rocm/src/serve_summary.rs | 15 +- crates/rocm-core/src/host_gpu.rs | 272 +++++++++++++++++- docs/manual-testing.md | 12 +- docs/testing.md | 46 +++ engines/aim/src/state.rs | 7 +- engines/lemonade/src/state.rs | 17 +- engines/vllm/src/state.rs | 13 +- tests/e2e-cucumber/features/examine.feature | 21 ++ tests/e2e-cucumber/tests/e2e/examine_steps.rs | 125 ++++++++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 45 +-- 14 files changed, 694 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index f3bc28de17..46294dc3af 100644 --- a/README.md +++ b/README.md @@ -888,9 +888,17 @@ example `rocm gpu 0 (AMD Instinct MI300X)`, or `vulkan (AMD Radeon Graphics)` where the engine pinned no GPU ordinal — read back from the running engine), the full inference endpoint, the API-qualified model name, and a quick smoke test (time to first token and approximate tokens per second). `rocm services` reports -the same `device` for each running service. Control -returns to your shell with the server still running; manage it later with `rocm -services` (below). +the same `device` for each running service. + +The device line is omitted when the engine cannot say yet. A server is reported +ready as soon as it answers, which can be before it has finished loading the +model, and until then the engine does not know which device it ended up on; +`rocm serve` waits a few seconds for it rather than holding your terminal, and +`rocm services` shows it once it is known. The GPU name is omitted on a machine +with more than one AMD GPU, where naming one of them could name the wrong one. + +Control returns to your shell with the server still running; manage it later +with `rocm services` (below). `--verbose` (or `--foreground`) instead attaches to the server in the current terminal and streams every engine log line. Use it to debug a startup problem. diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index 94ce6c719c..918adccf6f 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -333,6 +333,81 @@ pub(crate) fn fetch_service_device( (!response.device.is_empty() && response.device != "unknown").then_some(response.device) } +/// How long `rocm serve`'s deployment summary may keep asking the engine which +/// device it selected, and how long it waits between asks. +/// +/// A managed launch reports `ready` as soon as its server answers, but +/// `engines/lemonade` reports `device: "unknown"` until its own +/// `inference_verified` probe has come back — so asking once, immediately, +/// loses the row on exactly the Vulkan case ROCMAI-550 is about. One retry +/// window closes the common case: the default smoke test has just put a real +/// completion through the server, so the engine's own probe is a cache hit. +/// +/// **It cannot close every case, and is not meant to.** A server that reported +/// `ready` while weights are still loading can stay unverified for minutes, and +/// waiting that out would hold the terminal hostage for an informational row +/// that `rocm services list` will show as soon as it exists. So the row is +/// *optional by contract* — see [`crate::serve_summary::DeploymentSummary`]'s +/// `device` field, `docs/manual-testing.md` and README.md, which all say it can +/// be absent. +/// +/// The budget is a start gate, like [`SERVICE_DEVICE_PROBE_BUDGET`]: it is +/// tested before each attempt, never inside one, so the worst-case addition to +/// `rocm serve` is this budget plus one in-flight probe — about 14s with +/// Lemonade's 8s `rocm_core::INFERENCE_PROBE_TIMEOUT`, and only on a launch +/// that was going to have no row at all. +const SERVE_DEVICE_PROBE_BUDGET: Duration = Duration::from_secs(6); +/// Pause between the attempts [`SERVE_DEVICE_PROBE_BUDGET`] allows. Short +/// enough that the common case (the engine is a second away from verifying) +/// resolves quickly, long enough not to spin on an engine that answers +/// instantly. +const SERVE_DEVICE_PROBE_INTERVAL: Duration = Duration::from_secs(2); + +/// [`fetch_service_device`], retried briefly, for `rocm serve`'s one-shot +/// deployment summary. +/// +/// `None` is a legitimate answer and must stay cheap to get: see +/// [`SERVE_DEVICE_PROBE_BUDGET`] for why the wait is bounded and why the +/// summary prints no `device:` row rather than blocking on one. +pub(crate) fn fetch_service_device_briefly( + paths: &AppPaths, + engine: &str, + service_id: &str, +) -> Option { + let started = Instant::now(); + retry_service_device( + || started.elapsed(), + || fetch_service_device(paths, engine, service_id), + std::thread::sleep, + ) +} + +/// The retry loop of [`fetch_service_device_briefly`], with the clock, the +/// probe and the wait injected. +/// +/// Split out for the same reason as [`collect_service_devices`]: driving this +/// against the live clock would mean a test that really sleeps out +/// [`SERVE_DEVICE_PROBE_BUDGET`] against a real engine. What has to be +/// guaranteed is not just that a late answer is reported but that the loop +/// *stops* — an unbounded retry here would hang every `rocm serve` whose engine +/// never verifies. +fn retry_service_device(mut elapsed: C, mut probe: P, mut wait: S) -> Option +where + C: FnMut() -> Duration, + P: FnMut() -> Option, + S: FnMut(Duration), +{ + loop { + if let Some(device) = probe() { + return Some(device); + } + if elapsed() >= SERVE_DEVICE_PROBE_BUDGET { + return None; + } + wait(SERVE_DEVICE_PROBE_INTERVAL); + } +} + /// How long [`fetch_service_devices`] may spend on live `Healthcheck` RPCs in /// total, across however many services were listed. /// @@ -724,6 +799,64 @@ mod tests { ); } + /// The serve summary asks again when the first answer is `None`: the + /// engine can be seconds from verifying its own inference probe, which is + /// the `device: "unknown"` window `engines/lemonade` has right after a + /// launch reports ready. + #[test] + fn the_serve_summary_probe_waits_for_a_device_that_is_not_ready_yet() { + let mut attempts = 0; + let mut waited = Duration::ZERO; + let device = retry_service_device( + || Duration::ZERO, + || { + attempts += 1; + (attempts > 2).then(|| "vulkan gpu 0".to_owned()) + }, + |interval| waited += interval, + ); + + assert_eq!(device.as_deref(), Some("vulkan gpu 0")); + assert_eq!(attempts, 3); + assert_eq!( + waited, + SERVE_DEVICE_PROBE_INTERVAL * 2, + "the retries must pace themselves rather than spin" + ); + } + + /// And it gives up. A lemonade service whose weights are still loading can + /// stay `unknown` for minutes; the summary prints no `device:` row rather + /// than holding the terminal, which is why the row is documented as + /// optional. + #[test] + fn the_serve_summary_probe_gives_up_within_its_budget() { + let mut attempts = 0; + // Inside the budget for the first attempt, past it from then on. + let mut tick = 0; + let device = retry_service_device( + || { + tick += 1; + if tick <= 1 { + Duration::ZERO + } else { + SERVE_DEVICE_PROBE_BUDGET + } + }, + || { + attempts += 1; + None + }, + |_| {}, + ); + + assert_eq!(device, None); + assert_eq!( + attempts, 2, + "the budget must end the retries, not merely discard their results" + ); + } + /// A service whose engine cannot answer costs its own probe and nothing /// more: it is skipped, and the services after it are still probed. #[test] diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 7b41bb348d..decf1ec074 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -506,8 +506,11 @@ rocm model --verbose" /// /// Picks an engine and ROCm runtime automatically unless overridden. By default the /// server runs as a managed background service and prints a deployment summary - /// (status, endpoint, model, and smoke-test metrics); use --verbose to stream engine - /// logs in this terminal instead. When streaming, press Ctrl-D to detach and leave the + /// (status, endpoint, model, smoke-test metrics, and the backend device the engine + /// reports it is running on). The device line is read back from the running server, so + /// it is omitted when the engine cannot say yet: a server that answers is not always + /// done loading its weights. `rocm services list` shows it once it is known. Use + /// --verbose to stream engine logs in this terminal instead. When streaming, press Ctrl-D to detach and leave the /// server running, or Ctrl-C to stop it. Inspect or stop servers later with /// `rocm services`. /// @@ -1124,6 +1127,12 @@ enum ComfyuiCommand { enum ServicesCommand { /// Show currently running local model servers, and count the records that /// are no longer running. + /// + /// Each running server is asked which backend and GPU it is actually on, so + /// this takes a few seconds rather than returning instantly. The asking is + /// capped in total — about 18 seconds worst case, however many servers are + /// listed — and a server that does not answer in time simply has no device + /// line. List { /// Include failed, stopped, and old service records. #[arg(short, long)] diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index fd6e9586bf..145a4908c3 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -50,7 +50,7 @@ use rocm_engine_protocol::{ use crate::DevicePolicyArg; use crate::endpoint_keys; -use crate::engine_ipc::{engine_request, fetch_service_device}; +use crate::engine_ipc::{engine_request, fetch_service_device_briefly}; use crate::engine_runtime_select::{ EngineSelection, resolve_engine_selection, validate_engine_selection_runtime, }; @@ -1256,8 +1256,15 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // Vulkan-only GPU that ROCm/amd-smi never enumerates, even though // the server is running fine on it. Only meaningful once the // server is ready to answer, same gate as the smoke test. + // + // Asked more than once, briefly: `engines/lemonade` answers + // `"unknown"` until its own inference probe has come back, which is + // a real window right after a launch reports ready. The wait is + // bounded (`engine_ipc::SERVE_DEVICE_PROBE_BUDGET`) and `None` is a + // supported outcome — the summary then has no `device:` row, and + // `rocm services list` shows it once the engine can say. let device = (report.status == "ready") - .then(|| fetch_service_device(&paths, &selected_engine, &report.service_id)) + .then(|| fetch_service_device_briefly(&paths, &selected_engine, &report.service_id)) .flatten(); let mut notes = collect_serve_notes( diff --git a/apps/rocm/src/serve_summary.rs b/apps/rocm/src/serve_summary.rs index e0ee77df4d..3424650ff4 100644 --- a/apps/rocm/src/serve_summary.rs +++ b/apps/rocm/src/serve_summary.rs @@ -56,8 +56,19 @@ pub(crate) struct DeploymentSummary { /// `"ready"`, `"starting"`, or an existing service's status. pub status: String, /// The backend/device the engine actually selected (e.g. `"vulkan gpu 0"`, - /// `"rocm gpu 0"`), read from the engine's own `Healthcheck` response. - /// `None` when the query failed or the service was not ready to ask. + /// `"rocm gpu 0 (AMD Instinct MI300X)"`), read from the engine's own + /// `Healthcheck` response. + /// + /// **`None` is a supported outcome, and renders no `device:` row at all.** + /// A launch is `ready` once its server answers, but an engine can still be + /// unable to say which device it is on: `engines/lemonade` reports + /// `"unknown"` until its own inference probe returns, which for a model + /// still loading its weights can be minutes away. `rocm serve` retries + /// within `engine_ipc::SERVE_DEVICE_PROBE_BUDGET` and then prints the + /// summary without the row rather than making the user wait for it; + /// `rocm services list` shows it as soon as the engine can answer. So + /// neither a caller nor a test may read a missing row as a failure to + /// select a device. pub device: Option, /// True when an equivalent server was already running and nothing was spawned. pub already_running: bool, diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index 1a2eab0f4a..b02f8193ae 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -1812,6 +1812,29 @@ impl WindowsExamineInventory { .is_some_and(|d| d.driver_version.is_none() || d.driver_date.is_none()) } + /// How many of the listed adapters are recognizably AMD GPUs. + /// + /// Used only by [`pinned_gpu_name`], to answer "could the preferred + /// adapter's name belong to a card other than the pinned one?". Counts an + /// adapter when either half of [`Self::preferred_amd_display`]'s + /// recognition applies (an AMD PCI device id, or an AMD marketing name) — + /// deliberately *not* that function's last-resort "any adapter with a + /// name", which matches non-AMD displays and would read a laptop's Intel + /// panel as a second AMD GPU and drop a name that was fine. + fn amd_display_count(&self) -> usize { + self.displays + .iter() + .filter(|display| { + display + .pnp_device_id + .as_deref() + .and_then(amd_pci_device_id_from_pnp_id) + .is_some() + || gfx_target_from_amd_marketing_name(&display.name).is_some() + }) + .count() + } + fn amd_display_name(&self) -> Option { self.preferred_amd_display() .map(|display| display.name.trim()) @@ -1977,11 +2000,28 @@ impl ExamineSummary { // The covering family is the fact; it is already printed on its own // line above, so this one says what it means rather than repeating // it. - GpuSupportLevel::RocmHip => format!( - " rocm_support: GPU detected ({}) with full ROCm/HIP support; \ - install the SDK for it with `rocm install sdk`\n", - detected.unwrap_or("") - ), + // + // The advice is conditional because AGENTS.md wants a named command + // to be able to clear the condition it is printed for. On a host + // that already has a managed SDK, `rocm install sdk` clears + // nothing — it is the install that is already done — so the line + // says so and points at what is left to do instead. + // `detected_therock_family` is the installed managed runtime (see + // `detect_managed_therock_family`), as opposed to + // `compatible_therock_family`, which is what this GPU *could* run. + GpuSupportLevel::RocmHip => match self.detected_therock_family.as_deref() { + Some(installed) => format!( + " rocm_support: GPU detected ({}) with full ROCm/HIP support, and the \ + managed ROCm SDK for {installed} is already installed - serve on it with \ + `rocm serve `\n", + detected.unwrap_or("") + ), + None => format!( + " rocm_support: GPU detected ({}) with full ROCm/HIP support; \ + install the SDK for it with `rocm install sdk`\n", + detected.unwrap_or("") + ), + }, // Not folded in with `VulkanOnly`: a detection gap is not a // confirmed absence of ROCm support, and saying "Vulkan" here would // be the same false claim in the other direction. What the user can @@ -3365,6 +3405,116 @@ pub fn detect_host_gpu_summary(paths: Option<&AppPaths>) -> HostGpuSummary { detect_host_gpu_summary_fast(paths) } +/// The GPU name an engine may record at launch for a service pinned to +/// `gpu_indices`, for the `device:` row `rocm serve` and `rocm services list` +/// print (ROCMAI-550). +/// +/// `None` whenever the name cannot be attributed to the *pinned* device, which +/// is most of what this function is for. Every name probe on this host — +/// [`detect_host_gpu_summary`] on Linux, the Windows display inventory — answers +/// for the host's primary/preferred adapter, not for an ordinal. On a multi-GPU +/// host that would print `gpu 1 ()`: a row that looks specific +/// and is wrong, which is worse than the bare `gpu 1` it replaced. So the name +/// is reported only when the host has exactly **one** AMD GPU, where "the +/// primary one" and "the pinned one" cannot differ. +/// +/// The three built-in engines all call this instead of probing themselves, so +/// the rule lives in one place and `engines/aim` (which records no ordinal at +/// all) is held to it as well. +/// +/// Why not map the ordinal to a device properly: the ordinals in `gpu_indices` +/// are HIP ordinals *after* the visibility mask +/// ([`usable_amd_gpu_indices`]), so ordinal 0 under `ROCR_VISIBLE_DEVICES=1` +/// is the host's second card. Attributing a name would mean enumerating +/// devices in HIP order and re-applying that mask, and neither KFD topology +/// nodes nor DRM cards carry a marketing name — so it is a real probe to build +/// and one no lane here can verify (none of the CI hosts has two AMD GPUs). +/// Until that exists, a mask in effect is treated exactly like a second GPU: +/// no name. +#[must_use] +pub fn pinned_gpu_name(gpu_indices: &[u32]) -> Option { + gpu_name_for_single_gpu_host( + gpu_indices, + amd_gpu_count_for_naming(), + gpu_visibility_mask_in_effect(), + host_primary_gpu_name, + ) +} + +/// The decision [`pinned_gpu_name`] makes, with the host probes lifted out. +/// +/// Split out because every input here is a host fact a unit test cannot plant +/// (a second GPU, a Windows display inventory), while the rule itself — "a name +/// only when nothing else on this host could be the one that was pinned" — is +/// the whole point and is exactly what a regression would silently undo. +fn gpu_name_for_single_gpu_host( + gpu_indices: &[u32], + present_amd_gpus: Option, + mask_in_effect: bool, + name: F, +) -> Option +where + F: FnOnce() -> Option, +{ + // A pinned ordinal other than 0 cannot be the only GPU on the host, so it + // is already outside what the probes can attribute. Checked separately from + // the count so a disagreement between the two (a `--gpu 1` that got past + // validation on a host now reporting one GPU) also declines to name. + if gpu_indices.iter().any(|&index| index != 0) { + return None; + } + if mask_in_effect || present_amd_gpus != Some(1) { + return None; + } + name() +} + +/// How many AMD GPUs this host presents, for [`pinned_gpu_name`]'s +/// "could this name belong to another card?" test. `None` when it cannot be +/// established, which declines the name rather than guessing. +fn amd_gpu_count_for_naming() -> Option { + #[cfg(target_os = "linux")] + { + if !runtime_is_windows() { + // WSL2 exposes no per-device topology; the ROCm-over-dxg path + // reaches a single device, which is what `usable_amd_gpu_indices` + // reports there too. + if is_wsl_host() { + return Some(1); + } + return combine_amd_gpu_counts( + linux_kfd_gpu_node_count(), + linux_drm_amdgpu_card_count(), + ); + } + } + detect_windows_examine_inventory().map(|inventory| inventory.amd_display_count()) +} + +/// The host's primary/preferred AMD GPU name — correct only on a host with one +/// AMD GPU, which is why [`pinned_gpu_name`] is the only caller. +fn host_primary_gpu_name() -> Option { + if runtime_is_windows() { + return detect_windows_examine_inventory().and_then(|i| i.amd_display_name()); + } + detect_host_gpu_summary(None).name +} + +/// Whether a GPU visibility mask is narrowing what the engine can see, so a +/// HIP ordinal no longer indexes the host's own device order. +#[cfg(target_os = "linux")] +fn gpu_visibility_mask_in_effect() -> bool { + visibility_mask_from_env().is_some() +} + +/// Platform stub: the mask readers are Linux-only (see +/// [`visibility_mask_from_env`]), so nothing here can observe a mask. A mask +/// reader for another platform must extend this rather than leave it `false`. +#[cfg(not(target_os = "linux"))] +const fn gpu_visibility_mask_in_effect() -> bool { + false +} + #[cfg(windows)] fn detect_host_gpu_summary_fast(_paths: Option<&AppPaths>) -> HostGpuSummary { let windows_inventory = detect_windows_examine_inventory(); @@ -3949,6 +4099,99 @@ mod tests { ); } + /// The one host shape where the probes can attribute a name to the pinned + /// device: a single AMD GPU, unmasked, pinned to ordinal 0 (or to nothing, + /// which is how `engines/aim` records every launch). + #[test] + fn a_single_unmasked_gpu_is_named() { + for indices in [&[][..], &[0][..]] { + assert_eq!( + gpu_name_for_single_gpu_host(indices, Some(1), false, || Some( + "AMD Radeon Graphics".to_owned() + )), + Some("AMD Radeon Graphics".to_owned()) + ); + } + } + + /// The failure this exists to prevent: on a two-GPU host the name probe + /// answers for the primary adapter whatever ordinal was pinned, so + /// `gpu 1 ()` would be a specific-looking lie. The probe + /// must not even be called. + #[test] + fn a_second_gpu_on_the_host_withholds_the_name() { + let mut probed = false; + let name = gpu_name_for_single_gpu_host(&[1], Some(2), false, || { + probed = true; + Some("AMD Instinct MI300X".to_owned()) + }); + assert_eq!(name, None); + assert!(!probed, "a name that cannot be attributed must not be probed"); + + // Ordinal 0 on that same host is no safer: the probe answers for the + // host's primary adapter, which the mask/enumeration order need not + // agree with. + assert_eq!( + gpu_name_for_single_gpu_host(&[0], Some(2), false, || Some("AMD".to_owned())), + None + ); + } + + /// A visibility mask renumbers the devices, so ordinal 0 need not be the + /// host's first GPU and a one-device *view* is not a one-GPU host. + #[test] + fn a_visibility_mask_withholds_the_name() { + assert_eq!( + gpu_name_for_single_gpu_host(&[0], Some(1), true, || Some("AMD".to_owned())), + None + ); + } + + /// An unprobeable host count declines rather than guessing, so a platform + /// whose GPU count cannot be read reports the backend without a name. + #[test] + fn an_unknown_gpu_count_withholds_the_name() { + assert_eq!( + gpu_name_for_single_gpu_host(&[0], None, false, || Some("AMD".to_owned())), + None + ); + // And a host with no AMD GPU at all has nothing to name. + assert_eq!( + gpu_name_for_single_gpu_host(&[], Some(0), false, || Some("AMD".to_owned())), + None + ); + } + + /// Only AMD adapters count toward "is this host ambiguous?" — a laptop's + /// Intel panel alongside one Radeon is still one AMD GPU, and counting it + /// would drop a name that was correct. + #[test] + fn the_windows_display_count_counts_amd_adapters_only() { + let inventory = WindowsExamineInventory { + cpu_model: None, + system_ram_gib: None, + displays: vec![ + WindowsDisplayAdapter { + name: "Intel(R) UHD Graphics".to_owned(), + driver_version: None, + driver_date: None, + pnp_device_id: Some("PCI\\VEN_8086&DEV_9A49".to_owned()), + }, + WindowsDisplayAdapter { + name: "AMD Radeon(TM) Graphics".to_owned(), + driver_version: None, + driver_date: None, + pnp_device_id: Some("PCI\\VEN_1002&DEV_15BF".to_owned()), + }, + ], + }; + assert_eq!(inventory.amd_display_count(), 1); + assert_eq!( + inventory.amd_display_name(), + Some("AMD Radeon(TM) Graphics".to_owned()) + ); + } + /// The predicate must agree with `normalize_therock_family`, not merely /// happen to. /// @@ -5227,6 +5470,25 @@ Class Name: Display ); assert!(rendered.contains("rocm install sdk"), "{rendered}"); + // Same GPU, but the SDK it was being told to install is already there: + // the advice has to be something that is still left to do (AGENTS.md + // wants a named command to be able to clear the condition). + let mut installed = supported; + installed.detected_therock_family = Some("gfx110X-dgpu".to_owned()); + let rendered = installed.render_text(); + assert!( + rendered.contains("rocm_support: GPU detected (gfx1103) with full ROCm/HIP support"), + "{rendered}" + ); + assert!( + rendered.contains("managed ROCm SDK for gfx110X-dgpu is already installed"), + "{rendered}" + ); + assert!( + !rendered.contains("rocm install sdk"), + "an install that is already done must not be the advice: {rendered}" + ); + // No GPU found: the level is reported as unknown, and nothing about the // hardware is asserted in either direction. let missing = examine_summary_with_gpu(None, None); diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 435af13729..8b49d19273 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -289,9 +289,15 @@ Expected result: reports it in the same ` [gpu ] [()]` shape — so expect something like `rocm gpu 0 (AMD Instinct MI300X)`, with the ordinal omitted when the launch pinned none (no `--gpu`, nothing recorded in the engine's - `gpu_indices`) and the name omitted when the host probe could not read one. - `rocm services` shows the same `device` while it runs. The first word is the - half that decides this check. + `gpu_indices`) and the name omitted when the host probe could not read one or + the machine has more than one AMD GPU (where naming one could name the wrong + one). `rocm services` shows the same `device` while it runs. The first word is + the half that decides this check. +- The whole `device:` row may be missing from the deployment summary, and that + is not a failure of this check: the engine reports no device until it has + verified it can serve, which a model still loading its weights has not. Run + `rocm services list` and read the row there instead — the check is on what the + row says, not on which surface said it first. - The run does not fall back to CPU or Vulkan — a `device:` whose first word is `vulkan` fails this check even though the serve itself succeeds. - If the GPU cannot be used, the command fails with a clear error. diff --git a/docs/testing.md b/docs/testing.md index 1892bf2482..b5d15ef3c5 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2077,6 +2077,52 @@ hardware: helper, not the call site, and the four no-GPU scenarios above run where the gate opens either way. +## The Reported Backend Device And Support Level + +ROCMAI-550 made two surfaces report what this host can actually do instead of a +static ROCm claim. Both are worth knowing about before reading a failure in +them. + +**`rocm serve` and `rocm services list` print a `device:` row** — the backend +and GPU the engine says it selected, read back over the engine's own +`Healthcheck` RPC rather than inferred from the launch plan. Every engine +renders it through `rocm_engine_protocol::format_reported_device`, so the shape +is one and the same everywhere: a backend word, then an optional ` gpu `, +then an optional ` ()`. + +**The row is optional, and a missing one is not a defect.** An engine answers +`unknown` until it can really say — `engines/lemonade` until its own inference +probe returns, which for a model still loading its weights is minutes rather +than seconds — and the CLI drops `unknown` rather than printing it. `rocm serve` +retries within `engine_ipc::SERVE_DEVICE_PROBE_BUDGET` (~6s, plus one in-flight +probe) and then prints the summary without the row; `rocm services list` shows +it once the engine can answer. So: + +- a test may assert that a row, where present, is well-formed and that the two + surfaces agree — which is what `the services list reports the engine's actual + backend device` (serve-05, serve-07) does; +- a test may **not** assert that a `ready` serve printed one. + +The GPU name is optional in the same way, and deliberately so: +`rocm_core::pinned_gpu_name` reports a name only on a host with exactly one AMD +GPU and no visibility mask, because every name probe here answers for the +host's primary adapter rather than for a HIP ordinal. On a multi-GPU host the +row is the backend and ordinal with no name — correct and unspecific, rather +than specific and wrong. No CI lane has two AMD GPUs, so that branch is covered +by unit tests in [host_gpu.rs](../crates/rocm-core/src/host_gpu.rs) only. + +**`rocm examine` prints a `rocm_support` line on every host**, at one of three +levels (`gpu_support_level`): ROCm/HIP-supported, Vulkan-only, or unknown. +Unknown covers both "nothing detected" and "a target this build's catalog does +not recognise" — conflating the latter with "confirmed to have no ROCm build" +would turn a detection gap into a claim about the hardware. `examine-31` holds +the line against the `detected_gfx_target` and `compatible_therock_family` the +same report prints, so every lane checks the level its own hardware implies: + +```bash +cargo xtask e2e -- -n '(examine-31)' +``` + ## Windows Tool Notes The TheRock SDK wheel install path should not require users to install global diff --git a/engines/aim/src/state.rs b/engines/aim/src/state.rs index 7756e40eb0..4437ddfa2f 100644 --- a/engines/aim/src/state.rs +++ b/engines/aim/src/state.rs @@ -157,8 +157,11 @@ pub(crate) fn write_running_state( // so without it its row would be the bare word `rocm` on every // host. Captured once here rather than probed per healthcheck: // `rocm services list` healthchecks every ready service on its - // render path. - "gpu_name": rocm_core::detect_host_gpu_summary(None).name, + // render path. The empty ordinal list is the literal truth — the + // container is handed a device list, not an ordinal — and + // `rocm_core::pinned_gpu_name` still declines on any host where + // more than one AMD GPU could be the one that answered. + "gpu_name": rocm_core::pinned_gpu_name(&[]), "engine_recipe": engine_recipe, "started_at_unix_ms": current_unix_millis(), }), diff --git a/engines/lemonade/src/state.rs b/engines/lemonade/src/state.rs index 4f7e61f5b1..ec8410ca9f 100644 --- a/engines/lemonade/src/state.rs +++ b/engines/lemonade/src/state.rs @@ -71,13 +71,16 @@ pub(crate) fn write_running_state( "endpoint_url": endpoint_url(&request.host, request.port), "device_policy": device_policy_name(&request.device_policy), "gpu_indices": request.gpu_indices, - // The host's GPU name, so `healthcheck_service` can name a device - // and not just a backend (ROCMAI-550). Captured here, once, rather - // than probed per healthcheck: `rocm services list` healthchecks - // every ready service on its render path. Recorded even when it is - // `null`, so the key's absence means an older state file rather - // than a probe that came back empty. - "gpu_name": rocm_core::detect_host_gpu_summary(None).name, + // The name of the GPU this service was pinned to, so + // `healthcheck_service` can name a device and not just a backend + // (ROCMAI-550). Captured here, once, rather than probed per + // healthcheck: `rocm services list` healthchecks every ready + // service on its render path. `rocm_core::pinned_gpu_name` answers + // `null` whenever the host probe cannot be attributed to the pinned + // ordinal (a second AMD GPU on the host, a visibility mask); the + // key is recorded either way, so its absence means an older state + // file rather than a probe that declined. + "gpu_name": rocm_core::pinned_gpu_name(&request.gpu_indices), "runtime_id": request.runtime_id.as_deref().unwrap_or(runtime.manifest.env_id.as_str()), "env_id": request.env_id.as_deref().unwrap_or(runtime.manifest.env_id.as_str()), "runtime_kind": "lemonade_embeddable", diff --git a/engines/vllm/src/state.rs b/engines/vllm/src/state.rs index bcb2c447d9..355f7d3345 100644 --- a/engines/vllm/src/state.rs +++ b/engines/vllm/src/state.rs @@ -184,11 +184,14 @@ pub(crate) fn write_running_state( // reports. Introspection only, like the rest of this file — never // read back to relaunch. "gpu_indices": request.gpu_indices, - // The host's GPU name, so the `device:` row names a device and not - // only a backend (ROCMAI-550). Captured once here rather than - // probed per healthcheck, because `rocm services list` - // healthchecks every ready service on its render path. - "gpu_name": rocm_core::detect_host_gpu_summary(None).name, + // The name of the GPU this service was pinned to, so the `device:` + // row names a device and not only a backend (ROCMAI-550). Captured + // once here rather than probed per healthcheck, because `rocm + // services list` healthchecks every ready service on its render + // path. `rocm_core::pinned_gpu_name` answers `null` when the host + // probe cannot be attributed to the pinned ordinal, so the row + // falls back to `rocm gpu ` rather than naming another card. + "gpu_name": rocm_core::pinned_gpu_name(&request.gpu_indices), "runtime_id": runtime.runtime_id, "requested_runtime_id": request.runtime_id, "env_id": request.env_id.as_deref().unwrap_or(runtime.env_id.as_str()), diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index 34c732c3cc..8566f2c9c1 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -464,3 +464,24 @@ Feature: GPU detection and system inspection Given a machine with a ROCm install that was not set up by the CLI When the user inspects the system in machine-readable form Then the inspection reports a current install-method name + + # ROCMAI-550. `rocm examine` answers "can this machine run ROCm?" with a + # `rocm_support` line on EVERY host -- the ROCm-capable runners and the no-GPU + # mock lane included, not only the Vulkan-only hosts the ticket started from. + # Other surfaces send people here for that answer (`lemonade_engine_note` in + # apps/rocm/src/main.rs tells an unclassifiable host to run `rocm examine`), so + # silence there is a dead end rather than a smaller report. + # + # Deliberately untagged: the line's content is keyed to what this host turns + # out to be, so the lanes disagreeing about their GPUs is the coverage rather + # than an obstacle to it. The assertion is a cross-check and not a tautology -- + # the claim is held against `detected_gfx_target` and + # `compatible_therock_family` from the same report, so a line reading "full + # ROCm/HIP support" on the mock lane, or "not supported by ROCm" on an MI300X, + # fails here. + @id:examine-reports-this-hosts-rocm-support-level + Scenario: examine-31 - System inspection says whether this machine's GPU is supported by ROCm + When the user inspects the system + Then the inspection states this host's ROCm support level + And the support level agrees with the GPU the inspection detected + And the support level leaves a command that fits this host diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 67898e4079..95dd0831b9 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -2062,3 +2062,128 @@ async fn assert_examine_driver_age_follows_the_date(world: &mut E2eWorld) { "{age}" ); } + +/// The `rocm_support` line's value, or `None` when the report has no such line. +fn rocm_support_line(output: &str) -> Option<&str> { + field_value(output, "rocm_support") +} + +/// What `rocm examine` reported as the detected GPU target and the ROCm package +/// family covering it, with the report's `` placeholder read back as +/// "absent" so the two can be reasoned about as the CLI's own +/// `gpu_support_level` inputs. +fn detected_gpu_facts(output: &str) -> (Option<&str>, Option<&str>) { + let field = |name| field_value(output, name).filter(|value| *value != ""); + ( + field("detected_gfx_target"), + field("compatible_therock_family"), + ) +} + +/// ROCMAI-550: the report answers the support question on every host, including +/// the ones where the honest answer is "this build cannot tell". +#[then("the inspection states this host's ROCm support level")] +async fn assert_examine_states_a_support_level(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + let line = rocm_support_line(output).unwrap_or_else(|| { + panic!( + "`rocm examine` printed no `rocm_support` line. Other surfaces send \ + the user here to find out whether this GPU is supported, so there \ + is no host on which the answer may be silence:\n{output}" + ) + }); + assert!( + !line.is_empty(), + "the `rocm_support` line states nothing:\n{output}" + ); +} + +/// The cross-check. The claim is judged against the two facts the same report +/// prints, not against a hardcoded expectation, so this says something on every +/// lane: the mock lane detects no GPU and must not be told it has ROCm support; +/// the GPU runners have a covering package family and must not be told their +/// card has no ROCm build. +#[then("the support level agrees with the GPU the inspection detected")] +async fn assert_support_level_agrees_with_detection(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + let line = rocm_support_line(output).expect("no rocm_support line"); + let (detected, family) = detected_gpu_facts(output); + + match (detected, family) { + // A package family covers this GPU: full ROCm/HIP support, and the + // report must not hedge it into either of the weaker answers. + (_, Some(_)) => { + assert!( + line.contains("full ROCm/HIP support"), + "a GPU with a covering ROCm package family is supported, and the \ + support line must say so: {line}\n{output}" + ); + assert!( + !line.contains("not supported by ROCm") && !line.starts_with("unknown"), + "{line}\n{output}" + ); + } + // Nothing detected at all. The honest answer is "unknown"; claiming + // either support or a working Vulkan backend would be inventing + // hardware. + (None, None) => { + assert!( + line.starts_with("unknown"), + "no GPU was detected, so neither support nor its absence can be \ + claimed: {line}\n{output}" + ); + assert!( + !line.contains("Vulkan backend can still be used"), + "no backend may be promised for a GPU that was not found: \ + {line}\n{output}" + ); + } + // Detected, with no family covering it. Both answers are legitimate + // here and which one is correct depends on the catalog: `gfx90c` is + // *known* to have no ROCm build (Vulkan-only), anything unrecognised is + // a gap in what this build knows (unknown). What is never legitimate is + // claiming ROCm support for a GPU no package family covers. + (Some(gfx), None) => { + assert!( + line.contains(gfx), + "the support line must name the target it is judging: {line}\n{output}" + ); + assert!( + line.contains("not supported by ROCm") || line.contains("unknown"), + "no ROCm package family covers {gfx}, so the line must report \ + either a Vulkan-only GPU or an unknown one: {line}\n{output}" + ); + assert!( + !line.contains("full ROCm/HIP support"), + "a GPU with no covering package family must not be reported as \ + ROCm-supported: {line}\n{output}" + ); + } + } +} + +/// AGENTS.md wants a line that tells the user what they can do, and a named +/// command that can actually clear the condition it was printed for. The +/// specific pairing checked here is the one this round fixed: a host whose +/// managed SDK is already installed must not be told to install it. +#[then("the support level leaves a command that fits this host")] +async fn assert_support_level_advice_fits(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + let line = rocm_support_line(output).expect("no rocm_support line"); + assert!( + line.contains("rocm "), + "the support line must leave the user somewhere to go: {line}\n{output}" + ); + + // `detected_therock_family` is the managed SDK that is installed, as + // opposed to `compatible_therock_family`, which is what this GPU could run. + let sdk_installed = field_value(output, "detected_therock_family") + .is_some_and(|value| value != "" && value != ""); + if sdk_installed { + assert!( + !line.contains("rocm install sdk"), + "this host already has a managed ROCm SDK, so installing one clears \ + nothing: {line}\n{output}" + ); + } +} diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index c8c3ce1472..113958ea87 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1916,8 +1916,9 @@ fn device_row_value(text: &str) -> Option<&str> { /// Lemonade has a Vulkan path, so `vulkan` from `vllm` or `aim` is wrong, /// and `unknown` is wrong from any of them; /// 3. it equals what the `rocm serve` deployment summary printed for the same -/// service, which is the row the preceding Given produced and which nothing -/// else in these scenarios looks at. +/// service — *when* that summary printed a device row at all, which is a +/// documented maybe rather than a promise (see the comment at that +/// comparison, and `serve_summary::DeploymentSummary::device`). /// The row is polled rather than read once: see [`DEVICE_ROW_DEADLINE`] for /// the window in which a correct implementation legitimately has no row yet. /// Running out of that window is an assertion naming what was last seen, not a @@ -1986,30 +1987,36 @@ async fn assert_services_list_reports_device(world: &mut E2eWorld) { // at launch; nothing else in these scenarios inspects that summary, so // without this the summary half of ROCMAI-550 has no live assertion at all. // - // The summary row is conditional by design: `serve` only asks the engine - // once the launch already reported `ready`, and a managed launch can return - // while the weights are still loading (which is why the Given then polls - // `/v1/models`). So the assertion is two-sided rather than unconditional - - // a device row must agree with this one, and its absence must be explained - // by a not-yet-ready launch rather than by the row having been dropped. + // Only when the summary HAS the row. Its absence is not a defect and must + // not be asserted against, in either direction: + // + // * `serve` asks the engine once the launch reports `ready` and retries + // only within `engine_ipc::SERVE_DEVICE_PROBE_BUDGET` (~6s), because the + // row is informational and the alternative is holding a terminal; + // * `engines/lemonade` answers `"unknown"` until its own + // `inference_verified` probe returns, which for a model still loading can + // be far longer than that - which is exactly why the Given step then + // polls `/v1/models` for up to minutes; + // * so a correct `rocm serve` can print `status ready` with no `device:` + // row, while `rocm services list` - asked later, after this step has + // waited - has one. + // + // An earlier revision asserted that a `ready` summary must carry the row. + // That is the exact false failure this note describes, and it was invisible + // only because serve-07 is xfail'd on lemonade+linux (see the comment on + // that scenario in model_serving.feature). What remains asserted is the + // half that is a real invariant: the two surfaces read one engine over one + // RPC, so where both speak they must say the same thing. let summary = world .serve_summary .as_deref() .expect("the Given step must have recorded the serve summary"); - match device_row_value(summary) { - Some(summary_device) => assert_eq!( + if let Some(summary_device) = device_row_value(summary) { + assert_eq!( summary_device, device, "`rocm serve` and `rocm services list` must report the same device \ for one service; they read it from the same Healthcheck RPC" - ), - None => assert!( - !summary - .lines() - .any(|line| line.trim_start().starts_with("status") - && line.trim_end().ends_with("ready")), - "the serve summary reported `ready` but printed no `device:` row, so \ - the row was dropped rather than deferred:\n{summary}" - ), + ); } } From c6e62182c3db104b92f78aa663cf0cfe151e14e8 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 6 Nov 2026 09:06:45 +0200 Subject: [PATCH 11/59] fix(examine): point the unknown-tier advice at a surface that shows the row Review round on #678. The note promised that `rocm serve --engine lemonade` reports the backend on its `device:` row, while this PR's own docs say that row can be absent: the serve summary reads the device once, right after `ready`, and lemonade answers `unknown` until its own inference probe returns. The advice now names `rocm services list`, which shows the row once it is known, so the command can actually clear the condition it is printed for. The reasoning is recorded beside the arm rather than left implied. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/host_gpu/render.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index d9e4ff32c3..476cca612e 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -69,6 +69,14 @@ impl ExamineSummary { detected.unwrap_or("") ), }, + // The command names `rocm services list` rather than the serve + // summary on purpose. The summary reads the device once, right + // after `ready`, and lemonade answers `unknown` until its own + // inference probe returns — so the row it prints is legitimately + // absent for a short window. Pointing at the surface that shows it + // *once known* keeps the advice able to clear the condition it is + // printed for, which naming the summary would not. + // // Not folded in with `VulkanOnly`: a detection gap is not a // confirmed absence of ROCm support, and saying "Vulkan" here would // be the same false claim in the other direction. What the user can @@ -77,8 +85,9 @@ impl ExamineSummary { // thing on this host that can actually settle the question. GpuSupportLevel::Unknown => format!( " rocm_support: unknown - {}, so this build can confirm neither \ - ROCm support nor its absence; `rocm serve --engine \ - lemonade` reports the backend it picked on its `device:` row\n", + ROCm support nor its absence; serve a model with `rocm serve \ + --engine lemonade` and read the backend it picked from \ + the `device:` row of `rocm services list`\n", match detected { Some(gfx) => format!("GPU target {gfx} is not in this build's catalog"), None => "no AMD GPU was detected".to_owned(), From 767c58674f9e740e8867e6ffa27f2a811adc64fc Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Mon, 9 Nov 2026 12:01:32 +0200 Subject: [PATCH 12/59] fix(examine): stop sending a GPU-less host to a command serve would refuse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round on #678. The `Unknown` tier covers two different hosts and was giving both the same advice. A detected-but-unrecognised gfx target has a GPU, so `rocm serve` runs and the engine's own `device:` row settles what this catalog could not. A host with nothing detected is refused outright by the no-usable-GPU pre-flight under the default `gpu_required` policy, so naming that command was advice that could not clear the condition it was printed for (AGENTS.md section 3). The no-detection arm now points at `rocm diagnose`, whose `fix-3-rocm-kernel`, `fix-4-render-group` and `fix-5-amdgpu-load` entries are exactly the reasons a present GPU goes undetected. The sibling arm keeps the serve advice. `no_gpu_advice_names_a_command_that_host_can_run` asserts the behaviour rather than the wording, as asked: it pins the tier through `gpu_support_level` and then checks the rendered line does not name `rocm serve`. Falsified by reverting the arm — the test fails with the offending line quoted. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/host_gpu/render.rs | 118 ++++++++++++++++++++---- 1 file changed, 101 insertions(+), 17 deletions(-) diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index 476cca612e..ec2dedb9c2 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -83,16 +83,32 @@ impl ExamineSummary { // do is let an engine decide and then read back what it chose - // which is the `device:` row this same ticket added, and the only // thing on this host that can actually settle the question. - GpuSupportLevel::Unknown => format!( - " rocm_support: unknown - {}, so this build can confirm neither \ - ROCm support nor its absence; serve a model with `rocm serve \ - --engine lemonade` and read the backend it picked from \ - the `device:` row of `rocm services list`\n", - match detected { - Some(gfx) => format!("GPU target {gfx} is not in this build's catalog"), - None => "no AMD GPU was detected".to_owned(), - } - ), + // Two different hosts reach `Unknown`, and they need different + // advice because only one of them can serve anything. + // + // A detected-but-unrecognised target means a GPU *is* present, so + // `rocm serve` runs and the engine's own `device:` row settles what + // this catalog could not. With nothing detected, that same command + // is refused outright by the no-usable-GPU pre-flight under the + // default `gpu_required` policy — naming it would be advice that + // cannot clear the condition it is printed for (AGENTS.md section 3). + // `rocm diagnose` is the command that can: its `fix-3-rocm-kernel`, + // `fix-4-render-group` and `fix-5-amdgpu-load` entries are exactly + // the reasons a present GPU goes undetected. + GpuSupportLevel::Unknown => match detected { + Some(gfx) => format!( + " rocm_support: unknown - GPU target {gfx} is not in this \ + build's catalog, so this build can confirm neither ROCm \ + support nor its absence; serve a model with `rocm serve \ + --engine lemonade` and read the backend it picked \ + from the `device:` row of `rocm services list`\n" + ), + None => " rocm_support: unknown - no AMD GPU was detected, so \ + this build can confirm neither ROCm support nor its \ + absence; run `rocm diagnose` to find out why no GPU is \ + visible\n" + .to_owned(), + }, } } @@ -502,7 +518,18 @@ mod tests { #[test] fn examine_answers_the_support_question_the_unknown_engine_note_sends_users_here_for() { - for detected in [None, Some("gfx803")] { + // Both `Unknown` sub-cases must answer, and both must leave somewhere + // to go — but not the same somewhere. A detected-but-unrecognised + // target has a GPU, so `rocm serve` runs and its `device:` row settles + // the backend. With nothing detected that command is refused by the + // no-usable-GPU pre-flight, so the only honest next step is finding out + // why the GPU is invisible. See + // `no_gpu_advice_names_a_command_that_host_can_run`, which pins that + // distinction against the behaviour rather than the wording. + for (detected, expected_command) in [ + (None, "rocm diagnose"), + (Some("gfx803"), "rocm serve --engine lemonade"), + ] { let rendered = examine_summary_with_gpu(detected, None).render_text(); let line = rendered .lines() @@ -516,14 +543,71 @@ mod tests { // Says which level it is, and leaves a command that can settle it. assert!(line.contains("unknown"), "{line}"); assert!( - line.contains("rocm serve --engine lemonade"), - "an `unknown` answer still has to leave somewhere to go: {line}" - ); - assert!( - line.contains("device:"), - "and name where the real backend shows up: {line}" + line.contains(expected_command), + "an `unknown` answer still has to leave somewhere to go, and it \ + has to be somewhere this host can actually go: {line}" ); } + + // The detected sub-case also names where the backend shows up; the + // undetected one has no server to read it from. + let unrecognised = examine_summary_with_gpu(Some("gfx803"), None).render_text(); + assert!( + unrecognised.contains("device:"), + "a host that can serve must be told where the real backend shows up: \ + {unrecognised}" + ); + } + + /// The advice a host is given must name a command that host can actually + /// run — asserted against the serve pre-flight's own predicate, not against + /// the wording. + /// + /// `rocm serve` refuses on a host with no usable AMD GPU under the default + /// `gpu_required` policy, so telling an undetected host to serve a model is + /// advice that cannot clear the condition it is printed for. Both `Unknown` + /// sub-cases are driven here because they differ precisely on this: a + /// detected-but-unrecognised target has a GPU and may serve, nothing + /// detected may not. + /// + /// Written against `gpu_support_level` + the rendered line rather than a + /// string match on the sentence: a reworded message that still sent a + /// GPU-less host to `rocm serve` would pass a wording assertion and fail + /// this one. + #[test] + fn no_gpu_advice_names_a_command_that_host_can_run() { + let undetected = examine_summary_with_gpu(None, None).render_text(); + let line = undetected + .lines() + .find(|l| l.trim_start().starts_with("rocm_support:")) + .expect("every host gets a support line"); + assert_eq!( + gpu_support_level(None, None), + GpuSupportLevel::Unknown, + "precondition: nothing detected is the Unknown tier" + ); + assert!( + !line.contains("rocm serve"), + "`rocm serve` is refused on a host with no usable AMD GPU, so it \ + cannot be the next step offered to one: {line}" + ); + assert!( + line.contains("rocm diagnose"), + "dropping the unusable command must still leave a usable one: {line}" + ); + + // The sibling sub-case keeps the serve advice, because that host has a + // GPU and the engine's own device row is what settles the question. + let unrecognised = examine_summary_with_gpu(Some("gfx-not-in-catalog"), None).render_text(); + let line = unrecognised + .lines() + .find(|l| l.trim_start().starts_with("rocm_support:")) + .expect("every host gets a support line"); + assert!( + line.contains("rocm serve"), + "a detected GPU can serve, so the advice that settles its backend \ + must not be withheld: {line}" + ); } #[test] From 50ff66a89644422b7720dab084dbe782eb841cfd Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 10 Nov 2026 09:35:28 +0200 Subject: [PATCH 13/59] test(serve): show the Lemonade advice names a command this host is not refused `rocm install sdk` and `rocm examine` tell a gfx90c APU to run `rocm serve --engine lemonade`, but nothing showed serve's no-usable-GPU pre-flight accepts that host. It does: a zero KFD compute-node count falls back to the DRM amdgpu card count, so the host the refusal was printed on resolves ordinal 0. Pin that composition off one planted /sys, in a unit test and in an e2e scenario that runs the real binary down the route the CLI sends the user. Also close the test gaps review 5826143385 named: the positive `device:` row and the `unknown` filter now have assertions that run in default CI, the endpoint key file's scoping is pinned, and the examine support-line check asserts the specific command each tier owes instead of any `rocm ` word. Windows reads the display inventory once per `pinned_gpu_name` instead of twice, and the docs say the engine *reports* a device rather than naming the one it observed. Signed-off-by: Roman Sirokov --- README.md | 12 +- apps/rocm/src/engine_ipc.rs | 86 +++++++++++- apps/rocm/src/services_text.rs | 129 +++++++++++++++++- apps/rocm/src/therock/family.rs | 13 ++ crates/rocm-core/src/host_gpu.rs | 25 +++- .../rocm-core/src/host_gpu/gpu_visibility.rs | 85 ++++++++++++ crates/rocm-core/src/host_gpu/linux.rs | 36 ++++- crates/rocm-core/src/host_gpu/render.rs | 48 +++++-- docs/manual-testing.md | 3 +- engines/lemonade/src/device_policy.rs | 14 +- .../features/model_serving.feature | 24 ++++ tests/e2e-cucumber/tests/e2e/examine_steps.rs | 59 ++++++-- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 124 +++++++++++++++-- 13 files changed, 594 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index a5a2f518e6..1aff586922 100644 --- a/README.md +++ b/README.md @@ -1024,12 +1024,14 @@ the running service, or stop it first to serve on the new port. By default the server runs in the background under ROCm CLI's supervision and prints a deployment summary: a progress indicator while it starts, then a table -with the status, the engine and the backend device it actually selected (for +with the status, the engine and the backend device the engine reports (for example `rocm gpu 0 (AMD Instinct MI300X)`, or `vulkan (AMD Radeon Graphics)` -where the engine pinned no GPU ordinal — read back from the running engine), the -full inference endpoint, the API-qualified model name, and a quick smoke test -(time to first token and approximate tokens per second). `rocm services` reports -the same `device` for each running service. +where the engine pinned no GPU ordinal). It is read back from the running +engine, so it is that engine's own account of the backend it was launched on +rather than an independent measurement of the hardware. The summary also carries +the full inference endpoint, the API-qualified model name, and a quick smoke +test (time to first token and approximate tokens per second). `rocm services` +reports the same `device` for each running service. The device line is omitted when the engine cannot say yet. A server is reported ready as soon as it answers, which can be before it has finished loading the diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index 1f3876f753..f8603d13ff 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -321,11 +321,23 @@ pub(crate) fn fetch_service_device( endpoint_key_file.as_deref(), ) .ok()?; - // `"unknown"` is the engines' own device value when no state is on disk for - // this service id yet (see `engines/vllm/src/state.rs` and - // `engines/lemonade/src/lib.rs`), which is not more informative than - // showing nothing. - (!response.device.is_empty() && response.device != "unknown").then_some(response.device) + device_worth_reporting(response.device) +} + +/// The engine's reported device, or `None` when it said nothing usable. +/// +/// `"unknown"` is the engines' own device value when no state is on disk for +/// this service id yet (see `engines/vllm/src/state.rs` and +/// `engines/lemonade/src/lib.rs`), and it is not more informative than showing +/// nothing — a `device: unknown` row claims the CLI asked and learned +/// something, which is the opposite of what happened. +/// +/// A named function rather than an inline `then_some` because the only caller +/// is an RPC to a live engine: in default CI that call never succeeds, so the +/// filter is unreachable there and deleting it would be invisible until a +/// freshly-launched Lemonade service printed `device: unknown` on a GPU lane. +fn device_worth_reporting(device: String) -> Option { + (!device.trim().is_empty() && device != "unknown").then_some(device) } /// How long `rocm serve`'s deployment summary may keep asking the engine which @@ -718,6 +730,70 @@ mod tests { use super::*; use crate::tests::test_paths; + /// The `unknown` filter, which nothing else can reach: its caller is an RPC + /// to a live engine, so in default CI it never runs and deleting it stays + /// green. + #[test] + fn a_device_the_engine_does_not_know_is_not_reported_as_one() { + for nothing in ["unknown", "", " "] { + assert_eq!( + device_worth_reporting(nothing.to_owned()), + None, + "`{nothing}` is the engine saying it has no answer yet, and a \ + `device:` row would claim otherwise" + ); + } + for device in ["rocm gpu 0", "vulkan (AMD Radeon Graphics)"] { + assert_eq!( + device_worth_reporting(device.to_owned()), + Some(device.to_owned()) + ); + } + } + + /// The endpoint key file reaches a built-in engine through the process + /// environment, and is gone again afterwards. + /// + /// Both halves matter and neither was covered. Without the variable a + /// `--require-api-key` service's own `/v1/models` probe answers 401, the + /// engine reports the service as unlisted, and the `device:` row silently + /// vanishes for exactly the protected services. Without the restore, one + /// service's key would leak into the next service's probe in the same + /// `rocm services list` — `with_scoped_builtin_engine_env` mutates + /// process-wide state, which is why the probes are serialized in the first + /// place. + #[test] + fn the_endpoint_key_file_is_scoped_to_one_builtin_engine_call() { + let key = rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV; + // SAFETY-adjacent: the scoped setter takes the same process-wide lock + // this test observes through, so no other engine call can interleave. + let before = std::env::var_os(key); + let seen = with_scoped_builtin_engine_env( + None, + None, + Some(Path::new("/tmp/rocm-e2e-endpoint-key")), + || std::env::var_os(key), + ); + assert_eq!( + seen.as_deref(), + Some(std::ffi::OsStr::new("/tmp/rocm-e2e-endpoint-key")), + "the engine must be able to read the key file path it was given" + ); + assert_eq!( + std::env::var_os(key), + before, + "the key must not outlive the call it was scoped to" + ); + + // And nothing is set when no key file was supplied: a loopback service + // has none, and inventing one would point the engine at a file that + // does not exist. + assert_eq!( + with_scoped_builtin_engine_env(None, None, None, || std::env::var_os(key)), + before + ); + } + /// Every service inside the budget is probed and reported. #[test] fn service_device_probes_run_while_the_budget_holds() { diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index cbd64a9cab..e17f4ce77c 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -77,7 +77,7 @@ //! classification respectively — stay in `main.rs`'s shared `mod tests` //! instead. -use std::collections::VecDeque; +use std::collections::{HashMap, VecDeque}; use std::fmt::Write as _; use std::fs; use std::io::{self, BufRead, Read as _, Seek as _, SeekFrom}; @@ -379,6 +379,23 @@ fn write_past_attempts_hint(output: &mut String, past_attempts: usize, newest_id /// bounded and flat in the number of services, which is what makes it /// acceptable there. pub(crate) fn render_services_text(paths: &AppPaths, all: bool) -> Result { + render_services_text_with_devices(paths, all, |ready| fetch_service_devices(paths, ready)) +} + +/// [`render_services_text`] with the live device probe injected. +/// +/// Split out because the probe is the one thing a unit test cannot have: it is +/// a `Healthcheck` RPC to a running engine, so in default CI (no GPU, no +/// servers) it answers nothing and the `device:` row is never rendered at all — +/// deleting the row would stay green everywhere except the GPU lanes, which is +/// exactly the gap this closes. The seam takes only the probe, not the row: +/// which services are offered to it (`status == "ready"`) and where the row +/// lands both stay in the shared body, so a test can pin them. +fn render_services_text_with_devices( + paths: &AppPaths, + all: bool, + probe_devices: impl FnOnce(Vec<(&str, &str)>) -> HashMap, +) -> Result { // Counts describe the WHOLE registry, not the rows this view keeps. The // default view hides records that are not live, and counting the filtered // list made the header say "none ready" on a host whose only local servers @@ -434,12 +451,12 @@ pub(crate) fn render_services_text(paths: &AppPaths, all: bool) -> Result Result<()> { + use std::io::{Read, Write}; + use std::net::TcpListener; + + let (root, paths) = test_paths("services-list-device"); + paths.ensure()?; + let listener = TcpListener::bind(("127.0.0.1", 0))?; + let ready_port = listener.local_addr()?.port(); + let server = thread::spawn(move || -> Result<()> { + for _ in 0..2 { + let (mut stream, _) = listener.accept()?; + stream.set_read_timeout(Some(Duration::from_secs(2)))?; + let mut request = [0_u8; 512]; + let read = stream.read(&mut request)?; + let request = String::from_utf8_lossy(&request[..read]).into_owned(); + let body = if request.starts_with("POST /v1/chat/completions ") { + r#"{"choices":[{"message":{"content":"ok"}}]}"# + } else { + r#"{"data":[{"id":"Qwen/Qwen3.5"}]}"# + }; + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + )?; + } + Ok(()) + }); + let current_pid = std::process::id(); + for (service_id, status, port) in [ + ("svc-ready", "ready", ready_port), + ("svc-starting", "starting", 11438_u16), + ] { + let mut record = ManagedServiceRecord::new( + &paths, + service_id, + "vllm", + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + port, + "managed", + current_pid, + Some("therock-release".to_owned()), + None, + Some("gpu_required".to_owned()), + ); + record.status = status.to_owned(); + record.write()?; + } + + let mut offered: Vec<(String, String)> = Vec::new(); + let rendered = render_services_text_with_devices(&paths, false, |ready| { + offered = ready + .iter() + .map(|(id, engine)| ((*id).to_owned(), (*engine).to_owned())) + .collect(); + HashMap::from([( + "svc-ready".to_owned(), + "rocm gpu 0 (AMD Instinct MI300X)".to_owned(), + )]) + })?; + server + .join() + .expect("fake models server should not panic")?; + let _ = fs::remove_dir_all(root); + + assert_eq!( + offered, + [("svc-ready".to_owned(), "vllm".to_owned())], + "only a service ready to answer may be probed" + ); + assert!( + rendered.contains(" device: rocm gpu 0 (AMD Instinct MI300X)\n"), + "the device the engine reported must reach the list:\n{rendered}" + ); + assert!( + rendered + .contains(" engine: vllm\n device: rocm gpu 0 (AMD Instinct MI300X)\n model: "), + "the device belongs with the engine that selected it, above the \ + model:\n{rendered}" + ); + // The service the probe said nothing about keeps no row of its own. + assert_eq!( + rendered.matches(" device: ").count(), + 1, + "only the service with a reported device gets a row:\n{rendered}" + ); + Ok(()) + } + /// The default view lists only live servers. The counts used to be taken /// from that already-filtered list, so a host whose local servers had all /// failed was told "none ready" and shown an empty list - the records were diff --git a/apps/rocm/src/therock/family.rs b/apps/rocm/src/therock/family.rs index f8659bb67f..efcfaea7ce 100644 --- a/apps/rocm/src/therock/family.rs +++ b/apps/rocm/src/therock/family.rs @@ -186,6 +186,19 @@ pub(super) fn resolve_family( /// on a host with no ROCm, and fail under the no-CPU-fallback policy. Naming /// the engine makes the sentence true whatever the config says, because the /// explicit flag out-ranks every other selection source. +/// +/// **The named command is not refused on the host that is shown it.** `serve`'s +/// no-usable-GPU pre-flight is `host_has_no_usable_gpu` (`serve_cmd.rs`) over +/// `rocm_core::usable_amd_gpu_indices`, and on Linux that composes the KFD +/// compute-node count with the DRM `amdgpu` card count: a *zero* KFD count +/// falls back to DRM rather than refusing. A gfx90c APU whose kernel exposes no +/// compute node can only have been named by the DRM `ip_discovery` read, which +/// walks the same `card*/device` entries that count — so this branch is +/// reachable only on a host with at least one such card, and the pre-flight +/// therefore sees ordinal 0. `rocm_core`'s +/// `a_gfx90c_host_the_kernel_gives_no_compute_node_still_has_a_usable_gpu` +/// drives both halves off one planted `/sys`, and serve-50 runs the real binary +/// against that same host shape on a GPU-less lane. fn unsupported_host_gpu_message(raw_arch: &str) -> String { format!( "no ROCm SDK build exists for the detected GPU target `{raw_arch}`, so \ diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index cb18f3028d..4eaf10c80e 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -449,11 +449,20 @@ pub fn detect_host_gpu_summary(paths: Option<&AppPaths>) -> HostGpuSummary { /// no name. #[must_use] pub fn pinned_gpu_name(gpu_indices: &[u32]) -> Option { + // One Windows display-inventory read per call, shared by the two questions + // that need it. Both the count and the name come from the same `pnputil` / + // CIM query, and asking twice meant two PowerShell/WMI probes on the serve + // launch path. The read is eager rather than lazy only on the platform that + // uses it: off Windows nothing is probed here at all, and on Windows the + // count already needed it before any of the decisions below could run. + let windows_inventory = runtime_is_windows() + .then(detect_windows_examine_inventory) + .flatten(); gpu_name_for_single_gpu_host( gpu_indices, - amd_gpu_count_for_naming(), + amd_gpu_count_for_naming(windows_inventory.as_ref()), gpu_visibility_mask_in_effect(), - host_primary_gpu_name, + || host_primary_gpu_name(windows_inventory.as_ref()), ) } @@ -488,7 +497,11 @@ where /// How many AMD GPUs this host presents, for [`pinned_gpu_name`]'s /// "could this name belong to another card?" test. `None` when it cannot be /// established, which declines the name rather than guessing. -fn amd_gpu_count_for_naming() -> Option { +/// +/// `windows_inventory` is the display listing the caller already read, or +/// `None` off Windows (and when the listing failed); see [`pinned_gpu_name`] +/// for why it is passed in rather than read here. +fn amd_gpu_count_for_naming(windows_inventory: Option<&WindowsExamineInventory>) -> Option { #[cfg(target_os = "linux")] { if !runtime_is_windows() { @@ -504,14 +517,14 @@ fn amd_gpu_count_for_naming() -> Option { ); } } - detect_windows_examine_inventory().map(|inventory| inventory.amd_display_count()) + windows_inventory.map(WindowsExamineInventory::amd_display_count) } /// The host's primary/preferred AMD GPU name — correct only on a host with one /// AMD GPU, which is why [`pinned_gpu_name`] is the only caller. -fn host_primary_gpu_name() -> Option { +fn host_primary_gpu_name(windows_inventory: Option<&WindowsExamineInventory>) -> Option { if runtime_is_windows() { - return detect_windows_examine_inventory().and_then(|i| i.amd_display_name()); + return windows_inventory.and_then(WindowsExamineInventory::amd_display_name); } detect_host_gpu_summary(None).name } diff --git a/crates/rocm-core/src/host_gpu/gpu_visibility.rs b/crates/rocm-core/src/host_gpu/gpu_visibility.rs index 9a7be9a7f9..8d8fdcacf9 100644 --- a/crates/rocm-core/src/host_gpu/gpu_visibility.rs +++ b/crates/rocm-core/src/host_gpu/gpu_visibility.rs @@ -324,6 +324,91 @@ pub(super) fn visibility_mask_in_effect() -> bool { #[cfg(test)] mod tests { use super::*; + use crate::gfx_target::gfx_target_has_no_rocm_build; + use crate::test_support::temp_app_paths; + use std::fs; + + /// ROCMAI-550, and the one claim the Lemonade remediation rests on: the host + /// that gets told `rocm serve --engine lemonade` is a host + /// `rocm serve` will actually launch on. + /// + /// The shape is a Renoir/Cezanne laptop: the Vega iGPU is on the bus and + /// bound to `amdgpu`, but the kernel's KFD topology lists no compute node + /// for it, so `gfx90c` can only have been named by the DRM `ip_discovery` + /// read. `apps/rocm/src/therock/family.rs` prints + /// `unsupported_host_gpu_message` for exactly that target, and + /// `apps/rocm/src/serve_cmd.rs`'s `host_has_no_usable_gpu` — + /// `matches!(visible, Some([]))` over [`usable_amd_gpu_indices`] — decides + /// whether the command it names is refused. Both halves are driven here off + /// **one** planted `/sys`, so this is the composition and not two + /// independent claims: KFD contributes `Some(0)`, DRM contributes `Some(1)`, + /// and [`combine_amd_gpu_counts`]'s zero-KFD fallback is what turns that + /// into a usable ordinal. Delete that fallback and the detection half still + /// passes while the pre-flight half goes red, which is the failure the + /// reviewer asked to see ruled out. + /// + /// Planted rather than run on hardware because no CI lane has a gfx90c APU; + /// see `linux::linux_drm_amdgpu_card_count_in`. + #[test] + fn a_gfx90c_host_the_kernel_gives_no_compute_node_still_has_a_usable_gpu() { + let (root, _paths) = temp_app_paths("gfx90c-drm-only-host"); + let drm = root.join("sys/class/drm"); + let device = drm.join("card0/device"); + fs::create_dir_all(&device).expect("plant the DRM card"); + fs::write(device.join("vendor"), "0x1002\n").expect("plant the PCI vendor"); + // GC 9.0.12 is the Vega iGPU in Renoir / Lucienne / Cezanne / Barcelo. + let gc = device.join("ip_discovery/die/0/GC/0"); + fs::create_dir_all(&gc).expect("plant the GC IP block"); + for (file, value) in [("major", "9"), ("minor", "0"), ("revision", "12")] { + fs::write(gc.join(file), value).expect("plant the GC IP version"); + } + // A connector sub-node, which is not a card and must not be counted. + fs::create_dir_all(drm.join("card0-eDP-1")).expect("plant the DRM connector"); + // The KFD topology is readable and lists only the CPU node: this host's + // iGPU has no compute node, which is the whole premise. + let nodes = root.join("sys/class/kfd/kfd/topology/nodes"); + fs::create_dir_all(nodes.join("0")).expect("plant the KFD CPU node"); + fs::write( + nodes.join("0").join("properties"), + "cpu_cores_count 16\nsimd_count 0\ngfx_target_version 0\n", + ) + .expect("plant the KFD CPU node properties"); + + let detected = super::super::linux::detect_linux_drm_ip_discovery_gfx_target_in(&drm); + let kfd = super::super::linux::linux_kfd_gpu_node_count_in(&nodes); + let cards = super::super::linux::linux_drm_amdgpu_card_count_in(&drm); + fs::remove_dir_all(&root).ok(); + + // Half one: this is a host `unsupported_host_gpu_message` speaks for. + assert_eq!( + detected.as_deref(), + Some("gfx90c"), + "the DRM ip-discovery read is the only route left on a host with no \ + KFD compute node; without it there is no detected target and the \ + Lemonade advice is never printed" + ); + assert!(gfx_target_has_no_rocm_build("gfx90c")); + + // Half two: and `rocm serve`'s pre-flight does not refuse it. + assert_eq!( + (kfd, cards), + (Some(0), Some(1)), + "the kernel must report no compute node and the bus exactly one card" + ); + let present = combine_amd_gpu_counts(kfd, cards); + let usable = usable_amd_gpu_indices_from(present.expect("GPU presence"), None); + assert_eq!( + usable, + Some(vec![0]), + "serve pins HIP ordinal 0 on this host" + ); + assert!( + !matches!(usable.as_deref(), Some([])), + "`serve_cmd::host_has_no_usable_gpu` is `matches!(visible, Some([]))`; \ + anything else here means the advised command is refused for the very \ + condition it was printed to clear" + ); + } fn wsl_summary(librocdxg_in_cache: bool, on_ld_library_path: bool) -> super::super::WslSummary { super::super::WslSummary { diff --git a/crates/rocm-core/src/host_gpu/linux.rs b/crates/rocm-core/src/host_gpu/linux.rs index 701071d859..47dd97682d 100644 --- a/crates/rocm-core/src/host_gpu/linux.rs +++ b/crates/rocm-core/src/host_gpu/linux.rs @@ -204,7 +204,21 @@ fn natural_node_order(name: &str) -> (u64, String) { /// DRM class dir can't be read; `Some(0)` when it is readable with no AMD card. #[cfg(target_os = "linux")] pub(super) fn linux_drm_amdgpu_card_count() -> Option { - let entries = fs::read_dir(host_path("/sys/class/drm")).ok()?; + linux_drm_amdgpu_card_count_in(&host_path("/sys/class/drm")) +} + +/// The DRM card count, against a caller-supplied `/sys/class/drm`. +/// +/// Split out for the same reason as `linux_kfd_gpu_node_count_in`: this half of +/// the serve pre-flight's "is there a GPU here" answer is what carries a host +/// whose KFD reports no compute node — the gfx90c APUs +/// `unsupported_host_gpu_message` sends to Lemonade among them — and no CI lane +/// has that hardware. Planting the directory is the only way the composed +/// answer can be asserted at all; see +/// `a_gfx90c_host_the_kernel_gives_no_compute_node_still_has_a_usable_gpu`. +#[cfg(any(target_os = "linux", test))] +pub(super) fn linux_drm_amdgpu_card_count_in(class_dir: &Path) -> Option { + let entries = fs::read_dir(class_dir).ok()?; let count = entries .flatten() .filter(|entry| { @@ -236,7 +250,7 @@ pub(crate) fn linux_kfd_gpu_node_count() -> Option { /// branch is driven by tests -- the unreadable branches key off the real /// `/dev/kfd`, which a planted directory cannot stand in for. #[cfg(any(target_os = "linux", test))] -fn linux_kfd_gpu_node_count_in(nodes_dir: &Path) -> Option { +pub(super) fn linux_kfd_gpu_node_count_in(nodes_dir: &Path) -> Option { match fs::read_dir(nodes_dir) { Ok(entries) => Some( entries @@ -305,8 +319,22 @@ fn kfd_gfx_target_version_is_gpu(value: &str) -> bool { #[cfg(target_os = "linux")] fn detect_linux_drm_ip_discovery_gfx_target() -> Option { - let drm_dir = host_path("/sys/class/drm"); - let entries = fs::read_dir(drm_dir).ok()?; + detect_linux_drm_ip_discovery_gfx_target_in(&host_path("/sys/class/drm")) +} + +/// The DRM ip-discovery target read, against a caller-supplied `/sys/class/drm`. +/// +/// Split out for the same reason as [`linux_drm_amdgpu_card_count_in`], and +/// against the *same* directory on purpose: on a host whose KFD exposes no +/// compute node this is the only route left that can name a target, so it is +/// what puts a GPU in front of `unsupported_host_gpu_message` — and it walks +/// `card*/device` under `is_amdgpu_device`, exactly the cards that count then +/// hands to the serve pre-flight. Pointing one planted tree at both is what +/// makes "the host that gets the Lemonade advice is a host serve accepts" +/// assertable rather than argued. +#[cfg(any(target_os = "linux", test))] +pub(super) fn detect_linux_drm_ip_discovery_gfx_target_in(class_dir: &Path) -> Option { + let entries = fs::read_dir(class_dir).ok()?; for entry in entries.flatten() { let card_path = entry.path(); let Some(card_name) = card_path.file_name().and_then(|value| value.to_str()) else { diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index de83cb68c6..be70236505 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -95,18 +95,21 @@ impl ExamineSummary { // `rocm diagnose` is the command that can: its `fix-3-rocm-kernel`, // `fix-4-render-group` and `fix-5-amdgpu-load` entries are exactly // the reasons a present GPU goes undetected. + // + // Kept to one sentence each: the level word already says the CLI + // cannot answer, so spelling out "can confirm neither ROCm support + // nor its absence" in both arms was length without information. + // The distinction it was carrying lives in this comment and in + // `GpuSupportLevel`'s own docs, which is where it is load-bearing. GpuSupportLevel::Unknown => match detected { Some(gfx) => format!( - " rocm_support: unknown - GPU target {gfx} is not in this \ - build's catalog, so this build can confirm neither ROCm \ - support nor its absence; serve a model with `rocm serve \ - --engine lemonade` and read the backend it picked \ - from the `device:` row of `rocm services list`\n" + " rocm_support: unknown - {gfx} is not in this build's \ + catalog; serve with `rocm serve --engine lemonade`, \ + then read the `device:` row of `rocm services list` for the \ + backend it used\n" ), - None => " rocm_support: unknown - no AMD GPU was detected, so \ - this build can confirm neither ROCm support nor its \ - absence; run `rocm diagnose` to find out why no GPU is \ - visible\n" + None => " rocm_support: unknown - no AMD GPU was detected; run \ + `rocm diagnose` to find out why none is visible\n" .to_owned(), }, } @@ -272,6 +275,29 @@ mod tests { assert!(rendered.contains("model_cache_entries: 3")); } + /// The exact placeholders the report prints for the three GPU fields when + /// it has no value, pinned because they are read back as data. + /// + /// examine-35 decides which support tier a host is in by filtering these + /// out of `detected_gfx_target`, `compatible_therock_family` and + /// `detected_therock_family`. Reword one and that scenario stops seeing an + /// absent value as absent: a lane would be classified into the wrong tier + /// and its tier-specific command assertion would silently stop describing + /// the host it ran on. The two strings differ on purpose — "not detected" + /// is about a managed install, "unknown" about a probe — so neither may + /// stand in for the other. + #[test] + fn examine_render_spells_its_absent_value_placeholders() { + let rendered = examine_summary_with_gpu(None, None).render_text(); + for field in [ + "detected_gfx_target: ", + "compatible_therock_family: ", + "detected_therock_family: ", + ] { + assert!(rendered.contains(field), "missing `{field}`:\n{rendered}"); + } + } + fn examine_summary_with_gpu( detected_gfx_target: Option<&str>, compatible_therock_family: Option<&str>, @@ -660,9 +686,7 @@ mod tests { let unknown = examine_summary_with_gpu(Some("gfx803"), None); let rendered = unknown.render_text(); assert!( - rendered.contains( - "rocm_support: unknown - GPU target gfx803 is not in this build's catalog" - ), + rendered.contains("rocm_support: unknown - gfx803 is not in this build's catalog"), "{rendered}" ); assert!( diff --git a/docs/manual-testing.md b/docs/manual-testing.md index e74fa79950..d82a5d666e 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -306,7 +306,8 @@ rocm serve qwen --engine lemonade --device gpu_required --foreground --port 1143 Expected result: - The engine uses ROCm GPU execution. The deployment summary's `device:` row - reports what the engine actually selected, so check it by reading that row + is the engine's own account of the backend it was launched on (Lemonade reads + back the `backend_requested` it recorded), so check it by reading that row rather than inferring it from the absence of a fallback warning. Every engine reports it in the same ` [gpu ] [()]` shape — so expect something like `rocm gpu 0 (AMD Instinct MI300X)`, with the ordinal omitted diff --git a/engines/lemonade/src/device_policy.rs b/engines/lemonade/src/device_policy.rs index e0b65449d5..11a68d70bc 100644 --- a/engines/lemonade/src/device_policy.rs +++ b/engines/lemonade/src/device_policy.rs @@ -133,10 +133,16 @@ pub(crate) const fn device_policy_name(policy: &DevicePolicy) -> &'static str { } } -/// The device string reported once a model is loaded. Reflects the backend that -/// actually ran (`rocm`, `vulkan`, …), the pinned GPU ordinal when known, and -/// the GPU name recorded at launch, rather than a hardcoded value, so the -/// report matches observed execution. +/// The device string reported once a model is loaded: the backend this service +/// was launched on (`rocm`, `vulkan`, …), the pinned GPU ordinal when known, and +/// the GPU name recorded at launch. +/// +/// Read back from the service's own recorded `backend_requested` rather than +/// from a hardcoded value — so it follows the backend selection this launch +/// actually made, including a ROCm-to-Vulkan fallback. It is this engine's +/// account of what it asked llama.cpp for, not an independent measurement of +/// which device executed, which is why the CLI's docs say the row *reports* a +/// device rather than naming the one observed. /// /// Formatted by [`rocm_engine_protocol::format_reported_device`] so all three /// engines render one shape; see that function for why the ordinal-less row diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 1d5a8e194c..775f637127 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -820,3 +820,27 @@ Feature: Model serving And a stopped managed service is recorded on the default serve port When the user restarts the stopped service Then the restart is refused naming the service that holds the port + + # ROCMAI-550, and the evidence that the Vulkan remediation is not a dead end. + # `rocm install sdk` on a gfx90c APU refuses and tells the user to run `rocm + # serve --engine lemonade`; AGENTS.md section 3 only allows that if + # serve's no-usable-GPU pre-flight accepts the very host the refusal was + # printed on. That host is a Renoir/Cezanne laptop: the iGPU is on the bus and + # bound to amdgpu, but the kernel's KFD topology lists no compute node for it, + # so the target can only have come from the DRM ip-discovery read. + # + # The assertion is positive rather than "the GPU refusal is absent": an + # out-of-range `--gpu` is validated AFTER the pre-flight and prints the usable + # ordinals it found, so the refusal text here can only be produced by a + # pre-flight that resolved exactly one usable GPU on this host. Take away + # `combine_amd_gpu_counts`'s zero-KFD fallback to DRM and this turns into "no + # usable AMD GPU detected", which is what the step rejects by name. + # + # Linux only, and needs no AMD hardware: the host is planted under + # ROCM_CLI_TEST_HOST_ROOT (an `e2e-test-hooks` build), so the GPU-less mock + # lane is where this runs. + @id:serve-accepts-an-apu-the-kernel-gives-no-compute-node @requires-os:linux + Scenario: serve-50 - Serving is not refused on a Vulkan-only APU the kernel gives no compute node + Given a host whose only GPU is a gfx90c Vega iGPU the kernel reports no compute node for + When the user is told to serve on Lemonade because this GPU has no ROCm build + Then serving got past the no-GPU pre-flight and reported this host's usable GPU diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 786145426a..399146a688 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -2531,27 +2531,58 @@ async fn assert_support_level_agrees_with_detection(world: &mut E2eWorld) { } /// AGENTS.md wants a line that tells the user what they can do, and a named -/// command that can actually clear the condition it was printed for. The -/// specific pairing checked here is the one this round fixed: a host whose -/// managed SDK is already installed must not be told to install it. +/// command that can actually clear the condition it was printed for. +/// +/// So the assertion is the *specific* command each tier owes, not "some `rocm` +/// word appears": `contains("rocm ")` is satisfied by `rocm examine` echoing +/// its own name, by a stray `rocm diagnose` on a ROCm-capable host, and by +/// every wrong-tier command there is. Which tier applies is read from the same +/// three fields the renderer classifies on, so this says something different — +/// and falsifiable — on each lane rather than collapsing to one weak check. +/// +/// The placeholders those fields fall back to are pinned by +/// `rocm_core`'s `examine_render_spells_its_absent_value_placeholders`, so this +/// cannot silently start reading every host as "nothing detected" if the +/// renderer's wording changes. #[then("the support level leaves a command that fits this host")] async fn assert_support_level_advice_fits(world: &mut E2eWorld) { let output = world.cli_output.as_ref().expect("no command was run"); let line = rocm_support_line(output).expect("no rocm_support line"); + let (detected, family) = detected_gpu_facts(output); + // `detected_therock_family` is the managed SDK that *is* installed, as + // opposed to `compatible_therock_family`, which is what this GPU could run. + let installed_sdk = field_value(output, "detected_therock_family") + .filter(|value| *value != "" && *value != ""); + + let (expected, forbidden): (&str, &[&str]) = match (detected, family, installed_sdk) { + // ROCm-capable with the managed SDK already in place: installing one + // clears nothing, so the line must point at serving instead. + (_, Some(_), Some(_)) => ("rocm serve ", &["rocm install sdk"]), + // ROCm-capable with no managed SDK yet: that install is the next step. + (_, Some(_), None) => ("rocm install sdk", &[]), + // A GPU no package family covers — confirmed Vulkan-only (gfx90c) or + // merely unrecognized. Either way the SDK has nothing for it and + // Lemonade's Vulkan backend is what can serve on it. + (Some(_), None, _) => ( + "rocm serve --engine lemonade", + &["rocm install sdk"], + ), + // Nothing detected: `rocm serve` is refused outright by the + // no-usable-GPU pre-flight, so naming it would be advice that cannot + // clear the condition it is printed for. + (None, None, _) => ("rocm diagnose", &["rocm serve", "rocm install sdk"]), + }; + assert!( - line.contains("rocm "), - "the support line must leave the user somewhere to go: {line}\n{output}" + line.contains(expected), + "this host's support tier owes `{expected}`, and the line has to name a \ + command that can actually clear the condition it was printed for: \ + {line}\n{output}" ); - - // `detected_therock_family` is the managed SDK that is installed, as - // opposed to `compatible_therock_family`, which is what this GPU could run. - let sdk_installed = field_value(output, "detected_therock_family") - .is_some_and(|value| value != "" && value != ""); - if sdk_installed { + for command in forbidden { assert!( - !line.contains("rocm install sdk"), - "this host already has a managed ROCm SDK, so installing one clears \ - nothing: {line}\n{output}" + !line.contains(command), + "`{command}` cannot clear anything on this host: {line}\n{output}" ); } } diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 8026963b2a..f516b5bb59 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1287,12 +1287,7 @@ fn docker_call_log(world: &E2eWorld) -> std::path::PathBuf { /// by `e2e-test-hooks` builds) so the family the CLI detects does not depend on the /// runner's hardware. `gfx_target_version` is the kernel's decimal encoding. fn plant_single_gpu(world: &mut E2eWorld, gfx_target_version: u32) { - let host_root = world - .isolated_root - .as_ref() - .expect("scenario has no isolated root") - .path() - .join("host-root"); + let host_root = planted_host_root(world); let node = host_root.join("sys/class/kfd/kfd/topology/nodes/1"); std::fs::create_dir_all(&node).expect("failed to create the planted KFD node"); // `cpu_cores_count 0` with a nonzero `simd_count` makes the node a GPU. @@ -1301,9 +1296,7 @@ fn plant_single_gpu(world: &mut E2eWorld, gfx_target_version: u32) { format!("cpu_cores_count 0\nsimd_count 56\ngfx_target_version {gfx_target_version}\n"), ) .expect("failed to write the planted KFD node properties"); - world - .command_env - .push(("ROCM_CLI_TEST_HOST_ROOT", host_root.into_os_string())); + use_planted_host_root(world); } /// gfx942, TheRock family `gfx94X-dcgpu`. @@ -1318,6 +1311,119 @@ async fn host_with_a_w7900(world: &mut E2eWorld) { plant_single_gpu(world, 110_000); } +/// A Renoir/Cezanne laptop, planted under the simulated host root: the Vega +/// iGPU is on the PCI bus and bound to `amdgpu`, and the kernel's KFD topology +/// lists **only a CPU node** for it. +/// +/// Not `plant_single_gpu`: that writes a KFD GPU node, which is precisely the +/// thing this host does not have. With no compute node the target can only be +/// named by the DRM `ip_discovery` read (GC 9.0.12 -> `gfx90c`), and the usable +/// GPU count can only come from `combine_amd_gpu_counts`' zero-KFD fallback to +/// the DRM card count. Both are driven off this one directory, which is what +/// makes the scenario a statement about one host rather than two. +fn plant_gfx90c_apu_without_a_kfd_node(world: &mut E2eWorld) { + let host_root = planted_host_root(world); + let device = host_root.join("sys/class/drm/card0/device"); + std::fs::create_dir_all(&device).expect("failed to create the planted DRM card"); + std::fs::write(device.join("vendor"), "0x1002\n").expect("failed to write the planted vendor"); + let gc = device.join("ip_discovery/die/0/GC/0"); + std::fs::create_dir_all(&gc).expect("failed to create the planted GC IP block"); + for (file, value) in [("major", "9"), ("minor", "0"), ("revision", "12")] { + std::fs::write(gc.join(file), value).expect("failed to write the planted GC IP version"); + } + let cpu_node = host_root.join("sys/class/kfd/kfd/topology/nodes/0"); + std::fs::create_dir_all(&cpu_node).expect("failed to create the planted KFD node"); + std::fs::write( + cpu_node.join("properties"), + "cpu_cores_count 16\nsimd_count 0\ngfx_target_version 0\n", + ) + .expect("failed to write the planted KFD node properties"); + use_planted_host_root(world); +} + +/// Where a scenario's simulated `/` lives, by the same convention +/// [`plant_single_gpu`] uses. +fn planted_host_root(world: &E2eWorld) -> std::path::PathBuf { + world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .join("host-root") +} + +/// Point the *next* `rocm` invocation at the planted host. +/// +/// Needed once per invocation: `run_rocm_with_scenario_env` takes +/// `world.command_env` by `mem::take`, so a step that runs two commands would +/// silently run the second one against the real machine — which for this +/// scenario would mean asserting the serve pre-flight's answer for the runner's +/// own hardware instead of for the host the refusal was printed on. +fn use_planted_host_root(world: &mut E2eWorld) { + let host_root = planted_host_root(world); + world + .command_env + .push(("ROCM_CLI_TEST_HOST_ROOT", host_root.into_os_string())); +} + +#[given("a host whose only GPU is a gfx90c Vega iGPU the kernel reports no compute node for")] +async fn host_with_a_kfd_less_gfx90c(world: &mut E2eWorld) { + plant_gfx90c_apu_without_a_kfd_node(world); +} + +/// Take the route the CLI itself sends this host down: let the SDK install +/// refuse, then run the command that refusal names. +/// +/// Both halves run against the same planted host, in order, so the scenario +/// cannot pass by printing the advice on one machine and testing it on another. +/// `--gpu 99` is what makes the result readable: it is validated *after* the +/// no-usable-GPU pre-flight and reports the ordinals that pre-flight resolved, +/// turning "the host was accepted" into something the output states rather than +/// something inferred from the absence of a refusal. The model ref is arbitrary +/// and nothing is launched: both bails are reached before any runtime or engine +/// work. +#[when("the user is told to serve on Lemonade because this GPU has no ROCm build")] +async fn user_follows_the_lemonade_advice(world: &mut E2eWorld) { + let (install_stdout, install_stderr, install_rc) = + crate::run_rocm_with_scenario_env(world, &["install", "sdk", "--dry-run"]); + let refusal = format!("{install_stdout}\n{install_stderr}"); + assert_ne!( + install_rc, 0, + "a gfx90c host has no ROCm SDK build to install:\n{refusal}" + ); + assert!( + refusal.contains("gfx90c") && refusal.contains("rocm serve --engine lemonade"), + "the planted host must be detected as gfx90c and advised to serve on \ + Lemonade; without that this scenario is testing nothing:\n{refusal}" + ); + + use_planted_host_root(world); + let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env( + world, + &["serve", "some-model", "--engine", "lemonade", "--gpu", "99"], + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("serving got past the no-GPU pre-flight and reported this host's usable GPU")] +async fn assert_serve_accepted_the_drm_only_gpu(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + !output.contains("no usable AMD GPU"), + "`rocm install sdk` told this host to run `rocm serve --engine lemonade`, \ + and serve's pre-flight refused it for having no GPU - advice that cannot \ + clear the condition it was printed for:\n{output}" + ); + assert!( + output.contains("--gpu index 99 is not present on this host") + && output.contains("usable GPU indices: [0]"), + "the pre-flight must have resolved exactly the iGPU's ordinal, and said \ + so where `--gpu` is validated:\n{output}" + ); +} + fn run_serve_aim(world: &mut E2eWorld, args: &[&str]) { let mut full = vec!["serve"]; full.extend_from_slice(args); From 794e4b6a90c23a23c516040a4c5284f243b94cd0 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 11:10:41 +0200 Subject: [PATCH 14/59] feat(examine): probe Vulkan instead of inferring it from the gfx table `GpuSupportLevel::VulkanOnly` was derived from the gfx-target table alone: "no ROCm build" was read as "Vulkan then", on a machine nobody had looked at. Every surface that renders that level tells the user to go and serve a model on it, so a host with no Vulkan driver installed -- or with only a software rasteriser -- was sent to a backend that cannot start, with no CPU fallback to catch it. Adds a real probe (host_gpu/vulkan.rs): ICD manifests through `host_path` on Linux, the Khronos registry on Windows, and `vulkaninfo --summary` for device enumeration. Filesystem and subprocess evidence rather than dlopening libvulkan, because the loader reads the real ICD list and so could never be faked on the no-GPU lane -- and because running vendor driver code inside `examine` can hang a headless runner. `gpu_support_level` now takes that evidence. An installed driver and an enumerated device are kept apart (`DriverPresentDevicesUnverified`), since collapsing them would replace one unverified promise with another; a platform that cannot be probed keeps `VulkanOnly`, because withholding the only path such a host has, on the strength of a probe that never ran, would be a worse answer than the assumption being replaced. The negative case gets its own level. `NoGpuBackend` is not folded into `Unknown`: both halves there are established (the catalog confirms no ROCm build, the probe finds no driver), whereas `Unknown` means neither could be. Collapsing them commits, in the other direction, the conflation `Unknown` exists to prevent. A software rasteriser is never a GPU path. Mesa lavapipe is a complete Vulkan implementation running on the CPU, so counting it would reintroduce the silent CPU fallback through a backend rather than through a policy change. `rocm engines` and the codex bridge deliberately do not probe: they pass `VulkanStatus::Unknown` and answer exactly as before, so a listing command does not grow a subprocess. `rocm examine` already gathers the probe and is the surface those messages send people to. examine-49/50/51 cover the three answers on the ungated mock lane, which needs neither a Renoir APU nor a Vulkan driver -- only the KFD node and ICD state the harness can plant. Falsified: both negative scenarios fail when `gpu_support_level` is made to ignore the probe. Signed-off-by: Roman Sirokov --- apps/rocm/src/bridge_snapshot.rs | 1 + apps/rocm/src/driver_plan.rs | 1 + apps/rocm/src/examine_report_text.rs | 10 +- apps/rocm/src/main.rs | 24 +- crates/rocm-core/src/gfx_target.rs | 133 +++- crates/rocm-core/src/host_gpu.rs | 17 + crates/rocm-core/src/host_gpu/render.rs | 207 +++++- crates/rocm-core/src/host_gpu/vulkan.rs | 686 ++++++++++++++++++ crates/rocm-core/src/lib.rs | 12 +- docs/architecture.md | 2 +- tests/e2e-cucumber/features/examine.feature | 37 + tests/e2e-cucumber/tests/e2e/examine_steps.rs | 224 ++++++ 12 files changed, 1319 insertions(+), 35 deletions(-) create mode 100644 crates/rocm-core/src/host_gpu/vulkan.rs diff --git a/apps/rocm/src/bridge_snapshot.rs b/apps/rocm/src/bridge_snapshot.rs index 741c18ede7..115279e42a 100644 --- a/apps/rocm/src/bridge_snapshot.rs +++ b/apps/rocm/src/bridge_snapshot.rs @@ -40,6 +40,7 @@ pub(crate) fn build_codex_bridge_snapshot(paths: &AppPaths) -> Result) render_engine_inventory_text_for_host( paths, rocm_core::default_engine_for_host(&host_gpu), - rocm_core::gpu_support_level_for_host_gpu(&host_gpu), + // No Vulkan probe on this listing path, for the reason given on + // `host_gpu_support_level`: `rocm engines` is cheap by design, and + // `rocm examine` is where the verified answer lives. + rocm_core::gpu_support_level_for_host_gpu(&host_gpu, rocm_core::VulkanStatus::Unknown), ) } @@ -634,6 +641,7 @@ mod tests { detected_gfx_target: None, compatible_therock_family: None, detected_therock_family: None, + vulkan: None, driver: rocm_core::DriverSummary { policy: "linux_official_amd_dkms_wrapper".to_owned(), status: "amdgpu_available".to_owned(), diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index c18432bc61..3887afd02e 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -1081,6 +1081,15 @@ const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str rocm_core::GpuSupportLevel::RocmHip => { "default embedded Lemonade server with ROCm llama.cpp backend" } + // Only `rocm examine` reaches this level — the cheap probe behind + // `host_gpu_support_level` passes `VulkanStatus::Unknown` and so lands + // on `VulkanOnly` — but the arm is spelled out rather than folded into + // `Unknown`, because what it means is the opposite: `Unknown` promises + // that a command can still answer, and here the question is already + // answered and the answer is no. + rocm_core::GpuSupportLevel::NoGpuBackend => { + "default embedded Lemonade server; this GPU has no ROCm/HIP support and no Vulkan driver was found, so no GPU backend can run here" + } rocm_core::GpuSupportLevel::Unknown => { "default embedded Lemonade server; run `rocm examine` for this host's GPU support level, which says how to find the backend" } @@ -1093,8 +1102,21 @@ const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str /// The same two inputs `rocm examine` derives it from, read through /// `detect_host_gpu_summary` rather than the full examine gather — exactly what /// `render_engine_inventory_text_with_paths` (i.e. `rocm engines`) already does. +/// +/// **The Vulkan probe is deliberately not run here**, so this stays the cheap +/// probe its callers chose it for: the probe spawns `vulkaninfo` on Linux and +/// PowerShell on Windows, and `rocm engines` is a listing command that would +/// pay that on every invocation. Passing [`VulkanStatus::Unknown`] keeps these +/// surfaces answering exactly as they did before this probe existed — a +/// Vulkan-capable-by-the-table host still reads `VulkanOnly` — while `rocm +/// examine`, which already gathers the probe, is the surface that can tell a +/// host with a Vulkan driver from one without. That split is why +/// `lemonade_engine_note` sends the user to `rocm examine` for a real answer. pub(crate) fn host_gpu_support_level(paths: Option<&AppPaths>) -> rocm_core::GpuSupportLevel { - rocm_core::gpu_support_level_for_host_gpu(&rocm_core::detect_host_gpu_summary(paths)) + rocm_core::gpu_support_level_for_host_gpu( + &rocm_core::detect_host_gpu_summary(paths), + rocm_core::VulkanStatus::Unknown, + ) } fn refresh_startup_update_check_quietly() { diff --git a/crates/rocm-core/src/gfx_target.rs b/crates/rocm-core/src/gfx_target.rs index bd0a6bd6b0..391c145fd3 100644 --- a/crates/rocm-core/src/gfx_target.rs +++ b/crates/rocm-core/src/gfx_target.rs @@ -946,7 +946,7 @@ mod tests { therock_family: Some("gfx94X-dcgpu".to_owned()), }; assert_eq!( - gpu_support_level_for_host_gpu(&host_gpu), + gpu_support_level_for_host_gpu(&host_gpu, VulkanStatus::Available), GpuSupportLevel::RocmHip ); } @@ -959,7 +959,7 @@ mod tests { therock_family: None, }; assert_eq!( - gpu_support_level_for_host_gpu(&host_gpu), + gpu_support_level_for_host_gpu(&host_gpu, VulkanStatus::Available), GpuSupportLevel::VulkanOnly ); } @@ -967,32 +967,86 @@ mod tests { #[test] fn gpu_support_level_for_host_gpu_reads_an_undetected_summary_as_unknown() { assert_eq!( - gpu_support_level_for_host_gpu(&HostGpuSummary::default()), + gpu_support_level_for_host_gpu(&HostGpuSummary::default(), VulkanStatus::Available), GpuSupportLevel::Unknown ); } + use crate::host_gpu::VulkanStatus; + #[test] fn gpu_support_level_is_rocm_hip_when_a_family_covers_it() { assert_eq!( - gpu_support_level(Some("gfx90a"), Some("gfx90a")), + gpu_support_level(Some("gfx90a"), Some("gfx90a"), VulkanStatus::Available), + GpuSupportLevel::RocmHip + ); + // A covering family wins outright: a host whose Vulkan probe found + // nothing still has ROCm, and must not be downgraded for it. + assert_eq!( + gpu_support_level(Some("gfx90a"), Some("gfx90a"), VulkanStatus::Absent), GpuSupportLevel::RocmHip ); } #[test] fn gpu_support_level_is_unknown_without_detection_or_for_an_unrecognized_target() { - assert_eq!(gpu_support_level(None, None), GpuSupportLevel::Unknown); assert_eq!( - gpu_support_level(Some("gfx_totally_unrecognized"), None), + gpu_support_level(None, None, VulkanStatus::Available), + GpuSupportLevel::Unknown + ); + // Finding a Vulkan driver says nothing about whether ROCm would work on + // an unrecognized target, so it must not promote a detection gap into a + // support claim. + assert_eq!( + gpu_support_level( + Some("gfx_totally_unrecognized"), + None, + VulkanStatus::Available + ), GpuSupportLevel::Unknown ); } #[test] - fn gpu_support_level_is_vulkan_only_for_gfx90c() { + fn gpu_support_level_is_vulkan_only_for_gfx90c_with_a_vulkan_driver() { + assert_eq!( + gpu_support_level(Some("gfx90c"), None, VulkanStatus::Available), + GpuSupportLevel::VulkanOnly + ); + // An installed driver that enumerated no device is still worth routing + // to; the level says "Vulkan", and `examine`'s own line carries how sure + // the probe was. assert_eq!( - gpu_support_level(Some("gfx90c"), None), + gpu_support_level( + Some("gfx90c"), + None, + VulkanStatus::DriverPresentDevicesUnverified + ), + GpuSupportLevel::VulkanOnly + ); + } + + /// The defect this guards against: `VulkanOnly` used to be inferred from the + /// gfx table alone, so a Renoir laptop with no Vulkan driver installed — or + /// with only a CPU rasteriser — was told to go and serve a model on a + /// backend that cannot start. + #[test] + fn gpu_support_level_reports_no_gpu_backend_when_the_vulkan_probe_found_none() { + assert_eq!( + gpu_support_level(Some("gfx90c"), None, VulkanStatus::Absent), + GpuSupportLevel::NoGpuBackend, + "a confirmed-absent Vulkan driver must not read as a usable Vulkan path" + ); + } + + /// A platform the probe cannot read is not a platform without Vulkan. + /// Withholding the only path such a host has, on the strength of a probe + /// that never ran, would be a worse answer than the assumption this change + /// replaces. + #[test] + fn gpu_support_level_keeps_vulkan_when_the_probe_could_not_run() { + assert_eq!( + gpu_support_level(Some("gfx90c"), None, VulkanStatus::Unknown), GpuSupportLevel::VulkanOnly ); } @@ -1013,23 +1067,63 @@ mod tests { pub enum GpuSupportLevel { /// A TheRock package family covers this GPU: full ROCm/HIP support. RocmHip, - /// Recognized, confirmed to have no ROCm/HIP build, but Vulkan-based - /// engines (the default Lemonade backend) still work on it. + /// Recognized, confirmed to have no ROCm/HIP build, and a Vulkan GPU driver + /// was **found on this host**, so Vulkan-based engines (the default Lemonade + /// backend) run on it. + /// + /// The second half of that sentence is evidence, not inference. This variant + /// used to be returned from the gfx table alone — "no ROCm build" was taken + /// to mean "Vulkan then", on a machine nobody had looked at. Every surface + /// that renders this level tells the user to go and serve a model on it, so + /// a host with no Vulkan driver installed, or with only a software + /// rasteriser, was sent to a backend that cannot start. VulkanOnly, + /// Recognized, confirmed to have no ROCm/HIP build, **and** no Vulkan GPU + /// driver was found: this host has no GPU path at all. + /// + /// Deliberately not folded into [`Self::Unknown`]. Both halves here are + /// confirmed — the catalog establishes the missing ROCm build, and the + /// Vulkan probe looked and found nothing — whereas `Unknown` means the CLI + /// could not establish either. Collapsing them would commit, in the other + /// direction, the exact conflation this enum's `Unknown` exists to prevent: + /// a fact about the machine would be reported as a gap in what this build + /// knows, and the user would be sent to probe a question already answered. + NoGpuBackend, /// No GPU detected yet, or a gfx target this catalog doesn't recognize. Unknown, } -/// Classify the host's GPU support level from the same two facts -/// `ExamineSummary` and `resolve_family` already compute: the detected gfx -/// target and whichever TheRock family (if any) covers it. +/// Classify the host's GPU support level. +/// +/// Reads the two facts `ExamineSummary` and `resolve_family` already compute — +/// the detected gfx target and whichever TheRock family (if any) covers it — +/// plus what the Vulkan probe found. +/// +/// `vulkan` only ever decides between [`GpuSupportLevel::VulkanOnly`] and +/// [`GpuSupportLevel::NoGpuBackend`], and is consulted only once the catalog has +/// confirmed there is no ROCm build. A host with a covering family is `RocmHip` +/// whatever Vulkan reports, and an unrecognized target stays `Unknown`: finding +/// a Vulkan driver says nothing about whether ROCm would also work there, so it +/// must not be allowed to turn a detection gap into a support claim. +/// +/// [`VulkanStatus::Unknown`] — the platform could not be probed at all — keeps +/// `VulkanOnly`, because withholding the one path such a host has on the +/// strength of a probe that never ran would be worse than the old assumption, +/// not better. pub fn gpu_support_level( detected_gfx_target: Option<&str>, compatible_therock_family: Option<&str>, + vulkan: crate::host_gpu::VulkanStatus, ) -> GpuSupportLevel { match (detected_gfx_target, compatible_therock_family) { (_, Some(_)) => GpuSupportLevel::RocmHip, - (Some(gfx), None) if gfx_target_has_no_rocm_build(gfx) => GpuSupportLevel::VulkanOnly, + (Some(gfx), None) if gfx_target_has_no_rocm_build(gfx) => { + if vulkan.is_usable() || vulkan == crate::host_gpu::VulkanStatus::Unknown { + GpuSupportLevel::VulkanOnly + } else { + GpuSupportLevel::NoGpuBackend + } + } _ => GpuSupportLevel::Unknown, } } @@ -1041,10 +1135,19 @@ pub fn gpu_support_level( /// site that transposed them would classify a Vulkan-only host as `RocmHip` /// and still compile. `rocm engines` and the codex-bridge `engines` tool both /// derive the level from this struct, so both go through this. +/// +/// `vulkan` is passed in rather than probed here so this stays a pure function +/// of what the caller already holds: the probe runs a subprocess and reads the +/// filesystem, and burying that in a helper this cheap would make every caller +/// pay for it without saying so. #[must_use] -pub fn gpu_support_level_for_host_gpu(host_gpu: &HostGpuSummary) -> GpuSupportLevel { +pub fn gpu_support_level_for_host_gpu( + host_gpu: &HostGpuSummary, + vulkan: crate::host_gpu::VulkanStatus, +) -> GpuSupportLevel { gpu_support_level( host_gpu.gfx_target.as_deref(), host_gpu.therock_family.as_deref(), + vulkan, ) } diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index 87b6ad185f..db45107033 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -54,6 +54,7 @@ mod gpu_visibility; pub(crate) mod linux; mod os_facts; mod render; +mod vulkan; mod windows_inventory; mod wsl; @@ -73,6 +74,11 @@ pub(crate) use linux::detect_linux_sysfs_gfx_target; pub use linux::is_amdgpu_device; #[cfg(any(target_os = "linux", test))] pub(crate) use linux::{KfdGpuNode, kfd_gpu_nodes_in}; +pub use vulkan::{ + VulkanDevice, VulkanStatus, VulkanSupport, classify_vulkan_support, detect_vulkan_support, + icd_driver_label, icd_is_gpu_driver, parse_vulkaninfo_summary, + parse_windows_vulkan_icd_listing, +}; pub(crate) use windows_inventory::detect_windows_amd_driver_info; #[cfg(test)] use windows_inventory::parse_windows_examine_inventory; @@ -133,6 +139,15 @@ pub struct ExamineSummary { pub compatible_therock_family: Option, #[serde(default)] pub detected_therock_family: Option, + /// What the Vulkan probe found, when it could run. + /// + /// `#[serde(default)]` is load-bearing in the *reverse* direction: `rocm + /// remote doctor` deserializes summaries produced by other hosts, which may + /// be running a build from before this field existed. `None` means "not + /// probed", which [`ExamineSummary::vulkan_status`] maps to + /// [`VulkanStatus::Unknown`] — never to "no Vulkan". + #[serde(default)] + pub vulkan: Option, pub driver: DriverSummary, pub legacy_rocm: LegacyRocmSummary, #[serde(default)] @@ -284,6 +299,7 @@ impl ExamineSummary { detected_gfx_target, compatible_therock_family, detected_therock_family, + vulkan: Some(detect_vulkan_support()), driver: detect_driver_summary_with_windows_inventory( windows_inventory.as_ref(), wsl.as_ref(), @@ -730,6 +746,7 @@ mod tests { detected_gfx_target: None, compatible_therock_family: None, detected_therock_family: None, + vulkan: None, driver: windows_driver_summary( Some("AMD Radeon(TM) Graphics driver 30.0.13014.8".to_owned()), info, diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index 82eb8b8391..df88230d12 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -6,9 +6,42 @@ use super::ExamineSummary; use super::os_facts::format_gib_value; +use super::vulkan::VulkanStatus; use crate::gfx_target::{GpuSupportLevel, gpu_support_level}; impl ExamineSummary { + /// What the Vulkan probe found, or [`VulkanStatus::Unknown`] when this + /// summary carries no probe result — an older host's JSON read back by + /// `rocm remote doctor`, or a caller that built the struct by hand. + /// + /// Public because the `rocm` binary classifies its own support level from a + /// summary it already gathered, and must read the probe result the same way + /// this module does rather than reaching into the field and inventing its + /// own `None` handling. + #[must_use] + pub fn vulkan_status(&self) -> VulkanStatus { + self.vulkan + .as_ref() + .map_or(VulkanStatus::Unknown, |vulkan| vulkan.status) + } + + /// The `vulkan` line: the status, and the evidence behind it. + fn vulkan_line(&self) -> String { + let Some(vulkan) = self.vulkan.as_ref() else { + return "".to_owned(); + }; + let status = match vulkan.status { + VulkanStatus::Available => "available", + VulkanStatus::DriverPresentDevicesUnverified => "driver_present_devices_unverified", + VulkanStatus::Absent => "absent", + VulkanStatus::Unknown => "unknown", + }; + vulkan.detail.as_deref().map_or_else( + || status.to_owned(), + |detail| format!("{status} ({detail})"), + ) + } + /// The `rocm_support` line of the examine text. /// /// ROCMAI-550 asks `rocm examine` to "report the support level with one @@ -34,14 +67,49 @@ impl ExamineSummary { /// blank line. fn rocm_support_line(&self) -> String { let detected = self.detected_gfx_target.as_deref(); - match gpu_support_level(detected, self.compatible_therock_family.as_deref()) { + let vulkan = self.vulkan_status(); + match gpu_support_level(detected, self.compatible_therock_family.as_deref(), vulkan) { // `VulkanOnly` is only ever returned for a detected target, so the // fallback below is unreachable; it keeps the arm total without a // panic path. + // + // The claim is graded by what the probe actually established. An + // enumerated device is named, because naming it is what lets the + // reader check the claim rather than take it; an installed-but- + // unenumerated driver says so instead of borrowing the stronger + // sentence; and a platform that could not be probed admits it. All + // three still point at the same command, because all three are + // hosts worth trying. GpuSupportLevel::VulkanOnly => format!( - " rocm_support: GPU detected ({}) but not supported by ROCm; \ - the Vulkan backend can still be used - serve on it with \ - `rocm serve --engine lemonade`\n", + " rocm_support: GPU detected ({}) but not supported by ROCm; {} - serve on it \ + with `rocm serve --engine lemonade`\n", + detected.unwrap_or_default(), + match vulkan { + VulkanStatus::Available => self.vulkan.as_ref().map_or_else( + || "the Vulkan backend can be used".to_owned(), + |support| support.driver_label().map_or_else( + || "the Vulkan backend can be used".to_owned(), + |driver| format!("the Vulkan backend can be used ({driver})"), + ), + ), + VulkanStatus::DriverPresentDevicesUnverified => + "a Vulkan driver is installed, so the Vulkan backend is expected to work" + .to_owned(), + VulkanStatus::Absent | VulkanStatus::Unknown => + "Vulkan was not probed on this platform, so the Vulkan backend is \ + unconfirmed" + .to_owned(), + } + ), + // Both halves are established facts: the catalog confirmed there is + // no ROCm build, and the probe looked for a Vulkan driver and found + // none. Unlike `Unknown` below, there is nothing left to go and find + // out on this host, so the advice is to install a driver rather than + // to serve a model and read back what happened. + GpuSupportLevel::NoGpuBackend => format!( + " rocm_support: GPU detected ({}) but not supported by ROCm, and no Vulkan GPU \ + driver was found - install one (Mesa RADV on Linux, the AMD display driver on \ + Windows) to run models on this GPU\n", detected.unwrap_or_default() ), // The covering family is the fact; it is already printed on its own @@ -137,7 +205,7 @@ impl ExamineSummary { "false (this run's output is captured, so the CLI will not prompt)" }; format!( - "rocm examine\n os: {}\n arch: {}\n kernel: {}\n distro: {}\n cpu: {}\n system_ram: {}\n interactive_terminal: {}\n default_engine: {}\n detected_gfx_target: {}\n compatible_therock_family: {}\n{} detected_therock_family: {}\n driver_policy: {}\n driver_status: {}\n driver_detail: {}\n{} legacy_rocm_status: {}\n legacy_rocm_paths: {}\n legacy_rocm_version: {}\n legacy_rocm_detail: {}\n legacy_rocm_guidance: {}\n wsl: {}\n wsl_dxg_device: {}\n wsl_dxcore: {}\n wsl_librocdxg: {}\n wsl_rocdxg_dids: {}\n wsl_ldconfig_librocdxg: {}\n wsl_librocdxg_on_ld_library_path: {}\n wsl_librocdxg_beside_runtime: {}\n wsl_global_rocminfo: {}\n wsl_cargo: {}\n wsl_detail: {}\n managed_runtimes: {}\n managed_services: {}\n model_cache_entries: {}\n config_dir: {}\n data_dir: {}\n cache_dir: {}\n", + "rocm examine\n os: {}\n arch: {}\n kernel: {}\n distro: {}\n cpu: {}\n system_ram: {}\n interactive_terminal: {}\n default_engine: {}\n detected_gfx_target: {}\n compatible_therock_family: {}\n{} detected_therock_family: {}\n vulkan: {}\n driver_policy: {}\n driver_status: {}\n driver_detail: {}\n{} legacy_rocm_status: {}\n legacy_rocm_paths: {}\n legacy_rocm_version: {}\n legacy_rocm_detail: {}\n legacy_rocm_guidance: {}\n wsl: {}\n wsl_dxg_device: {}\n wsl_dxcore: {}\n wsl_librocdxg: {}\n wsl_rocdxg_dids: {}\n wsl_ldconfig_librocdxg: {}\n wsl_librocdxg_on_ld_library_path: {}\n wsl_librocdxg_beside_runtime: {}\n wsl_global_rocminfo: {}\n wsl_cargo: {}\n wsl_detail: {}\n managed_runtimes: {}\n managed_services: {}\n model_cache_entries: {}\n config_dir: {}\n data_dir: {}\n cache_dir: {}\n", self.os, self.arch, self.kernel.as_deref().unwrap_or(""), @@ -155,6 +223,7 @@ impl ExamineSummary { self.detected_therock_family .as_deref() .unwrap_or(""), + self.vulkan_line(), self.driver.policy, self.driver.status, self.driver.detail.as_deref().unwrap_or(""), @@ -217,8 +286,8 @@ mod tests { use super::*; use crate::LegacyRocmSummary; use crate::gfx_target::normalize_therock_family; - use crate::host_gpu::DriverSummary; use crate::host_gpu::tests::windows_summary_with_driver; + use crate::host_gpu::{DriverSummary, VulkanDevice, VulkanSupport}; use std::path::PathBuf; #[test] @@ -235,6 +304,7 @@ mod tests { detected_gfx_target: None, compatible_therock_family: Some("gfx120X-all".to_owned()), detected_therock_family: None, + vulkan: None, driver: DriverSummary { policy: "windows_validate_only".to_owned(), status: "amd_display_driver_detected".to_owned(), @@ -314,6 +384,7 @@ mod tests { detected_gfx_target: detected_gfx_target.map(str::to_owned), compatible_therock_family: compatible_therock_family.map(str::to_owned), detected_therock_family: None, + vulkan: None, driver: DriverSummary { policy: "windows_validate_only".to_owned(), status: "amd_display_driver_detected".to_owned(), @@ -336,12 +407,41 @@ mod tests { } } + /// A gfx90c summary carrying the given Vulkan evidence. + /// + /// gfx90c is detected from its PCI ID but has no ROCm package family, so it + /// is the target that reaches the Vulkan-graded arms at all. + fn gfx90c_summary_with_vulkan(vulkan: Option) -> ExamineSummary { + let family = normalize_therock_family("gfx90c"); + let mut summary = examine_summary_with_gpu(Some("gfx90c"), family.as_deref()); + summary.vulkan = vulkan; + summary + } + + fn vulkan_support(status: VulkanStatus, detail: &str) -> Option { + Some(VulkanSupport { + status, + detail: Some(detail.to_owned()), + devices: vec![VulkanDevice { + name: "AMD Radeon Graphics (RADV RENOIR)".to_owned(), + driver_name: Some("radv".to_owned()), + vendor_id: Some("0x1002".to_owned()), + device_type: Some("PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU".to_owned()), + ..VulkanDevice::default() + }], + ..VulkanSupport::default() + }) + } + #[test] fn examine_render_says_a_gpu_without_a_rocm_family_is_unsupported() { - // gfx90c is detected from its PCI ID but has no ROCm package family. - let family = normalize_therock_family("gfx90c"); - let summary = examine_summary_with_gpu(Some("gfx90c"), family.as_deref()); + let summary = gfx90c_summary_with_vulkan(vulkan_support( + VulkanStatus::Available, + "AMD Radeon Graphics (RADV RENOIR) through radv", + )); + let rendered = summary.render_text(); + assert!( rendered.contains("detected_gfx_target: gfx90c"), "{rendered}" @@ -351,9 +451,92 @@ mod tests { "{rendered}" ); assert!( - rendered.contains("the Vulkan backend can still be used"), + rendered.contains("the Vulkan backend can be used (radv)"), + "a confirmed device must be named, so the claim can be checked: {rendered}" + ); + assert!( + rendered.contains("vulkan: available"), + "the evidence line must carry the verdict too: {rendered}" + ); + } + + /// The defect this guards against: the `VulkanOnly` line used to promise + /// "the Vulkan backend can still be used" on every gfx90c host, derived from + /// the gfx table alone. A Renoir laptop with no Vulkan driver installed was + /// told to go and serve a model on a backend that cannot start — and under + /// the GPU-required policy there is nothing for it to fall back to. + #[test] + fn examine_render_does_not_promise_vulkan_on_a_host_that_has_none() { + let summary = gfx90c_summary_with_vulkan(Some(VulkanSupport { + status: VulkanStatus::Absent, + detail: Some("no Vulkan driver found".to_owned()), + ..VulkanSupport::default() + })); + + let rendered = summary.render_text(); + + assert!( + !rendered.contains("the Vulkan backend can be used"), + "a host with no Vulkan driver must not be promised one: {rendered}" + ); + assert!( + rendered.contains("no Vulkan GPU driver was found"), "{rendered}" ); + assert!( + rendered.contains("install one"), + "the advice must be able to clear the condition it is printed for: {rendered}" + ); + assert!(rendered.contains("vulkan: absent"), "{rendered}"); + } + + /// An installed driver that enumerated no device is a weaker claim than a + /// device that answered, and says so rather than borrowing the stronger + /// sentence. + #[test] + fn examine_render_separates_an_installed_driver_from_a_confirmed_device() { + let summary = gfx90c_summary_with_vulkan(Some(VulkanSupport { + status: VulkanStatus::DriverPresentDevicesUnverified, + detail: Some("Mesa RADV is installed, but no device was enumerated".to_owned()), + ..VulkanSupport::default() + })); + + let rendered = summary.render_text(); + + assert!( + rendered.contains("a Vulkan driver is installed, so the Vulkan backend is expected"), + "{rendered}" + ); + assert!( + rendered.contains("vulkan: driver_present_devices_unverified"), + "{rendered}" + ); + } + + /// A summary with no probe result at all — an older host's JSON read back + /// through `rocm remote doctor` — must not read as "no Vulkan". It keeps + /// the Vulkan advice, because withholding the only path such a host has on + /// the strength of a probe that never ran would be worse than the + /// assumption this change replaces. + #[test] + fn examine_render_treats_an_unprobed_summary_as_unconfirmed_not_absent() { + let summary = gfx90c_summary_with_vulkan(None); + + let rendered = summary.render_text(); + + assert!(rendered.contains("vulkan: "), "{rendered}"); + assert!( + rendered.contains("the Vulkan backend is unconfirmed"), + "{rendered}" + ); + assert!( + !rendered.contains("no Vulkan GPU driver was found"), + "an unprobed host is not a host without Vulkan: {rendered}" + ); + assert!( + rendered.contains("rocm serve --engine lemonade"), + "it is still worth trying, so the command stays: {rendered}" + ); } /// An unmapped target must not be *claimed* unsupported, even though every @@ -461,6 +644,7 @@ mod tests { detected_gfx_target: None, compatible_therock_family: None, detected_therock_family: None, + vulkan: None, driver: DriverSummary { policy: "linux_official_amd_dkms_wrapper".to_owned(), status: "amdgpu_available".to_owned(), @@ -515,6 +699,7 @@ mod tests { detected_gfx_target: None, compatible_therock_family: None, detected_therock_family: None, + vulkan: None, driver: DriverSummary { policy: "linux_official_amd_dkms_wrapper".to_owned(), status: "amdgpu_available".to_owned(), @@ -610,7 +795,7 @@ mod tests { .find(|l| l.trim_start().starts_with("rocm_support:")) .expect("every host gets a support line"); assert_eq!( - gpu_support_level(None, None), + gpu_support_level(None, None, VulkanStatus::Unknown), GpuSupportLevel::Unknown, "precondition: nothing detected is the Unknown tier" ); diff --git a/crates/rocm-core/src/host_gpu/vulkan.rs b/crates/rocm-core/src/host_gpu/vulkan.rs new file mode 100644 index 0000000000..cc8cdeeaf3 --- /dev/null +++ b/crates/rocm-core/src/host_gpu/vulkan.rs @@ -0,0 +1,686 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Whether this host can run a model through Vulkan. +//! +//! The CLI has always *asserted* this for a GPU with no ROCm build -- `examine` +//! told a gfx90c user "the Vulkan backend can still be used" without ever +//! looking. This probe is what makes that sentence evidence rather than a +//! promise, and it is what lets a host outside the ROCm compatibility matrix be +//! described by what it can do instead of only by what it cannot. +//! +//! Two sources, in increasing order of strength: +//! +//! 1. An **installable driver**: an ICD manifest on Linux (`/usr/share/vulkan/ +//! icd.d` and friends), or the Khronos registry key on Windows. This proves a +//! driver is *installed*, not that a device enumerates. +//! 2. An **enumerated device**: `vulkaninfo --summary`, when it is on PATH. +//! This proves a device actually answers. +//! +//! The two are kept apart by [`VulkanStatus`] rather than collapsed, because +//! collapsing them would replace one unverified promise with another. +//! +//! **A software rasteriser is not a GPU.** Mesa's lavapipe (`lvp_icd.*.json`, +//! `PHYSICAL_DEVICE_TYPE_CPU`) presents a complete Vulkan implementation that +//! runs entirely on the CPU. Counting it here would route a user onto CPU +//! execution while reporting a GPU path -- precisely the silent CPU fallback +//! `AGENTS.md` §6 forbids -- so it is recognised in order to be excluded, and +//! named in `detail` so the exclusion is visible rather than mysterious. + +use super::capture::capture_optional_command; +#[cfg(target_os = "linux")] +use crate::host_path; +use serde::{Deserialize, Serialize}; + +/// The directories a Linux Vulkan loader reads ICD manifests from. +/// +/// Read through [`host_path`] so a simulated host root can plant them, which is +/// what lets the no-GPU CI lane cover this probe at all. +#[cfg(target_os = "linux")] +const LINUX_ICD_DIRS: [&str; 4] = [ + "/usr/share/vulkan/icd.d", + "/usr/local/share/vulkan/icd.d", + "/etc/vulkan/icd.d", + "/usr/lib/x86_64-linux-gnu/vulkan/icd.d", +]; + +/// How sure the CLI is that a model can run through Vulkan here. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum VulkanStatus { + /// A Vulkan device answered and it is a real GPU. + Available, + /// A driver is installed, but no device was enumerated -- `vulkaninfo` is + /// not on PATH, so nothing confirmed a device answers. Likely to work; + /// not established. + DriverPresentDevicesUnverified, + /// The probe looked and found no GPU Vulkan driver. A software rasteriser + /// alone lands here, not in [`Self::Available`]. + Absent, + /// The probe could not look (an unsupported platform). + /// + /// The default: a value nobody has filled in is one nobody has probed, and + /// defaulting to `Absent` would let an unset field read as a fact about the + /// machine. + #[default] + Unknown, +} + +impl VulkanStatus { + /// Whether a model may be routed through Vulkan on the strength of this. + #[must_use] + pub const fn is_usable(self) -> bool { + matches!(self, Self::Available | Self::DriverPresentDevicesUnverified) + } +} + +/// One physical device as `vulkaninfo --summary` reported it. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct VulkanDevice { + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub driver_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub driver_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub api_version: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vendor_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub device_id: Option, + /// The raw `deviceType`, e.g. `PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub device_type: Option, +} + +impl VulkanDevice { + /// AMD's PCI vendor id, as `vulkaninfo` prints it. + const AMD_VENDOR_ID: &'static str = "0x1002"; + + #[must_use] + pub fn is_amd(&self) -> bool { + self.vendor_id + .as_deref() + .is_some_and(|id| id.eq_ignore_ascii_case(Self::AMD_VENDOR_ID)) + } + + /// Whether this device is real hardware rather than a software rasteriser. + /// + /// A missing `deviceType` reads as hardware: the field is optional in the + /// summary output, and treating an unreported type as CPU would discard a + /// working GPU. Lavapipe always reports its type, so the only entries this + /// lets through unverified are ones that named no type at all. + #[must_use] + pub fn is_hardware(&self) -> bool { + !self + .device_type + .as_deref() + .is_some_and(|kind| kind.to_ascii_uppercase().contains("TYPE_CPU")) + } +} + +/// What the host offers for Vulkan, and the evidence behind it. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct VulkanSupport { + pub status: VulkanStatus, + /// The ICD manifests or registry entries found, as the loader names them. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub icds: Vec, + /// The devices `vulkaninfo` enumerated, when it was available. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub devices: Vec, + /// A one-line human summary, including why anything was excluded. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +impl VulkanSupport { + /// The driver this host would use, for the `examine` line. + /// + /// Prefers what a device actually reported (`AMD proprietary driver`, + /// `radv`) over what an ICD filename implies, because the filename is a + /// guess about a vendor and the device's own `driverName` is not. + #[must_use] + pub fn driver_label(&self) -> Option { + self.devices + .iter() + .find(|device| device.is_amd() && device.is_hardware()) + .and_then(|device| device.driver_name.clone()) + .or_else(|| { + self.icds + .iter() + .find_map(|icd| icd_driver_label(icd)) + .map(str::to_owned) + }) + } +} + +/// The driver an ICD manifest's file name implies, or `None` when the name is +/// not one this table knows. +/// +/// Deliberately includes the software rasterisers so a caller can *exclude* +/// them by name; [`icd_is_gpu_driver`] is the predicate that does so. +#[must_use] +pub fn icd_driver_label(icd: &str) -> Option<&'static str> { + let name = icd + .rsplit(['/', '\\']) + .next() + .unwrap_or(icd) + .to_ascii_lowercase(); + if name.starts_with("radeon_icd") { + return Some("Mesa RADV"); + } + if name.starts_with("amd_icd") || name.starts_with("amdvlk") { + return Some("AMDVLK"); + } + if name.starts_with("lvp_icd") { + return Some("Mesa lavapipe (software)"); + } + if name.starts_with("vk_swiftshader") { + return Some("SwiftShader (software)"); + } + if name.starts_with("nvidia_icd") { + return Some("NVIDIA"); + } + if name.starts_with("intel_icd") || name.starts_with("intel_hasvk") { + return Some("Intel"); + } + None +} + +/// Whether an ICD manifest names a driver that runs on a GPU. +/// +/// An unrecognised manifest counts as a GPU driver: a vendor the table does not +/// list yet is far more likely to be real hardware than a new software +/// rasteriser, and the cost of the two errors is not symmetric -- missing a +/// working GPU path sends the user to CPU, while admitting an unknown one is +/// corrected by the device enumeration that follows. +#[must_use] +pub fn icd_is_gpu_driver(icd: &str) -> bool { + !icd_driver_label(icd).is_some_and(|label| label.contains("(software)")) +} + +/// Parse `vulkaninfo --summary` output into its physical devices. +/// +/// Tolerates the loader warnings `vulkaninfo` prints to stdout before the +/// report, and the `Instance Extensions` / `Instance Layers` sections ahead of +/// `Devices:` -- only `GPUn:` blocks and their indented `key = value` lines are +/// read. +#[must_use] +pub fn parse_vulkaninfo_summary(output: &str) -> Vec { + let mut devices = Vec::new(); + let mut current: Option = None; + for line in output.lines() { + let trimmed = line.trim(); + if trimmed.starts_with("GPU") && trimmed.ends_with(':') { + if let Some(device) = current.take() { + devices.push(device); + } + current = Some(VulkanDevice::default()); + continue; + } + let Some(device) = current.as_mut() else { + continue; + }; + let Some((key, value)) = trimmed.split_once('=') else { + // A non-indented, non-`key = value` line ends the block: the + // summary's device section is the last thing printed, but a caller + // may hand us trailing output. + if !trimmed.is_empty() && !line.starts_with([' ', '\t']) { + devices.push(current.take().unwrap_or_default()); + } + continue; + }; + let value = value.trim().to_owned(); + if value.is_empty() { + continue; + } + match key.trim() { + "deviceName" => device.name = value, + "driverName" => device.driver_name = Some(value), + "driverID" => device.driver_id = Some(value), + "apiVersion" => device.api_version = Some(value), + "vendorID" => device.vendor_id = Some(value), + "deviceID" => device.device_id = Some(value), + "deviceType" => device.device_type = Some(value), + _ => {} + } + } + if let Some(device) = current.take() { + devices.push(device); + } + devices.retain(|device| !device.name.is_empty()); + devices +} + +/// The ICD manifest paths listed under the Windows Khronos registry key, as +/// [`WINDOWS_VULKAN_ICD_SCRIPT`] prints them (one `ICD\t` row each). +#[must_use] +pub fn parse_windows_vulkan_icd_listing(output: &str) -> Vec { + output + .lines() + .filter_map(|line| line.trim().strip_prefix("ICD\t")) + .map(str::trim) + .filter(|path| !path.is_empty()) + .map(str::to_owned) + .collect() +} + +/// Enumerate both Khronos ICD keys. The WOW6432Node key carries the 32-bit +/// drivers; a 64-bit-only install lists nothing there, which is not an error. +#[cfg(any(windows, test))] +pub(super) const WINDOWS_VULKAN_ICD_SCRIPT: &str = r#"foreach ($key in 'HKLM:\SOFTWARE\Khronos\Vulkan\Drivers','HKLM:\SOFTWARE\WOW6432Node\Khronos\Vulkan\Drivers') { if (Test-Path $key) { foreach ($name in (Get-Item $key).GetValueNames()) { "ICD`t$name" } } }"#; + +/// Every Vulkan ICD manifest this host's loader would read. +#[cfg(target_os = "linux")] +fn detect_icds() -> Vec { + let mut icds = Vec::new(); + for dir in LINUX_ICD_DIRS { + let Ok(entries) = std::fs::read_dir(host_path(dir)) else { + continue; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.extension().is_some_and(|ext| ext == "json") + && let Some(name) = path.file_name().and_then(|name| name.to_str()) + { + icds.push(name.to_owned()); + } + } + } + icds.sort(); + icds.dedup(); + icds +} + +#[cfg(windows)] +fn detect_icds() -> Vec { + capture_optional_command( + "powershell", + &[ + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-Command", + WINDOWS_VULKAN_ICD_SCRIPT, + ], + ) + .map(|output| parse_windows_vulkan_icd_listing(&output)) + .unwrap_or_default() +} + +#[cfg(not(any(target_os = "linux", windows)))] +const fn detect_icds() -> Vec { + Vec::new() +} + +/// The devices `vulkaninfo --summary` reports, or an empty list when the tool +/// is not installed. Absence of the tool is not absence of a driver, which is +/// why this cannot decide [`VulkanStatus`] on its own. +fn detect_vulkan_devices() -> Vec { + capture_optional_command("vulkaninfo", &["--summary"]) + .map(|output| parse_vulkaninfo_summary(&output)) + .unwrap_or_default() +} + +/// Whether this platform's Vulkan surfaces can be read at all. +const fn vulkan_is_probeable() -> bool { + cfg!(any(target_os = "linux", windows)) +} + +/// What this host offers for Vulkan. +#[must_use] +pub fn detect_vulkan_support() -> VulkanSupport { + classify_vulkan_support( + detect_icds(), + detect_vulkan_devices(), + vulkan_is_probeable(), + ) +} + +/// The pure decision behind [`detect_vulkan_support`], split out so every +/// combination of evidence can be tested without a Vulkan install. +#[must_use] +pub fn classify_vulkan_support( + icds: Vec, + devices: Vec, + probeable: bool, +) -> VulkanSupport { + if !probeable { + return VulkanSupport { + status: VulkanStatus::Unknown, + icds, + devices, + detail: Some("Vulkan support is not probed on this platform".to_owned()), + }; + } + + let amd_gpu = devices + .iter() + .find(|device| device.is_amd() && device.is_hardware()); + if let Some(device) = amd_gpu { + let driver = device.driver_name.as_deref().unwrap_or("an unnamed driver"); + return VulkanSupport { + detail: Some(format!("{} through {driver}", device.name)), + status: VulkanStatus::Available, + icds, + devices, + }; + } + + let software_only = !devices.is_empty() && devices.iter().all(|device| !device.is_hardware()); + if software_only { + return VulkanSupport { + status: VulkanStatus::Absent, + detail: Some( + "only a software (CPU) Vulkan device was found; that is not a GPU path".to_owned(), + ), + icds, + devices, + }; + } + + let gpu_icds: Vec<&String> = icds.iter().filter(|icd| icd_is_gpu_driver(icd)).collect(); + if gpu_icds.is_empty() { + let detail = if icds.is_empty() { + "no Vulkan driver found".to_owned() + } else { + "only a software (CPU) Vulkan driver is installed; that is not a GPU path".to_owned() + }; + return VulkanSupport { + status: VulkanStatus::Absent, + detail: Some(detail), + icds, + devices, + }; + } + + let named = gpu_icds + .iter() + .find_map(|icd| icd_driver_label(icd)) + .map_or_else( + || "a Vulkan driver is installed".to_owned(), + |label| format!("{label} is installed"), + ); + VulkanSupport { + status: VulkanStatus::DriverPresentDevicesUnverified, + detail: Some(format!( + "{named}, but no device was enumerated (install `vulkaninfo` to confirm)" + )), + icds, + devices, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The real `vulkaninfo --summary` output from a Ryzen 7 4700U (gfx90c, + /// Renoir) on Windows 11 with Adrenalin 31.0.21925.1001 -- the machine + /// ROCMAI-517 is about. Note the loader warnings ahead of the report and + /// the instance sections before `Devices:`; both must be skipped. + const RENOIR_WINDOWS_SUMMARY: &str = "\ +WARNING: [Loader Message] Code 0 : windows_read_data_files_in_registry: Registry lookup failed to get layer manifest files. +========== +VULKANINFO +========== + +Vulkan Instance Version: 1.3.301 + + +Instance Extensions: count = 13 +------------------------------- +VK_EXT_debug_report : extension revision 10 +VK_KHR_surface : extension revision 25 + +Instance Layers: count = 1 +-------------------------- +VK_LAYER_AMD_switchable_graphics AMD switchable graphics layer 1.3.260 version 1 + +Devices: +======== +GPU0: +\tapiVersion = 1.3.260 +\tdriverVersion = 2.0.279 +\tvendorID = 0x1002 +\tdeviceID = 0x1636 +\tdeviceType = PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU +\tdeviceName = AMD Radeon(TM) Graphics +\tdriverID = DRIVER_ID_AMD_PROPRIETARY +\tdriverName = AMD proprietary driver +\tdriverInfo = 26.5.2 (AMD proprietary shader compiler) +\tconformanceVersion = 1.3.3.1 +"; + + #[test] + fn the_renoir_igpu_is_read_out_of_a_real_vulkaninfo_summary() { + let devices = parse_vulkaninfo_summary(RENOIR_WINDOWS_SUMMARY); + + assert_eq!(devices.len(), 1, "{devices:?}"); + let gpu = &devices[0]; + assert_eq!(gpu.name, "AMD Radeon(TM) Graphics"); + assert_eq!(gpu.vendor_id.as_deref(), Some("0x1002")); + // The same device id the gfx-target table maps to gfx90c. + assert_eq!(gpu.device_id.as_deref(), Some("0x1636")); + assert_eq!(gpu.driver_name.as_deref(), Some("AMD proprietary driver")); + assert_eq!(gpu.driver_id.as_deref(), Some("DRIVER_ID_AMD_PROPRIETARY")); + assert_eq!(gpu.api_version.as_deref(), Some("1.3.260")); + assert!(gpu.is_amd(), "vendor 0x1002 is AMD"); + assert!(gpu.is_hardware(), "an integrated GPU is hardware"); + } + + #[test] + fn a_real_renoir_host_reports_vulkan_available_and_names_its_driver() { + let devices = parse_vulkaninfo_summary(RENOIR_WINDOWS_SUMMARY); + + let support = classify_vulkan_support(Vec::new(), devices, true); + + assert_eq!(support.status, VulkanStatus::Available); + assert!(support.status.is_usable()); + assert_eq!( + support.driver_label().as_deref(), + Some("AMD proprietary driver"), + "the device's own driverName must win over any ICD filename guess" + ); + } + + /// The defect this guards against: lavapipe is a complete Vulkan + /// implementation that runs on the CPU. Reporting it as a usable Vulkan + /// path would route a GPU-required launch onto CPU execution while telling + /// the user a GPU path was chosen -- the silent CPU fallback AGENTS.md §6 + /// forbids, reached through a backend rather than a policy bug. + #[test] + fn a_software_rasteriser_is_never_a_usable_vulkan_path() { + let lavapipe = "\ +GPU0: +\tapiVersion = 1.3.255 +\tvendorID = 0x10005 +\tdeviceType = PHYSICAL_DEVICE_TYPE_CPU +\tdeviceName = llvmpipe (LLVM 15.0.7, 256 bits) +\tdriverName = llvmpipe +"; + let devices = parse_vulkaninfo_summary(lavapipe); + assert_eq!(devices.len(), 1); + assert!(!devices[0].is_hardware()); + + let support = + classify_vulkan_support(vec!["lvp_icd.x86_64.json".to_owned()], devices, true); + + assert_eq!( + support.status, + VulkanStatus::Absent, + "a CPU rasteriser must not read as a Vulkan GPU path: {support:?}" + ); + assert!(!support.status.is_usable()); + assert!( + support + .detail + .as_deref() + .is_some_and(|detail| detail.contains("not a GPU path")), + "{support:?}" + ); + } + + #[test] + fn a_lavapipe_only_install_is_absent_even_before_any_device_is_enumerated() { + let support = + classify_vulkan_support(vec!["lvp_icd.x86_64.json".to_owned()], Vec::new(), true); + + assert_eq!(support.status, VulkanStatus::Absent); + assert!(!icd_is_gpu_driver("lvp_icd.x86_64.json")); + assert!(icd_is_gpu_driver("radeon_icd.x86_64.json")); + } + + #[test] + fn an_installed_driver_with_no_enumerated_device_stays_unverified() { + let support = + classify_vulkan_support(vec!["radeon_icd.x86_64.json".to_owned()], Vec::new(), true); + + assert_eq!(support.status, VulkanStatus::DriverPresentDevicesUnverified); + assert!( + support.status.is_usable(), + "an installed GPU driver is still worth routing to" + ); + assert_eq!(support.driver_label().as_deref(), Some("Mesa RADV")); + assert!( + support + .detail + .as_deref() + .is_some_and(|detail| detail.contains("no device was enumerated")), + "the gap must be stated, not papered over: {support:?}" + ); + } + + #[test] + fn a_host_with_nothing_installed_is_absent_and_a_platform_that_cannot_look_is_unknown() { + let nothing = classify_vulkan_support(Vec::new(), Vec::new(), true); + assert_eq!(nothing.status, VulkanStatus::Absent); + assert_eq!(nothing.detail.as_deref(), Some("no Vulkan driver found")); + assert!(nothing.driver_label().is_none()); + + // macOS and friends: no evidence either way, which must not read as + // "no Vulkan" any more than it reads as "Vulkan works". + let unprobeable = classify_vulkan_support(Vec::new(), Vec::new(), false); + assert_eq!(unprobeable.status, VulkanStatus::Unknown); + assert!(!unprobeable.status.is_usable()); + } + + #[test] + fn an_unknown_vendors_icd_is_assumed_to_be_hardware() { + // A vendor the table does not list yet must not be mistaken for a + // software rasteriser; the device enumeration corrects it if wrong. + assert!(icd_is_gpu_driver("some_new_vendor_icd.x86_64.json")); + assert_eq!(icd_driver_label("some_new_vendor_icd.x86_64.json"), None); + + let support = classify_vulkan_support( + vec!["some_new_vendor_icd.x86_64.json".to_owned()], + Vec::new(), + true, + ); + assert_eq!(support.status, VulkanStatus::DriverPresentDevicesUnverified); + } + + #[test] + fn a_discrete_amd_card_is_preferred_over_a_software_device_in_the_same_report() { + let mixed = "\ +GPU0: +\tvendorID = 0x10005 +\tdeviceType = PHYSICAL_DEVICE_TYPE_CPU +\tdeviceName = llvmpipe +\tdriverName = llvmpipe +GPU1: +\tvendorID = 0x1002 +\tdeviceType = PHYSICAL_DEVICE_TYPE_DISCRETE_GPU +\tdeviceName = AMD Radeon RX 7900 XTX +\tdriverName = radv +"; + let support = classify_vulkan_support(Vec::new(), parse_vulkaninfo_summary(mixed), true); + + assert_eq!(support.status, VulkanStatus::Available); + assert_eq!(support.driver_label().as_deref(), Some("radv")); + assert!( + support + .detail + .as_deref() + .is_some_and(|detail| detail.contains("AMD Radeon RX 7900 XTX")), + "{support:?}" + ); + } + + #[test] + fn a_non_amd_gpu_does_not_make_vulkan_an_amd_path() { + let nvidia = "\ +GPU0: +\tvendorID = 0x10de +\tdeviceType = PHYSICAL_DEVICE_TYPE_DISCRETE_GPU +\tdeviceName = NVIDIA GeForce GTX 1050 Ti +\tdriverName = NVIDIA +"; + let support = classify_vulkan_support( + vec!["nvidia_icd.json".to_owned()], + parse_vulkaninfo_summary(nvidia), + true, + ); + + // The driver is installed and is not software, so the honest answer is + // "unverified for AMD", not "available". + assert_eq!(support.status, VulkanStatus::DriverPresentDevicesUnverified); + } + + #[test] + fn the_windows_registry_listing_is_read_into_icd_paths() { + let output = "ICD\tC:\\Windows\\System32\\DriverStore\\FileRepository\\u0397342.inf_amd64_x\\amdvlk64.json\nICD\tC:\\Windows\\SysWOW64\\amdvlk32.json\n"; + + let icds = parse_windows_vulkan_icd_listing(output); + + assert_eq!(icds.len(), 2, "{icds:?}"); + assert!(icds[0].ends_with("amdvlk64.json")); + assert_eq!(icd_driver_label(&icds[0]), Some("AMDVLK")); + assert!( + icd_is_gpu_driver(&icds[0]), + "a Windows path separator must not defeat the filename table" + ); + } + + /// Both Khronos keys must be queried. A 32-bit AMDVLK registration lives + /// only under `WOW6432Node`, so reading the 64-bit key alone would report + /// "no Vulkan driver" on a host that has one. + #[test] + fn the_windows_icd_query_reads_both_the_native_and_the_wow6432_key() { + assert!( + WINDOWS_VULKAN_ICD_SCRIPT.contains(r"HKLM:\SOFTWARE\Khronos\Vulkan\Drivers"), + "{WINDOWS_VULKAN_ICD_SCRIPT}" + ); + assert!( + WINDOWS_VULKAN_ICD_SCRIPT + .contains(r"HKLM:\SOFTWARE\WOW6432Node\Khronos\Vulkan\Drivers"), + "{WINDOWS_VULKAN_ICD_SCRIPT}" + ); + // A missing key is ordinary on a host with no Vulkan at all, so the + // script must test for it rather than let PowerShell error out. + assert!( + WINDOWS_VULKAN_ICD_SCRIPT.contains("Test-Path"), + "{WINDOWS_VULKAN_ICD_SCRIPT}" + ); + // The rows the parser consumes. + assert!( + WINDOWS_VULKAN_ICD_SCRIPT.contains("\"ICD`t$name\""), + "{WINDOWS_VULKAN_ICD_SCRIPT}" + ); + } + + #[test] + fn an_empty_registry_key_yields_no_icds_rather_than_a_blank_entry() { + assert!(parse_windows_vulkan_icd_listing("").is_empty()); + assert!(parse_windows_vulkan_icd_listing("ICD\t\n").is_empty()); + assert!(parse_windows_vulkan_icd_listing("unrelated output\n").is_empty()); + } + + #[test] + fn a_vulkaninfo_report_with_no_devices_yields_none() { + assert!(parse_vulkaninfo_summary("Vulkan Instance Version: 1.3.301\n").is_empty()); + assert!(parse_vulkaninfo_summary("").is_empty()); + } +} diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 28623e3ff1..9c64329a8e 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -137,12 +137,12 @@ pub use gfx_target::{ }; pub use hardware_root::host_path; pub use host_gpu::{ - DriverSummary, ExamineSummary, HostGpuSummary, WSL_ROCDXG_READY, WindowsGpuListing, - WslHostDriverProbe, WslSummary, default_engine_for_host, default_engine_for_platform, - detect_gpu_driver_version, detect_host_gfx_target, detect_host_gpu_diagnostics, - detect_host_gpu_summary, detect_system_ram_gib, has_usable_amd_gpu, is_amdgpu_device, - is_wsl_host, pinned_gpu_name, preferred_serve_engine_for_host_gpu_summary, - usable_amd_gpu_indices, windows_amd_gpu_listing, + DriverSummary, ExamineSummary, HostGpuSummary, VulkanDevice, VulkanStatus, VulkanSupport, + WSL_ROCDXG_READY, WindowsGpuListing, WslHostDriverProbe, WslSummary, default_engine_for_host, + default_engine_for_platform, detect_gpu_driver_version, detect_host_gfx_target, + detect_host_gpu_diagnostics, detect_host_gpu_summary, detect_system_ram_gib, + detect_vulkan_support, has_usable_amd_gpu, is_amdgpu_device, is_wsl_host, pinned_gpu_name, + preferred_serve_engine_for_host_gpu_summary, usable_amd_gpu_indices, windows_amd_gpu_listing, }; #[cfg(any(target_os = "linux", test))] use host_gpu::{KfdGpuNode, kfd_gpu_nodes_in}; diff --git a/docs/architecture.md b/docs/architecture.md index 124a86c6e7..c551f8a5d5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -202,7 +202,7 @@ Already-extracted subsystem modules, following the same full-domain-extraction p - [`gguf.rs`](../crates/rocm-core/src/gguf.rs) — reads the shape of a GGUF model (layers, KV heads, head width, parameter count, data offset) from its header over any `Read + Seek`, never touching the weights. - [`gguf_fit.rs`](../crates/rocm-core/src/gguf_fit.rs) — pure arithmetic over that header: weights + KV cache at a context size and KV type + compute buffer + runtime reserve against dedicated VRAM, the largest context that fits, and what would make a model that does not fit, fit. - [`gguf_source.rs`](../crates/rocm-core/src/gguf_source.rs) — where a header comes from: a local file, or a Hugging Face file, found by one repository-listing request (and a `HEAD` request for a file the listing does not size) and read by HTTP range requests (`HF_ENDPOINT` selects a mirror; redirects are followed by hand, up to five, and never from https to plain http). The only code in the estimate that reads a file or the network. -- [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) — (ROCMAI-85, Phase 7a of ROCMAI-27's sequencing) GPU/host hardware detection — see the module's own doc comment for what it owns. Its leaf concerns are child modules under `host_gpu/`, with `host_gpu.rs` kept as the module root: [`capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs) (running an optional tool under a timeout, trying each spelling [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs) gives for it), [`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) (kernel version, distro name, installed RAM, and the CPU model and vendor, which one reader returns together for both the summary and `Examination::probe`: Linux from `/proc/cpuinfo`, Windows from the CPU row of the system inventory), [`linux.rs`](../crates/rocm-core/src/host_gpu/linux.rs) (the sysfs, KFD and DRM probes), [`gpu_visibility.rs`](../crates/rocm-core/src/host_gpu/gpu_visibility.rs) (which AMD GPUs this process may use, from the KFD/DRM counts and the `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` masks, plus `windows_amd_gpu_listing`, the native-Windows adapter bound `rocm serve --gpu ` is checked against), [`windows_inventory.rs`](../crates/rocm-core/src/host_gpu/windows_inventory.rs) (the Windows display-adapter inventory and its parsers, including the AMD-adapter count behind `windows_amd_gpu_listing` and the system query whose CPU row the CPU reader in `os_facts.rs` takes), [`wsl.rs`](../crates/rocm-core/src/host_gpu/wsl.rs) (WSL detection, the librocdxg chain and the Windows host driver as seen from the guest), [`render.rs`](../crates/rocm-core/src/host_gpu/render.rs) (the `rocm examine` text of an `ExamineSummary`, as a second `impl` block), [`diagnostics.rs`](../crates/rocm-core/src/host_gpu/diagnostics.rs) (the GPU detection diagnostics text) and [`engine_default.rs`](../crates/rocm-core/src/host_gpu/engine_default.rs) (the default serve engine, and the vLLM preference for the TheRock families that serve through it). +- [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) — (ROCMAI-85, Phase 7a of ROCMAI-27's sequencing) GPU/host hardware detection — see the module's own doc comment for what it owns. Its leaf concerns are child modules under `host_gpu/`, with `host_gpu.rs` kept as the module root: [`capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs) (running an optional tool under a timeout, trying each spelling [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs) gives for it), [`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) (kernel version, distro name, installed RAM, and the CPU model and vendor, which one reader returns together for both the summary and `Examination::probe`: Linux from `/proc/cpuinfo`, Windows from the CPU row of the system inventory), [`linux.rs`](../crates/rocm-core/src/host_gpu/linux.rs) (the sysfs, KFD and DRM probes), [`gpu_visibility.rs`](../crates/rocm-core/src/host_gpu/gpu_visibility.rs) (which AMD GPUs this process may use, from the KFD/DRM counts and the `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` masks, plus `windows_amd_gpu_listing`, the native-Windows adapter bound `rocm serve --gpu ` is checked against), [`windows_inventory.rs`](../crates/rocm-core/src/host_gpu/windows_inventory.rs) (the Windows display-adapter inventory and its parsers, including the AMD-adapter count behind `windows_amd_gpu_listing` and the system query whose CPU row the CPU reader in `os_facts.rs` takes), [`wsl.rs`](../crates/rocm-core/src/host_gpu/wsl.rs) (WSL detection, the librocdxg chain and the Windows host driver as seen from the guest), [`render.rs`](../crates/rocm-core/src/host_gpu/render.rs) (the `rocm examine` text of an `ExamineSummary`, as a second `impl` block), [`vulkan.rs`](../crates/rocm-core/src/host_gpu/vulkan.rs) (whether a model can run through Vulkan here, read from ICD manifests, the Windows Khronos registry and `vulkaninfo --summary` — the evidence `gpu_support_level` in [`gfx_target.rs`](../crates/rocm-core/src/gfx_target.rs) needs to tell a GPU with no ROCm build that still has a Vulkan driver from one with no GPU path at all, and where a software rasteriser is recognised in order to be excluded), [`diagnostics.rs`](../crates/rocm-core/src/host_gpu/diagnostics.rs) (the GPU detection diagnostics text) and [`engine_default.rs`](../crates/rocm-core/src/host_gpu/engine_default.rs) (the default serve engine, and the vLLM preference for the TheRock families that serve through it). - [`host-probing.md`](host-probing.md) — a design note, not a module: [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) and [`examine.rs`](../crates/rocm-core/src/examine.rs) are two separate host-probing stacks that read some of the same facts twice (the Windows GPU query, the marketing-name table and the subprocess runners). It proposes which of each pair should survive, how the two differ and the order in which they could converge, and records what each convergence decided; it changes no code itself. Three pairs are converged: the `PATH` lookup ([`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs)'s `which` is now the only copy) and the kernel release ([`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs)'s `detect_kernel_version` is now the only `uname -r` reader, which [`host_env.rs`](../crates/rocm-core/src/examine/host_env.rs)'s `probe_os` calls for `kernel_release`, and `rocm examine --json` runs it once: [`examine.rs`](../crates/rocm-core/src/examine.rs)'s `Examination::probe_with_summary` probes the host and hands the examination's kernel release to the summary), and the CPU model ([`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) has one reader per platform for the model and vendor, which `probe_cpu_linux` and `probe_cpu_windows` record and the summary reads). It is a snapshot, to be retired when the convergence tickets close. - [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs) — the JSONL store engine behind the automation stores: the sibling `.lock` file (`jsonl_lock_path`, re-exported at the crate root), single-line appends, torn-tail repair, size-based rotation (`JsonlRotation`) and the tail-first readers that span rotated generations. It is generic over the record type; [`automation.rs`](../crates/rocm-core/src/automation.rs) supplies the record types and each store's policy, and nothing in this module depends on it, in production or in its own tests: those drive the engine with a small record type of their own. The module is private; its engine items are `pub(crate)`. - [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) — (ROCMAI-85, Phase 7a) the TheRock managed-runtime-environment surface — see the module's own doc comment for what it owns. diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index b0f1d34c59..0e38e084fc 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -736,3 +736,40 @@ Feature: GPU detection and system inspection Then the inspection states this host's ROCm support level And the support level agrees with the GPU the inspection detected And the support level leaves a command that fits this host + + # The three below close the gap this PR's own notes call unclosable: the + # `VulkanOnly` door needs a Renoir/Cezanne Vega APU, and no runner has one. + # It does not need the hardware, only the two signals the CLI reads for it -- + # a KFD node naming the target, and the host's Vulkan driver state -- both of + # which the harness can plant. So these run on the ungated mock lane, which is + # the lane that has neither. + # + # They exist because the support level used to be inferred: "no ROCm build" + # was taken to mean "Vulkan then", on a machine nobody had looked at. Each + # scenario pins one of the three answers that inference could not tell apart. + @id:examine-names-the-vulkan-driver-an-unsupported-gpu-can-use @requires-os:linux + Scenario: examine-49 - A GPU the official stack rejects is told which way it can still run models + Given a machine whose GPU has no official support but has a working Vulkan driver + When the user inspects that machine + Then the inspection reports the GPU as unsupported by the official stack + And the inspection reports that models can still run through Vulkan + And the inspection names the Vulkan driver it found + + @id:examine-does-not-promise-vulkan-without-a-driver @requires-os:linux + Scenario: examine-50 - A machine with no way to run models is not told it has one + Given a machine whose GPU has no official support and no Vulkan driver + When the user inspects that machine + Then the inspection reports the GPU as unsupported by the official stack + And the inspection does not claim models can run through Vulkan + And the inspection says no Vulkan driver was found + + # A software renderer is a complete Vulkan implementation that runs on the + # CPU. Counting it would put a GPU-required run on the processor while + # reporting a GPU path - the one outcome the no-fallback rule exists to + # prevent, reached through a backend rather than a policy change. + @id:examine-does-not-count-a-software-renderer-as-a-gpu-path @requires-os:linux + Scenario: examine-51 - A software renderer does not count as a way to run on the GPU + Given a machine whose GPU has no official support and only a software renderer + When the user inspects that machine + Then the inspection does not claim models can run through Vulkan + And the inspection says the only renderer it found runs on the CPU diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index f63af84819..6d92fc5769 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -1704,6 +1704,25 @@ async fn assert_no_amdgpu_fault_evidence(world: &mut E2eWorld) { } } +/// The human-readable twin of the machine-readable planted-machine step, for +/// scenarios whose subject is a sentence the report prints rather than a field +/// it serialises. +/// +/// It has to be its own step rather than reusing "the user inspects the +/// system", because that one runs through `run_rocm`, which does not consume +/// `World::command_env` -- so a planted host root never reaches the child and +/// the CLI silently describes the *runner* instead. Not hypothetical: it is +/// what made the first draft of examine-49/50/51 report the lane's own MI300X +/// as `gfx942` while claiming to be about a Renoir APU. +#[when("the user inspects that machine")] +async fn user_inspects_planted_machine_text(world: &mut E2eWorld) { + let (stdout, stderr, rc) = + crate::run_rocm_with_scenario_env(world, &["examine", "--framework", "skip"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + #[when("the user inspects that machine in machine-readable form")] async fn user_inspects_planted_machine(world: &mut E2eWorld) { let (stdout, stderr, rc) = @@ -3244,3 +3263,208 @@ async fn assert_support_level_advice_fits(world: &mut E2eWorld) { ); } } + +/// A KFD `gfx_target_version` naming gfx90c, the Vega iGPU in Renoir / +/// Lucienne / Cezanne / Barcelo APUs. +/// +/// The kernel packs the target as `major * 10000 + minor * 100 + step`, so +/// `9.0.12` is `90012` and decodes to `gfx90c` (step 12 is the hex `c`). This +/// is the one target the CLI recognizes *and* refuses to map to a ROCm package +/// family, so it is what reaches the support level these scenarios are about. +const KFD_GFX90C_TARGET_VERSION: &str = "90012"; + +/// `vulkaninfo --summary` on a host whose Vega iGPU is driven by Mesa RADV. +const RADV_VULKANINFO_SUMMARY: &str = "\ +Devices: +======== +GPU0: +\tapiVersion = 1.3.274 +\tvendorID = 0x1002 +\tdeviceID = 0x1636 +\tdeviceType = PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU +\tdeviceName = AMD Radeon Graphics (RADV RENOIR) +\tdriverID = DRIVER_ID_MESA_RADV +\tdriverName = radv +"; + +/// `vulkaninfo --summary` on a host with only Mesa's CPU rasteriser: a complete +/// Vulkan implementation, and not a GPU. +const LAVAPIPE_VULKANINFO_SUMMARY: &str = "\ +Devices: +======== +GPU0: +\tapiVersion = 1.3.255 +\tvendorID = 0x10005 +\tdeviceType = PHYSICAL_DEVICE_TYPE_CPU +\tdeviceName = llvmpipe (LLVM 15.0.7, 256 bits) +\tdriverName = llvmpipe +"; + +/// Plant a host whose GPU is identified but has no ROCm package family, +/// carrying the Vulkan state the caller describes. +/// +/// `icd_manifest` is the ICD file name the loader would find and +/// `vulkaninfo_summary` what `vulkaninfo --summary` prints; `None` for either +/// means that half of the evidence is absent. +/// +/// The `vulkaninfo` stub is planted in **both** directions. When a scenario +/// wants the tool absent it is planted as a failing command rather than left +/// out, because `PATH` still reaches the rest of the machine: a runner with a +/// real `vulkaninfo` installed would otherwise answer for it, and a scenario +/// about a host with no Vulkan driver has to be hermetic about the host and not +/// only about the planted root. +fn plant_unsupported_gpu( + world: &mut E2eWorld, + icd_manifest: Option<&str>, + vulkaninfo_summary: Option<&str>, +) { + let root = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .to_owned(); + let host_root = root.join("host-root"); + + let node = host_root.join("sys/class/kfd/kfd/topology/nodes/1"); + std::fs::create_dir_all(&node).expect("failed to create planted KFD node"); + std::fs::write( + node.join("properties"), + format!( + "cpu_cores_count 0\nsimd_count 56\ngfx_target_version {KFD_GFX90C_TARGET_VERSION}\n" + ), + ) + .expect("failed to write planted KFD node properties"); + + if let Some(manifest) = icd_manifest { + let icd_dir = host_root.join("usr/share/vulkan/icd.d"); + std::fs::create_dir_all(&icd_dir).expect("failed to create planted ICD dir"); + std::fs::write( + icd_dir.join(manifest), + "{\"file_format_version\":\"1.0.0\",\"ICD\":{\"library_path\":\"libvulkan_fake.so\"}}\n", + ) + .expect("failed to write planted ICD manifest"); + } + + let bin = root.join("fake-bin"); + std::fs::create_dir_all(&bin).expect("failed to create the fake PATH dir"); + // The same empty-bus `lspci` the other GPU fixtures plant: the probe must + // succeed without describing the runner's real cards. + rocm_core::fake_executable::write_fake_executable(&bin.join("lspci"), "#!/bin/sh\nexit 0\n") + .expect("failed to write the fake lspci"); + let vulkaninfo = match vulkaninfo_summary { + Some(summary) => format!("#!/bin/sh\ncat <<'ROCM_E2E_EOF'\n{summary}ROCM_E2E_EOF\n"), + None => "#!/bin/sh\nexit 1\n".to_owned(), + }; + rocm_core::fake_executable::write_fake_executable(&bin.join("vulkaninfo"), &vulkaninfo) + .expect("failed to write the fake vulkaninfo"); + + let path = std::env::join_paths(std::iter::once(bin).chain(std::env::split_paths( + &std::env::var_os("PATH").unwrap_or_default(), + ))) + .expect("PATH entries cannot hold a separator"); + world.command_env.push(("PATH", path)); + world + .command_env + .push(("ROCM_CLI_TEST_HOST_ROOT", host_root.into_os_string())); +} + +#[given("a machine whose GPU has no official support but has a working Vulkan driver")] +async fn unsupported_gpu_with_vulkan(world: &mut E2eWorld) { + plant_unsupported_gpu( + world, + Some("radeon_icd.x86_64.json"), + Some(RADV_VULKANINFO_SUMMARY), + ); +} + +#[given("a machine whose GPU has no official support and no Vulkan driver")] +async fn unsupported_gpu_without_vulkan(world: &mut E2eWorld) { + plant_unsupported_gpu(world, None, None); +} + +#[given("a machine whose GPU has no official support and only a software renderer")] +async fn unsupported_gpu_with_software_renderer(world: &mut E2eWorld) { + plant_unsupported_gpu( + world, + Some("lvp_icd.x86_64.json"), + Some(LAVAPIPE_VULKANINFO_SUMMARY), + ); +} + +#[then("the inspection reports the GPU as unsupported by the official stack")] +async fn assert_gpu_unsupported_by_rocm(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + assert!( + output.contains("rocm_support:") && output.contains("not supported by ROCm"), + "the inspection must say the GPU was identified and has no ROCm build:\n{output}" + ); +} + +#[then("the inspection reports that models can still run through Vulkan")] +async fn assert_vulkan_offered(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + assert!( + output.contains("the Vulkan backend can be used"), + "a host with a working Vulkan device must be told so:\n{output}" + ); + assert!( + output.contains("vulkan: available"), + "the machine-readable line must agree with the sentence:\n{output}" + ); +} + +#[then("the inspection names the Vulkan driver it found")] +async fn assert_vulkan_driver_named(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + // The driver the planted device reported, not a guess from the ICD file + // name: naming it is what lets a reader check the claim. + assert!( + output.contains("radv"), + "the inspection must name the driver it found:\n{output}" + ); +} + +#[then("the inspection does not claim models can run through Vulkan")] +async fn assert_vulkan_not_promised(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + assert!( + !output.contains("the Vulkan backend can be used"), + "a host with no usable Vulkan GPU must not be promised one:\n{output}" + ); + assert!( + !output.contains("Vulkan backend is expected to work"), + "a host with no usable Vulkan GPU must not be promised one:\n{output}" + ); +} + +#[then("the inspection says no Vulkan driver was found")] +async fn assert_no_vulkan_driver_reported(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + assert!( + output.contains("no Vulkan GPU driver was found"), + "the inspection must say why no GPU path is open:\n{output}" + ); + assert!( + output.contains("install one"), + "the advice must be able to clear the condition it is printed for:\n{output}" + ); +} + +#[then("the inspection says the only renderer it found runs on the CPU")] +async fn assert_software_renderer_reported(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + assert!( + output.contains("not a GPU path"), + "a CPU rasteriser must be named as such, not counted as a GPU:\n{output}" + ); + // The support level must agree, not just the evidence line. Without this the + // scenario still passed when `gpu_support_level` ignored the probe entirely: + // the absence of a Vulkan promise was coming from the status wording alone, + // so nothing here held the classification itself. + assert!( + output.contains("no Vulkan GPU driver was found"), + "a host whose only renderer is the CPU has no GPU backend, and the support \ + line must say so:\n{output}" + ); +} From 30144c525465bf04a4c28faa611e5ce5cd880eda Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 11:19:58 +0200 Subject: [PATCH 15/59] fix(engine): clear the endpoint key file for a child that gets no key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `engine_stdio_command` set `ROCM_SERVE_API_KEY_FILE` when the caller passed one and otherwise left the variable alone. `rocm` passes `None` for every external engine plugin — a third-party binary — so whenever this process had inherited the variable, the path to a mode-0600 file holding a live endpoint credential was handed straight through. Clear it on the `None` branch. Living in the shared stdio client rather than at the `rocm` call site means `rocmd`'s plugin calls get the same treatment. The test asserts CLEARED rather than absent: `Command` records `env_remove` as an explicit `(key, None)`, so a plain "is it there" check would pass with the clearing deleted. Verified by deleting it. Signed-off-by: Roman Sirokov --- .../rocm-engine-protocol/src/stdio_client.rs | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 5a8989e5a7..67511ad01d 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -155,6 +155,15 @@ where } if let Some(key_file) = options.endpoint_key_file { command.env(ENDPOINT_API_KEY_FILE_ENV, key_file); + } else { + // Cleared, not merely left unset. The caller said this child gets no + // endpoint key, and an engine plugin is a third-party binary: a value + // this process happens to have inherited must not reach it just + // because nothing overwrote it. `rocm`'s plugin calls pass `None` + // here, so this is what keeps a 0600 key-file path off their + // environment. See + // `an_external_plugin_child_is_not_handed_the_endpoint_key_file`. + command.env_remove(ENDPOINT_API_KEY_FILE_ENV); } if options.request_progress { command.env(ENGINE_PROGRESS_STDERR_ENV, "1"); @@ -675,6 +684,46 @@ mod tests { assert!(added_env(&command).is_empty(), "{:?}", added_env(&command)); } + /// A child told it gets no endpoint key must have the variable CLEARED, + /// not merely left unset. + /// + /// `rocm` passes `endpoint_key_file: None` for every external engine + /// plugin — a third-party binary — and the path it would otherwise carry + /// points at a mode-0600 file holding a live endpoint credential. Merely + /// not setting it would still hand the value through whenever this process + /// inherited one. + /// + /// Asserts the distinction directly: `Command` records `env_remove` as an + /// explicit `(key, None)` entry, so a plain "is it absent" check would pass + /// even with the clearing deleted. + #[test] + fn an_external_plugin_child_is_not_handed_the_endpoint_key_file() { + let key_var = std::ffi::OsStr::new(ENDPOINT_API_KEY_FILE_ENV); + let command = engine_stdio_command( + Path::new("/an/engine"), + ["stdio"], + &EngineStdioOptions { + app_paths: None, + env_root: None, + endpoint_key_file: None, + request_progress: false, + forward_stderr: false, + note_failing_exit_after_answer: false, + }, + ); + + let entry = command + .get_envs() + .find(|(key, _)| *key == key_var) + .map(|(_, value)| value); + assert_eq!( + entry, + Some(None), + "the key-file variable must be explicitly CLEARED when no key file \ + is passed, not merely left unset" + ); + } + #[test] fn each_option_sets_exactly_its_own_variable() { let paths = AppPaths { From d24a38d20840357431b47a99b585c461c605682b Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 11:20:52 +0200 Subject: [PATCH 16/59] docs(services): condense the device-probe doc blocks to clear the size gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `apps/rocm/src/services_text.rs` crossed the 1500-line module-size limit that ROCMAI-889 made blocking — main is at 1458 and this change's 164 added lines took it to 1512, so it is this PR's overflow, not an inherited one. Trimmed the two doc blocks the change itself added, which a reviewer had already called out as heavy. The load-bearing facts stay: the ~18s bound and where its arithmetic lives, why the chat tool pays it rather than rendering a device-less view, and what the probe seam does and does not cover. What went was restatement. Signed-off-by: Roman Sirokov --- apps/rocm/src/services_text.rs | 40 ++++++++++++---------------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index 3e9aa8b7aa..cdac966bb4 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -405,33 +405,23 @@ fn format_anthropic_endpoint_line(url: &str, responding: bool) -> String { /// `rocm services list`, and the chat `services` tool, which renders the same /// text. /// -/// **This is no longer a pure local read.** Since ROCMAI-550 it makes one live -/// `Healthcheck` RPC per *ready* service to fill the `device:` row, capped in -/// aggregate by `engine_ipc::SERVICE_DEVICE_PROBE_BUDGET`. Worst case is that -/// budget plus one in-flight probe — about 18s with the built-in engines, not -/// 10s, because the budget gates the *start* of each probe and never -/// interrupts one. That doc comment carries the arithmetic and why a -/// per-probe deadline was not taken. -/// -/// The chat tool pays the same bound deliberately rather than rendering a -/// device-less view: the two surfaces would then disagree about one service -/// while claiming to show the same list, and "which GPU is this running on" is -/// a question asked in chat at least as often as at the prompt. The cost is -/// bounded and flat in the number of services, which is what makes it -/// acceptable there. +/// **No longer a pure local read.** Since ROCMAI-550 it makes one live +/// `Healthcheck` RPC per *ready* service to fill the `device:` row, bounded in +/// aggregate by `engine_ipc::SERVICE_DEVICE_PROBE_BUDGET` — see that constant +/// for the arithmetic behind the ~18s worst case. The chat tool pays the same +/// bound rather than render a device-less view, which would have the two +/// surfaces disagree while claiming to show the same list. pub(crate) fn render_services_text(paths: &AppPaths, all: bool) -> Result { render_services_text_with_devices(paths, all, |ready| fetch_service_devices(paths, ready)) } /// [`render_services_text`] with the live device probe injected. /// -/// Split out because the probe is the one thing a unit test cannot have: it is -/// a `Healthcheck` RPC to a running engine, so in default CI (no GPU, no -/// servers) it answers nothing and the `device:` row is never rendered at all — -/// deleting the row would stay green everywhere except the GPU lanes, which is -/// exactly the gap this closes. The seam takes only the probe, not the row: -/// which services are offered to it (`status == "ready"`) and where the row -/// lands both stay in the shared body, so a test can pin them. +/// Split out because the probe is the one thing a unit test cannot have: in +/// default CI it answers nothing, so deleting the `device:` row would stay +/// green outside the GPU lanes. The seam takes only the probe — which services +/// are offered to it and where the row lands stay in the shared body, so a +/// test can pin both. fn render_services_text_with_devices( paths: &AppPaths, all: bool, @@ -465,11 +455,9 @@ fn render_services_text_with_devices( } else { writeln!(output, "No local servers are running.") }; - // Gate the separator on the same condition as the hint, not just on - // `!all`: `write_past_attempts_hint` early-returns at zero, so a host - // that has never served would otherwise gain a blank line between "No - // local servers are running." and the next line - a visible change to - // the one output this PR is not meant to touch. + // Gated on the same condition as the hint, not just `!all`: + // `write_past_attempts_hint` early-returns at zero, so a host that has + // never served would otherwise gain a stray blank line here. if !all && counts.past_attempts > 0 { write_past_attempts_hint( &mut output, From 6eef7b32144e329ec9a3a3e5a0fd5852e37ba594 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 13:04:21 +0200 Subject: [PATCH 17/59] docs(examine): correct the support-level section and move items above the tests `docs/testing.md` still described three support levels and pointed at `examine-39`, which is main's kfd scenario rather than anything here. It now names four levels, says why `Unknown` is distinct from the confirmed `NoGpuBackend`, cites `examine-48`, and lists the three Vulkan-tier scenarios with the lane they run on. A stale `examine-39` in `host_gpu/render.rs` is corrected too. `GpuSupportLevel` and its functions had drifted back below `#[cfg(test)] mod tests` in `gfx_target.rs`, with a `use` stranded mid-tests. Items above the test module, import hoisted to the module header. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/gfx_target.rs | 203 ++++++++++++------------ crates/rocm-core/src/host_gpu/render.rs | 2 +- docs/testing.md | 24 +-- 3 files changed, 116 insertions(+), 113 deletions(-) diff --git a/crates/rocm-core/src/gfx_target.rs b/crates/rocm-core/src/gfx_target.rs index 391c145fd3..8b843bb6b0 100644 --- a/crates/rocm-core/src/gfx_target.rs +++ b/crates/rocm-core/src/gfx_target.rs @@ -551,9 +551,110 @@ pub(crate) fn gfx_target_from_gc_version(major: u32, minor: u32, revision: u32) Some(format!("gfx{major}{minor:x}{revision:x}")) } +/// How well this host's GPU is supported, for messages that tell the user what +/// they can actually do rather than just reporting raw detection fields. +/// +/// `Unknown` deliberately covers both "nothing detected" and "a gfx target we +/// don't recognize" rather than being called `Unsupported` — see the doc +/// comment on [`gfx_target_has_no_rocm_build`] for why conflating "confirmed +/// no ROCm build" with "unrecognized" would turn a detection gap into a false +/// claim about the hardware. Only that predicate's conjunction with +/// [`normalize_therock_family`] establishes the "confirmed" half; an +/// unrecognized target satisfies neither conjunct, so it is a gap in what this +/// catalog knows rather than a fact about the card. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuSupportLevel { + /// A TheRock package family covers this GPU: full ROCm/HIP support. + RocmHip, + /// Recognized, confirmed to have no ROCm/HIP build, and a Vulkan GPU driver + /// was **found on this host**, so Vulkan-based engines (the default Lemonade + /// backend) run on it. + /// + /// The second half of that sentence is evidence, not inference. This variant + /// used to be returned from the gfx table alone — "no ROCm build" was taken + /// to mean "Vulkan then", on a machine nobody had looked at. Every surface + /// that renders this level tells the user to go and serve a model on it, so + /// a host with no Vulkan driver installed, or with only a software + /// rasteriser, was sent to a backend that cannot start. + VulkanOnly, + /// Recognized, confirmed to have no ROCm/HIP build, **and** no Vulkan GPU + /// driver was found: this host has no GPU path at all. + /// + /// Deliberately not folded into [`Self::Unknown`]. Both halves here are + /// confirmed — the catalog establishes the missing ROCm build, and the + /// Vulkan probe looked and found nothing — whereas `Unknown` means the CLI + /// could not establish either. Collapsing them would commit, in the other + /// direction, the exact conflation this enum's `Unknown` exists to prevent: + /// a fact about the machine would be reported as a gap in what this build + /// knows, and the user would be sent to probe a question already answered. + NoGpuBackend, + /// No GPU detected yet, or a gfx target this catalog doesn't recognize. + Unknown, +} + +/// Classify the host's GPU support level. +/// +/// Reads the two facts `ExamineSummary` and `resolve_family` already compute — +/// the detected gfx target and whichever TheRock family (if any) covers it — +/// plus what the Vulkan probe found. +/// +/// `vulkan` only ever decides between [`GpuSupportLevel::VulkanOnly`] and +/// [`GpuSupportLevel::NoGpuBackend`], and is consulted only once the catalog has +/// confirmed there is no ROCm build. A host with a covering family is `RocmHip` +/// whatever Vulkan reports, and an unrecognized target stays `Unknown`: finding +/// a Vulkan driver says nothing about whether ROCm would also work there, so it +/// must not be allowed to turn a detection gap into a support claim. +/// +/// [`VulkanStatus::Unknown`] — the platform could not be probed at all — keeps +/// `VulkanOnly`, because withholding the one path such a host has on the +/// strength of a probe that never ran would be worse than the old assumption, +/// not better. +pub fn gpu_support_level( + detected_gfx_target: Option<&str>, + compatible_therock_family: Option<&str>, + vulkan: crate::host_gpu::VulkanStatus, +) -> GpuSupportLevel { + match (detected_gfx_target, compatible_therock_family) { + (_, Some(_)) => GpuSupportLevel::RocmHip, + (Some(gfx), None) if gfx_target_has_no_rocm_build(gfx) => { + if vulkan.is_usable() || vulkan == crate::host_gpu::VulkanStatus::Unknown { + GpuSupportLevel::VulkanOnly + } else { + GpuSupportLevel::NoGpuBackend + } + } + _ => GpuSupportLevel::Unknown, + } +} + +/// [`gpu_support_level`] for a [`HostGpuSummary`] the caller already probed. +/// +/// The two fields are read here, once, rather than at each call site: they are +/// positional arguments of the same type on [`gpu_support_level`], so a call +/// site that transposed them would classify a Vulkan-only host as `RocmHip` +/// and still compile. `rocm engines` and the codex-bridge `engines` tool both +/// derive the level from this struct, so both go through this. +/// +/// `vulkan` is passed in rather than probed here so this stays a pure function +/// of what the caller already holds: the probe runs a subprocess and reads the +/// filesystem, and burying that in a helper this cheap would make every caller +/// pay for it without saying so. +#[must_use] +pub fn gpu_support_level_for_host_gpu( + host_gpu: &HostGpuSummary, + vulkan: crate::host_gpu::VulkanStatus, +) -> GpuSupportLevel { + gpu_support_level( + host_gpu.gfx_target.as_deref(), + host_gpu.therock_family.as_deref(), + vulkan, + ) +} + #[cfg(test)] mod tests { use super::*; + use crate::host_gpu::VulkanStatus; #[test] fn normalize_therock_family_maps_gfx1101_to_gfx110x_all() { @@ -972,8 +1073,6 @@ mod tests { ); } - use crate::host_gpu::VulkanStatus; - #[test] fn gpu_support_level_is_rocm_hip_when_a_family_covers_it() { assert_eq!( @@ -1051,103 +1150,3 @@ mod tests { ); } } - -/// How well this host's GPU is supported, for messages that tell the user what -/// they can actually do rather than just reporting raw detection fields. -/// -/// `Unknown` deliberately covers both "nothing detected" and "a gfx target we -/// don't recognize" rather than being called `Unsupported` — see the doc -/// comment on [`gfx_target_has_no_rocm_build`] for why conflating "confirmed -/// no ROCm build" with "unrecognized" would turn a detection gap into a false -/// claim about the hardware. Only that predicate's conjunction with -/// [`normalize_therock_family`] establishes the "confirmed" half; an -/// unrecognized target satisfies neither conjunct, so it is a gap in what this -/// catalog knows rather than a fact about the card. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum GpuSupportLevel { - /// A TheRock package family covers this GPU: full ROCm/HIP support. - RocmHip, - /// Recognized, confirmed to have no ROCm/HIP build, and a Vulkan GPU driver - /// was **found on this host**, so Vulkan-based engines (the default Lemonade - /// backend) run on it. - /// - /// The second half of that sentence is evidence, not inference. This variant - /// used to be returned from the gfx table alone — "no ROCm build" was taken - /// to mean "Vulkan then", on a machine nobody had looked at. Every surface - /// that renders this level tells the user to go and serve a model on it, so - /// a host with no Vulkan driver installed, or with only a software - /// rasteriser, was sent to a backend that cannot start. - VulkanOnly, - /// Recognized, confirmed to have no ROCm/HIP build, **and** no Vulkan GPU - /// driver was found: this host has no GPU path at all. - /// - /// Deliberately not folded into [`Self::Unknown`]. Both halves here are - /// confirmed — the catalog establishes the missing ROCm build, and the - /// Vulkan probe looked and found nothing — whereas `Unknown` means the CLI - /// could not establish either. Collapsing them would commit, in the other - /// direction, the exact conflation this enum's `Unknown` exists to prevent: - /// a fact about the machine would be reported as a gap in what this build - /// knows, and the user would be sent to probe a question already answered. - NoGpuBackend, - /// No GPU detected yet, or a gfx target this catalog doesn't recognize. - Unknown, -} - -/// Classify the host's GPU support level. -/// -/// Reads the two facts `ExamineSummary` and `resolve_family` already compute — -/// the detected gfx target and whichever TheRock family (if any) covers it — -/// plus what the Vulkan probe found. -/// -/// `vulkan` only ever decides between [`GpuSupportLevel::VulkanOnly`] and -/// [`GpuSupportLevel::NoGpuBackend`], and is consulted only once the catalog has -/// confirmed there is no ROCm build. A host with a covering family is `RocmHip` -/// whatever Vulkan reports, and an unrecognized target stays `Unknown`: finding -/// a Vulkan driver says nothing about whether ROCm would also work there, so it -/// must not be allowed to turn a detection gap into a support claim. -/// -/// [`VulkanStatus::Unknown`] — the platform could not be probed at all — keeps -/// `VulkanOnly`, because withholding the one path such a host has on the -/// strength of a probe that never ran would be worse than the old assumption, -/// not better. -pub fn gpu_support_level( - detected_gfx_target: Option<&str>, - compatible_therock_family: Option<&str>, - vulkan: crate::host_gpu::VulkanStatus, -) -> GpuSupportLevel { - match (detected_gfx_target, compatible_therock_family) { - (_, Some(_)) => GpuSupportLevel::RocmHip, - (Some(gfx), None) if gfx_target_has_no_rocm_build(gfx) => { - if vulkan.is_usable() || vulkan == crate::host_gpu::VulkanStatus::Unknown { - GpuSupportLevel::VulkanOnly - } else { - GpuSupportLevel::NoGpuBackend - } - } - _ => GpuSupportLevel::Unknown, - } -} - -/// [`gpu_support_level`] for a [`HostGpuSummary`] the caller already probed. -/// -/// The two fields are read here, once, rather than at each call site: they are -/// positional arguments of the same type on [`gpu_support_level`], so a call -/// site that transposed them would classify a Vulkan-only host as `RocmHip` -/// and still compile. `rocm engines` and the codex-bridge `engines` tool both -/// derive the level from this struct, so both go through this. -/// -/// `vulkan` is passed in rather than probed here so this stays a pure function -/// of what the caller already holds: the probe runs a subprocess and reads the -/// filesystem, and burying that in a helper this cheap would make every caller -/// pay for it without saying so. -#[must_use] -pub fn gpu_support_level_for_host_gpu( - host_gpu: &HostGpuSummary, - vulkan: crate::host_gpu::VulkanStatus, -) -> GpuSupportLevel { - gpu_support_level( - host_gpu.gfx_target.as_deref(), - host_gpu.therock_family.as_deref(), - vulkan, - ) -} diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index df88230d12..c97f075f47 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -348,7 +348,7 @@ mod tests { /// The exact placeholders the report prints for the three GPU fields when /// it has no value, pinned because they are read back as data. /// - /// examine-39 decides which support tier a host is in by filtering these + /// examine-48 decides which support tier a host is in by filtering these /// out of `detected_gfx_target`, `compatible_therock_family` and /// `detected_therock_family`. Reword one and that scenario stops seeing an /// absent value as absent: a lane would be classified into the wrong tier diff --git a/docs/testing.md b/docs/testing.md index e7e63d7604..e64ab4cdde 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -3882,16 +3882,20 @@ row is the backend and ordinal with no name — correct and unspecific, rather than specific and wrong. No CI lane has two AMD GPUs, so that branch is covered by unit tests in [host_gpu.rs](../crates/rocm-core/src/host_gpu.rs) only. -**`rocm examine` prints a `rocm_support` line on every host**, at one of three -levels (`gpu_support_level`): ROCm/HIP-supported, Vulkan-only, or unknown. -Unknown covers both "nothing detected" and "a target this build's catalog does -not recognise" — conflating the latter with "confirmed to have no ROCm build" -would turn a detection gap into a claim about the hardware. `examine-39` holds -the line against the `detected_gfx_target` and `compatible_therock_family` the -same report prints, so every lane checks the level its own hardware implies: - -```bash -cargo xtask e2e -- -n '(examine-39)' +**`rocm examine` prints a `rocm_support` line on every host**, at one of four +levels (`gpu_support_level`): ROCm/HIP-supported, Vulkan-only, no GPU backend, +or unknown. Unknown covers both "nothing detected" and "a target this build's +catalog does not recognise" — conflating the latter with "confirmed to have no +ROCm build" would turn a detection gap into a claim about the hardware, which +is why it is distinct from the `NoGpuBackend` level that *is* a confirmed +answer. `examine-48` holds the line against the `detected_gfx_target` and +`compatible_therock_family` the same report prints, so every lane checks the +level its own hardware implies; `examine-49`, `examine-50` and `examine-51` +cover the Vulkan tiers (a driver present, none present, and a software +renderer that must not count as a GPU path) and are `@requires-os:linux`: + +```bash +cargo xtask e2e -- -n '(examine-4[89]|examine-5[01])' ``` ## Windows Tool Notes From 4ff8d08c2cd43825a794f96b81ebf3449319f2bc Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 13:45:02 +0200 Subject: [PATCH 18/59] fix(engine,examine): scope the key-file clearing, and stop claiming Vulkan is absent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three review findings, all correctness rather than wording. The endpoint-key clearing was added to the SHARED stdio builder, so it reached `rocmd` too — and `rocmd` passes `endpoint_key_file: None` for a loopback service that simply needs no key, which is not the same statement as "this child may not see one". Its own test expects a keyless child to inherit the ambient value, and only passed because the variable happened to be unset. The withholding is now an explicit option the caller asks for: `rocm` sets it for external plugins, `rocmd` does not. Both directions are asserted. The Vulkan probe reported `Absent` when it found no ICD manifest and enumerated no device — but neither source is exhaustive (the Linux search ignores `VK_ICD_FILENAMES`, Windows reads only the legacy Khronos key, and `vulkaninfo` is often not installed), and `Absent` renders as a confirmed "no Vulkan GPU driver was found - install one". A host with a working driver could be told to install one it already has. No evidence either way is now `Unknown`; `Absent` is kept for a probe that found something and ruled it out. The support-level scenario required `rocm serve --engine lemonade` for any detected GPU with no covering family, but a `NoGpuBackend` host is given no serve command at all. Since the scenario is untagged it would have failed on exactly that hardware. It branches on the rendered tier now, as the renderer does. Scenario ids shift again after merging main (which took examine-48): examine-49..52 and the references that cite them. Signed-off-by: Roman Sirokov --- apps/rocm/src/engine_ipc.rs | 3 + apps/rocmd/src/engine_ipc.rs | 4 ++ crates/rocm-core/src/host_gpu/render.rs | 2 +- crates/rocm-core/src/host_gpu/vulkan.rs | 58 ++++++++++++++++--- .../rocm-engine-protocol/src/stdio_client.rs | 47 ++++++++++++--- docs/testing.md | 6 +- tests/e2e-cucumber/tests/e2e/examine_steps.rs | 11 +++- 7 files changed, 110 insertions(+), 21 deletions(-) diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index ff9793975e..b987168589 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -222,6 +222,9 @@ const fn stdio_options<'a>( app_paths: paths, env_root, endpoint_key_file: None, + // An external plugin is a third-party binary: never hand it a path to + // a live endpoint credential, including one we inherited ourselves. + withhold_inherited_key_file: true, request_progress: stream_progress, forward_stderr: true, note_failing_exit_after_answer: false, diff --git a/apps/rocmd/src/engine_ipc.rs b/apps/rocmd/src/engine_ipc.rs index f4749c3eb2..eda8243ac2 100644 --- a/apps/rocmd/src/engine_ipc.rs +++ b/apps/rocmd/src/engine_ipc.rs @@ -211,6 +211,10 @@ const fn stdio_options<'a>( app_paths: Some(paths), env_root: None, endpoint_key_file, + // `None` here means "this loopback service needs no key", not "this + // child may not see one" — these are the daemon's own children, and + // clearing would change what they inherit. + withhold_inherited_key_file: false, request_progress: false, forward_stderr: false, note_failing_exit_after_answer: true, diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index c97f075f47..721b6ed6a3 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -348,7 +348,7 @@ mod tests { /// The exact placeholders the report prints for the three GPU fields when /// it has no value, pinned because they are read back as data. /// - /// examine-48 decides which support tier a host is in by filtering these + /// examine-49 decides which support tier a host is in by filtering these /// out of `detected_gfx_target`, `compatible_therock_family` and /// `detected_therock_family`. Reword one and that scenario stops seeing an /// absent value as absent: a lane would be classified into the wrong tier diff --git a/crates/rocm-core/src/host_gpu/vulkan.rs b/crates/rocm-core/src/host_gpu/vulkan.rs index cc8cdeeaf3..b1e2b08913 100644 --- a/crates/rocm-core/src/host_gpu/vulkan.rs +++ b/crates/rocm-core/src/host_gpu/vulkan.rs @@ -383,14 +383,33 @@ pub fn classify_vulkan_support( let gpu_icds: Vec<&String> = icds.iter().filter(|icd| icd_is_gpu_driver(icd)).collect(); if gpu_icds.is_empty() { - let detail = if icds.is_empty() { - "no Vulkan driver found".to_owned() - } else { - "only a software (CPU) Vulkan driver is installed; that is not a GPU path".to_owned() - }; + // No ICD *and* no enumerated device is an absence of evidence, not + // evidence of absence: neither source is exhaustive. The Linux ICD + // search is a fixed directory list that ignores `VK_ICD_FILENAMES` and + // `VK_DRIVER_FILES`; the Windows one reads only the legacy Khronos + // registry key, which current AMD drivers need not populate; and + // `vulkaninfo` is frequently not installed on either. A working driver + // can therefore leave both empty, and `Absent` is rendered to the user + // as a confirmed fact ("no Vulkan GPU driver was found - install one"). + // Say we could not tell instead, and keep `Absent` for the cases below + // where something was actually found and ruled out. + if icds.is_empty() && devices.is_empty() { + return VulkanSupport { + status: VulkanStatus::Unknown, + detail: Some( + "no Vulkan driver was found, but this host exposes nothing that confirms one is absent either (no ICD manifest, and `vulkaninfo` did not enumerate)" + .to_owned(), + ), + icds, + devices, + }; + } return VulkanSupport { status: VulkanStatus::Absent, - detail: Some(detail), + detail: Some( + "only a software (CPU) Vulkan driver is installed; that is not a GPU path" + .to_owned(), + ), icds, devices, }; @@ -553,13 +572,34 @@ GPU0: ); } + /// No ICD and no enumerated device is an absence of evidence, not evidence + /// of absence — neither source is exhaustive (the Linux ICD search ignores + /// `VK_ICD_FILENAMES`, the Windows one reads only the legacy registry key, + /// and `vulkaninfo` is often not installed). `Absent` renders as a + /// confirmed "no Vulkan GPU driver was found - install one", so a host with + /// a working driver would be told to install one it already has. #[test] - fn a_host_with_nothing_installed_is_absent_and_a_platform_that_cannot_look_is_unknown() { + fn no_evidence_either_way_is_not_reported_as_a_confirmed_absence() { let nothing = classify_vulkan_support(Vec::new(), Vec::new(), true); - assert_eq!(nothing.status, VulkanStatus::Absent); - assert_eq!(nothing.detail.as_deref(), Some("no Vulkan driver found")); + assert_eq!( + nothing.status, + VulkanStatus::Unknown, + "an empty probe must not claim the driver is absent" + ); + assert!( + nothing + .detail + .as_deref() + .is_some_and(|detail| detail.contains("confirms")), + "the detail must say the absence is unconfirmed: {nothing:?}" + ); assert!(nothing.driver_label().is_none()); + // What `Absent` is now reserved for: something WAS found and ruled out. + let software_only = + classify_vulkan_support(vec!["lvp_icd.x86_64.json".to_owned()], Vec::new(), true); + assert_eq!(software_only.status, VulkanStatus::Absent); + // macOS and friends: no evidence either way, which must not read as // "no Vulkan" any more than it reads as "Vulkan works". let unprobeable = classify_vulkan_support(Vec::new(), Vec::new(), false); diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 67511ad01d..2dd776d15b 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -108,6 +108,15 @@ pub struct EngineStdioOptions<'a> { /// its HTTP probe of a protected endpoint authenticates. The caller decides /// whether a file holds a usable key; this only carries it. pub endpoint_key_file: Option<&'a Path>, + /// With no `endpoint_key_file`, also CLEAR [`ENDPOINT_API_KEY_FILE_ENV`] + /// rather than leave whatever this process inherited to pass through. + /// + /// For a child that must not see a credential path at all — `rocm`'s + /// external engine plugins are third-party binaries. Off by default, + /// because "no key file for this request" and "this child may not have one" + /// are different statements: `rocmd` passes `None` for a loopback service + /// that simply needs no key, and its children are its own. + pub withhold_inherited_key_file: bool, /// Ask the engine for progress on stderr ([`ENGINE_PROGRESS_STDERR_ENV`]). pub request_progress: bool, /// Write the child's stderr to this process's stderr as it arrives, so a @@ -155,13 +164,12 @@ where } if let Some(key_file) = options.endpoint_key_file { command.env(ENDPOINT_API_KEY_FILE_ENV, key_file); - } else { - // Cleared, not merely left unset. The caller said this child gets no - // endpoint key, and an engine plugin is a third-party binary: a value - // this process happens to have inherited must not reach it just - // because nothing overwrote it. `rocm`'s plugin calls pass `None` - // here, so this is what keeps a 0600 key-file path off their - // environment. See + } else if options.withhold_inherited_key_file { + // Cleared, not merely left unset: a value this process inherited must + // not reach a third-party plugin binary just because nothing + // overwrote it. Only for callers that ask — `rocmd` passes `None` for + // a loopback service that needs no key, and clearing there would + // change what its own children see. See // `an_external_plugin_child_is_not_handed_the_endpoint_key_file`. command.env_remove(ENDPOINT_API_KEY_FILE_ENV); } @@ -706,6 +714,7 @@ mod tests { app_paths: None, env_root: None, endpoint_key_file: None, + withhold_inherited_key_file: true, request_progress: false, forward_stderr: false, note_failing_exit_after_answer: false, @@ -722,6 +731,27 @@ mod tests { "the key-file variable must be explicitly CLEARED when no key file \ is passed, not merely left unset" ); + + // The other direction, which is what keeps this scoped: a caller that + // did not ask (`rocmd`, for a loopback service that simply needs no + // key) must leave the child's inherited value alone. + let kept = engine_stdio_command( + Path::new("/an/engine"), + ["stdio"], + &EngineStdioOptions { + app_paths: None, + env_root: None, + endpoint_key_file: None, + withhold_inherited_key_file: false, + request_progress: false, + forward_stderr: false, + note_failing_exit_after_answer: false, + }, + ); + assert!( + kept.get_envs().all(|(key, _)| key != key_var), + "a caller that did not ask to withhold must not touch the variable" + ); } #[test] @@ -738,6 +768,7 @@ mod tests { app_paths: Some(&paths), env_root: Some(Path::new("/envs")), endpoint_key_file: Some(Path::new("/keys/svc.endpoint-key")), + withhold_inherited_key_file: false, request_progress: true, forward_stderr: true, note_failing_exit_after_answer: false, @@ -771,6 +802,7 @@ mod tests { "ROCM_SERVE_API_KEY_FILE", EngineStdioOptions { endpoint_key_file: Some(Path::new("/keys/k")), + withhold_inherited_key_file: false, ..EngineStdioOptions::default() }, ), @@ -1192,6 +1224,7 @@ mod tests { app_paths: Some(&paths), env_root: Some(Path::new("/envs")), endpoint_key_file: Some(Path::new("/keys/k")), + withhold_inherited_key_file: false, request_progress: true, forward_stderr: false, note_failing_exit_after_answer: false, diff --git a/docs/testing.md b/docs/testing.md index 696bf8d7d9..eb48cce036 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -3895,14 +3895,14 @@ or unknown. Unknown covers both "nothing detected" and "a target this build's catalog does not recognise" — conflating the latter with "confirmed to have no ROCm build" would turn a detection gap into a claim about the hardware, which is why it is distinct from the `NoGpuBackend` level that *is* a confirmed -answer. `examine-48` holds the line against the `detected_gfx_target` and +answer. `examine-49` holds the line against the `detected_gfx_target` and `compatible_therock_family` the same report prints, so every lane checks the -level its own hardware implies; `examine-49`, `examine-50` and `examine-51` +level its own hardware implies; `examine-50`, `examine-51` and `examine-52` cover the Vulkan tiers (a driver present, none present, and a software renderer that must not count as a GPU path) and are `@requires-os:linux`: ```bash -cargo xtask e2e -- -n '(examine-4[89]|examine-5[01])' +cargo xtask e2e -- -n '(examine-49|examine-5[012])' ``` ## Windows Tool Notes diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 245cb11f3b..a5bc2b38c3 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -1712,7 +1712,7 @@ async fn assert_no_amdgpu_fault_evidence(world: &mut E2eWorld) { /// system", because that one runs through `run_rocm`, which does not consume /// `World::command_env` -- so a planted host root never reaches the child and /// the CLI silently describes the *runner* instead. Not hypothetical: it is -/// what made the first draft of examine-49/50/51 report the lane's own MI300X +/// what made the first draft of examine-50/51/52 report the lane's own MI300X /// as `gfx942` while claiming to be about a Renoir APU. #[when("the user inspects that machine")] async fn user_inspects_planted_machine_text(world: &mut E2eWorld) { @@ -3331,6 +3331,15 @@ async fn assert_support_level_advice_fits(world: &mut E2eWorld) { (_, Some(_), Some(_)) => ("rocm serve ", &["rocm install sdk"]), // ROCm-capable with no managed SDK yet: that install is the next step. (_, Some(_), None) => ("rocm install sdk", &[]), + // A GPU no package family covers, and the Vulkan probe found no GPU + // driver either: there is no backend to serve on, so the line owes a + // way to GET one rather than a serve command that would be refused. + // Branched on the rendered tier the way the renderer itself is — a + // flat expectation here failed on exactly this hardware, since this + // scenario is untagged and runs on every lane. + (Some(_), None, _) if line.contains("no Vulkan GPU driver was found") => { + ("install one", &["rocm serve", "rocm install sdk"]) + } // A GPU no package family covers — confirmed Vulkan-only (gfx90c) or // merely unrecognized. Either way the SDK has nothing for it and // Lemonade's Vulkan backend is what can serve on it. From c837335f7f0e81f94ef9e911c80067337dd89303 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 15:20:42 +0200 Subject: [PATCH 19/59] refactor(examine): split the Vulkan probe and the protocol change out of this PR On review: at ~5.7k lines this mixed the ROCMAI-550 messaging change with two things that are not it, and the reviewer asked twice for a split. Out, into their own PRs: - the Vulkan probe (`host_gpu/vulkan.rs`, `VulkanStatus`), the `NoGpuBackend` support level, the `vulkan:` examine line and the three scenarios that cover the Vulkan tiers. `gpu_support_level` goes back to the three levels the ticket describes; - `EngineStdioOptions::withhold_inherited_key_file`, which is hardening of the shared engine-protocol crate and belongs to neither ticket. What stays is ROCMAI-550 itself: the per-host support level in place of a static ROCm claim, the `rocm_support` line, the install-sdk advice that names a command the host can actually run, and the live `device:` row with the key-file plumbing it needs. Signed-off-by: Roman Sirokov --- apps/rocm/src/bridge_snapshot.rs | 1 - apps/rocm/src/driver_plan.rs | 1 - apps/rocm/src/engine_ipc.rs | 3 - apps/rocm/src/examine_report_text.rs | 10 +- apps/rocm/src/main.rs | 24 +- apps/rocmd/src/engine_ipc.rs | 4 - crates/rocm-core/src/gfx_target.rs | 132 +--- crates/rocm-core/src/host_gpu.rs | 17 - crates/rocm-core/src/host_gpu/render.rs | 207 +---- crates/rocm-core/src/host_gpu/vulkan.rs | 726 ------------------ crates/rocm-core/src/lib.rs | 12 +- .../rocm-engine-protocol/src/stdio_client.rs | 47 +- docs/architecture.md | 4 +- tests/e2e-cucumber/features/examine.feature | 37 - tests/e2e-cucumber/tests/e2e/examine_steps.rs | 224 ------ 15 files changed, 43 insertions(+), 1406 deletions(-) delete mode 100644 crates/rocm-core/src/host_gpu/vulkan.rs diff --git a/apps/rocm/src/bridge_snapshot.rs b/apps/rocm/src/bridge_snapshot.rs index 115279e42a..741c18ede7 100644 --- a/apps/rocm/src/bridge_snapshot.rs +++ b/apps/rocm/src/bridge_snapshot.rs @@ -40,7 +40,6 @@ pub(crate) fn build_codex_bridge_snapshot(paths: &AppPaths) -> Result( app_paths: paths, env_root, endpoint_key_file: None, - // An external plugin is a third-party binary: never hand it a path to - // a live endpoint credential, including one we inherited ourselves. - withhold_inherited_key_file: true, request_progress: stream_progress, forward_stderr: true, note_failing_exit_after_answer: false, diff --git a/apps/rocm/src/examine_report_text.rs b/apps/rocm/src/examine_report_text.rs index 20aef803e3..12d66bee2b 100644 --- a/apps/rocm/src/examine_report_text.rs +++ b/apps/rocm/src/examine_report_text.rs @@ -133,13 +133,9 @@ pub(crate) fn examine_human_report( let mut output = render_examine_plain_header(&summary); output.push_str(&summary.render_text()); append_examine_runtime_state(&mut output, paths, config)?; - // `summary` already carries the Vulkan probe result, so the support level - // here is the verified one: this is the surface other messages send people - // to when they want a real answer. let gpu_support_level = rocm_core::gpu_support_level( summary.detected_gfx_target.as_deref(), summary.compatible_therock_family.as_deref(), - summary.vulkan_status(), ); append_examine_engine_inventory( &mut output, @@ -208,10 +204,7 @@ pub(crate) fn render_engine_inventory_text_with_paths(paths: Option<&AppPaths>) render_engine_inventory_text_for_host( paths, rocm_core::default_engine_for_host(&host_gpu), - // No Vulkan probe on this listing path, for the reason given on - // `host_gpu_support_level`: `rocm engines` is cheap by design, and - // `rocm examine` is where the verified answer lives. - rocm_core::gpu_support_level_for_host_gpu(&host_gpu, rocm_core::VulkanStatus::Unknown), + rocm_core::gpu_support_level_for_host_gpu(&host_gpu), ) } @@ -641,7 +634,6 @@ mod tests { detected_gfx_target: None, compatible_therock_family: None, detected_therock_family: None, - vulkan: None, driver: rocm_core::DriverSummary { policy: "linux_official_amd_dkms_wrapper".to_owned(), status: "amdgpu_available".to_owned(), diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 3887afd02e..c18432bc61 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -1081,15 +1081,6 @@ const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str rocm_core::GpuSupportLevel::RocmHip => { "default embedded Lemonade server with ROCm llama.cpp backend" } - // Only `rocm examine` reaches this level — the cheap probe behind - // `host_gpu_support_level` passes `VulkanStatus::Unknown` and so lands - // on `VulkanOnly` — but the arm is spelled out rather than folded into - // `Unknown`, because what it means is the opposite: `Unknown` promises - // that a command can still answer, and here the question is already - // answered and the answer is no. - rocm_core::GpuSupportLevel::NoGpuBackend => { - "default embedded Lemonade server; this GPU has no ROCm/HIP support and no Vulkan driver was found, so no GPU backend can run here" - } rocm_core::GpuSupportLevel::Unknown => { "default embedded Lemonade server; run `rocm examine` for this host's GPU support level, which says how to find the backend" } @@ -1102,21 +1093,8 @@ const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str /// The same two inputs `rocm examine` derives it from, read through /// `detect_host_gpu_summary` rather than the full examine gather — exactly what /// `render_engine_inventory_text_with_paths` (i.e. `rocm engines`) already does. -/// -/// **The Vulkan probe is deliberately not run here**, so this stays the cheap -/// probe its callers chose it for: the probe spawns `vulkaninfo` on Linux and -/// PowerShell on Windows, and `rocm engines` is a listing command that would -/// pay that on every invocation. Passing [`VulkanStatus::Unknown`] keeps these -/// surfaces answering exactly as they did before this probe existed — a -/// Vulkan-capable-by-the-table host still reads `VulkanOnly` — while `rocm -/// examine`, which already gathers the probe, is the surface that can tell a -/// host with a Vulkan driver from one without. That split is why -/// `lemonade_engine_note` sends the user to `rocm examine` for a real answer. pub(crate) fn host_gpu_support_level(paths: Option<&AppPaths>) -> rocm_core::GpuSupportLevel { - rocm_core::gpu_support_level_for_host_gpu( - &rocm_core::detect_host_gpu_summary(paths), - rocm_core::VulkanStatus::Unknown, - ) + rocm_core::gpu_support_level_for_host_gpu(&rocm_core::detect_host_gpu_summary(paths)) } fn refresh_startup_update_check_quietly() { diff --git a/apps/rocmd/src/engine_ipc.rs b/apps/rocmd/src/engine_ipc.rs index eda8243ac2..f4749c3eb2 100644 --- a/apps/rocmd/src/engine_ipc.rs +++ b/apps/rocmd/src/engine_ipc.rs @@ -211,10 +211,6 @@ const fn stdio_options<'a>( app_paths: Some(paths), env_root: None, endpoint_key_file, - // `None` here means "this loopback service needs no key", not "this - // child may not see one" — these are the daemon's own children, and - // clearing would change what they inherit. - withhold_inherited_key_file: false, request_progress: false, forward_stderr: false, note_failing_exit_after_answer: true, diff --git a/crates/rocm-core/src/gfx_target.rs b/crates/rocm-core/src/gfx_target.rs index 8b843bb6b0..092c288a46 100644 --- a/crates/rocm-core/src/gfx_target.rs +++ b/crates/rocm-core/src/gfx_target.rs @@ -566,63 +566,23 @@ pub(crate) fn gfx_target_from_gc_version(major: u32, minor: u32, revision: u32) pub enum GpuSupportLevel { /// A TheRock package family covers this GPU: full ROCm/HIP support. RocmHip, - /// Recognized, confirmed to have no ROCm/HIP build, and a Vulkan GPU driver - /// was **found on this host**, so Vulkan-based engines (the default Lemonade - /// backend) run on it. - /// - /// The second half of that sentence is evidence, not inference. This variant - /// used to be returned from the gfx table alone — "no ROCm build" was taken - /// to mean "Vulkan then", on a machine nobody had looked at. Every surface - /// that renders this level tells the user to go and serve a model on it, so - /// a host with no Vulkan driver installed, or with only a software - /// rasteriser, was sent to a backend that cannot start. + /// Recognized, confirmed to have no ROCm/HIP build, but Vulkan-based + /// engines (the default Lemonade backend) still work on it. VulkanOnly, - /// Recognized, confirmed to have no ROCm/HIP build, **and** no Vulkan GPU - /// driver was found: this host has no GPU path at all. - /// - /// Deliberately not folded into [`Self::Unknown`]. Both halves here are - /// confirmed — the catalog establishes the missing ROCm build, and the - /// Vulkan probe looked and found nothing — whereas `Unknown` means the CLI - /// could not establish either. Collapsing them would commit, in the other - /// direction, the exact conflation this enum's `Unknown` exists to prevent: - /// a fact about the machine would be reported as a gap in what this build - /// knows, and the user would be sent to probe a question already answered. - NoGpuBackend, /// No GPU detected yet, or a gfx target this catalog doesn't recognize. Unknown, } -/// Classify the host's GPU support level. -/// -/// Reads the two facts `ExamineSummary` and `resolve_family` already compute — -/// the detected gfx target and whichever TheRock family (if any) covers it — -/// plus what the Vulkan probe found. -/// -/// `vulkan` only ever decides between [`GpuSupportLevel::VulkanOnly`] and -/// [`GpuSupportLevel::NoGpuBackend`], and is consulted only once the catalog has -/// confirmed there is no ROCm build. A host with a covering family is `RocmHip` -/// whatever Vulkan reports, and an unrecognized target stays `Unknown`: finding -/// a Vulkan driver says nothing about whether ROCm would also work there, so it -/// must not be allowed to turn a detection gap into a support claim. -/// -/// [`VulkanStatus::Unknown`] — the platform could not be probed at all — keeps -/// `VulkanOnly`, because withholding the one path such a host has on the -/// strength of a probe that never ran would be worse than the old assumption, -/// not better. +/// Classify the host's GPU support level from the same two facts +/// `ExamineSummary` and `resolve_family` already compute: the detected gfx +/// target and whichever TheRock family (if any) covers it. pub fn gpu_support_level( detected_gfx_target: Option<&str>, compatible_therock_family: Option<&str>, - vulkan: crate::host_gpu::VulkanStatus, ) -> GpuSupportLevel { match (detected_gfx_target, compatible_therock_family) { (_, Some(_)) => GpuSupportLevel::RocmHip, - (Some(gfx), None) if gfx_target_has_no_rocm_build(gfx) => { - if vulkan.is_usable() || vulkan == crate::host_gpu::VulkanStatus::Unknown { - GpuSupportLevel::VulkanOnly - } else { - GpuSupportLevel::NoGpuBackend - } - } + (Some(gfx), None) if gfx_target_has_no_rocm_build(gfx) => GpuSupportLevel::VulkanOnly, _ => GpuSupportLevel::Unknown, } } @@ -634,27 +594,17 @@ pub fn gpu_support_level( /// site that transposed them would classify a Vulkan-only host as `RocmHip` /// and still compile. `rocm engines` and the codex-bridge `engines` tool both /// derive the level from this struct, so both go through this. -/// -/// `vulkan` is passed in rather than probed here so this stays a pure function -/// of what the caller already holds: the probe runs a subprocess and reads the -/// filesystem, and burying that in a helper this cheap would make every caller -/// pay for it without saying so. #[must_use] -pub fn gpu_support_level_for_host_gpu( - host_gpu: &HostGpuSummary, - vulkan: crate::host_gpu::VulkanStatus, -) -> GpuSupportLevel { +pub fn gpu_support_level_for_host_gpu(host_gpu: &HostGpuSummary) -> GpuSupportLevel { gpu_support_level( host_gpu.gfx_target.as_deref(), host_gpu.therock_family.as_deref(), - vulkan, ) } #[cfg(test)] mod tests { use super::*; - use crate::host_gpu::VulkanStatus; #[test] fn normalize_therock_family_maps_gfx1101_to_gfx110x_all() { @@ -1047,7 +997,7 @@ mod tests { therock_family: Some("gfx94X-dcgpu".to_owned()), }; assert_eq!( - gpu_support_level_for_host_gpu(&host_gpu, VulkanStatus::Available), + gpu_support_level_for_host_gpu(&host_gpu), GpuSupportLevel::RocmHip ); } @@ -1060,7 +1010,7 @@ mod tests { therock_family: None, }; assert_eq!( - gpu_support_level_for_host_gpu(&host_gpu, VulkanStatus::Available), + gpu_support_level_for_host_gpu(&host_gpu), GpuSupportLevel::VulkanOnly ); } @@ -1068,7 +1018,7 @@ mod tests { #[test] fn gpu_support_level_for_host_gpu_reads_an_undetected_summary_as_unknown() { assert_eq!( - gpu_support_level_for_host_gpu(&HostGpuSummary::default(), VulkanStatus::Available), + gpu_support_level_for_host_gpu(&HostGpuSummary::default()), GpuSupportLevel::Unknown ); } @@ -1076,76 +1026,24 @@ mod tests { #[test] fn gpu_support_level_is_rocm_hip_when_a_family_covers_it() { assert_eq!( - gpu_support_level(Some("gfx90a"), Some("gfx90a"), VulkanStatus::Available), - GpuSupportLevel::RocmHip - ); - // A covering family wins outright: a host whose Vulkan probe found - // nothing still has ROCm, and must not be downgraded for it. - assert_eq!( - gpu_support_level(Some("gfx90a"), Some("gfx90a"), VulkanStatus::Absent), + gpu_support_level(Some("gfx90a"), Some("gfx90a")), GpuSupportLevel::RocmHip ); } #[test] fn gpu_support_level_is_unknown_without_detection_or_for_an_unrecognized_target() { + assert_eq!(gpu_support_level(None, None), GpuSupportLevel::Unknown); assert_eq!( - gpu_support_level(None, None, VulkanStatus::Available), - GpuSupportLevel::Unknown - ); - // Finding a Vulkan driver says nothing about whether ROCm would work on - // an unrecognized target, so it must not promote a detection gap into a - // support claim. - assert_eq!( - gpu_support_level( - Some("gfx_totally_unrecognized"), - None, - VulkanStatus::Available - ), + gpu_support_level(Some("gfx_totally_unrecognized"), None), GpuSupportLevel::Unknown ); } #[test] - fn gpu_support_level_is_vulkan_only_for_gfx90c_with_a_vulkan_driver() { - assert_eq!( - gpu_support_level(Some("gfx90c"), None, VulkanStatus::Available), - GpuSupportLevel::VulkanOnly - ); - // An installed driver that enumerated no device is still worth routing - // to; the level says "Vulkan", and `examine`'s own line carries how sure - // the probe was. - assert_eq!( - gpu_support_level( - Some("gfx90c"), - None, - VulkanStatus::DriverPresentDevicesUnverified - ), - GpuSupportLevel::VulkanOnly - ); - } - - /// The defect this guards against: `VulkanOnly` used to be inferred from the - /// gfx table alone, so a Renoir laptop with no Vulkan driver installed — or - /// with only a CPU rasteriser — was told to go and serve a model on a - /// backend that cannot start. - #[test] - fn gpu_support_level_reports_no_gpu_backend_when_the_vulkan_probe_found_none() { - assert_eq!( - gpu_support_level(Some("gfx90c"), None, VulkanStatus::Absent), - GpuSupportLevel::NoGpuBackend, - "a confirmed-absent Vulkan driver must not read as a usable Vulkan path" - ); - } - - /// A platform the probe cannot read is not a platform without Vulkan. - /// Withholding the only path such a host has, on the strength of a probe - /// that never ran, would be a worse answer than the assumption this change - /// replaces. - #[test] - fn gpu_support_level_keeps_vulkan_when_the_probe_could_not_run() { + fn gpu_support_level_is_vulkan_only_for_gfx90c() { assert_eq!( - gpu_support_level(Some("gfx90c"), None, VulkanStatus::Unknown), + gpu_support_level(Some("gfx90c"), None), GpuSupportLevel::VulkanOnly ); } diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index db45107033..87b6ad185f 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -54,7 +54,6 @@ mod gpu_visibility; pub(crate) mod linux; mod os_facts; mod render; -mod vulkan; mod windows_inventory; mod wsl; @@ -74,11 +73,6 @@ pub(crate) use linux::detect_linux_sysfs_gfx_target; pub use linux::is_amdgpu_device; #[cfg(any(target_os = "linux", test))] pub(crate) use linux::{KfdGpuNode, kfd_gpu_nodes_in}; -pub use vulkan::{ - VulkanDevice, VulkanStatus, VulkanSupport, classify_vulkan_support, detect_vulkan_support, - icd_driver_label, icd_is_gpu_driver, parse_vulkaninfo_summary, - parse_windows_vulkan_icd_listing, -}; pub(crate) use windows_inventory::detect_windows_amd_driver_info; #[cfg(test)] use windows_inventory::parse_windows_examine_inventory; @@ -139,15 +133,6 @@ pub struct ExamineSummary { pub compatible_therock_family: Option, #[serde(default)] pub detected_therock_family: Option, - /// What the Vulkan probe found, when it could run. - /// - /// `#[serde(default)]` is load-bearing in the *reverse* direction: `rocm - /// remote doctor` deserializes summaries produced by other hosts, which may - /// be running a build from before this field existed. `None` means "not - /// probed", which [`ExamineSummary::vulkan_status`] maps to - /// [`VulkanStatus::Unknown`] — never to "no Vulkan". - #[serde(default)] - pub vulkan: Option, pub driver: DriverSummary, pub legacy_rocm: LegacyRocmSummary, #[serde(default)] @@ -299,7 +284,6 @@ impl ExamineSummary { detected_gfx_target, compatible_therock_family, detected_therock_family, - vulkan: Some(detect_vulkan_support()), driver: detect_driver_summary_with_windows_inventory( windows_inventory.as_ref(), wsl.as_ref(), @@ -746,7 +730,6 @@ mod tests { detected_gfx_target: None, compatible_therock_family: None, detected_therock_family: None, - vulkan: None, driver: windows_driver_summary( Some("AMD Radeon(TM) Graphics driver 30.0.13014.8".to_owned()), info, diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index 721b6ed6a3..e6f0f479f2 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -6,42 +6,9 @@ use super::ExamineSummary; use super::os_facts::format_gib_value; -use super::vulkan::VulkanStatus; use crate::gfx_target::{GpuSupportLevel, gpu_support_level}; impl ExamineSummary { - /// What the Vulkan probe found, or [`VulkanStatus::Unknown`] when this - /// summary carries no probe result — an older host's JSON read back by - /// `rocm remote doctor`, or a caller that built the struct by hand. - /// - /// Public because the `rocm` binary classifies its own support level from a - /// summary it already gathered, and must read the probe result the same way - /// this module does rather than reaching into the field and inventing its - /// own `None` handling. - #[must_use] - pub fn vulkan_status(&self) -> VulkanStatus { - self.vulkan - .as_ref() - .map_or(VulkanStatus::Unknown, |vulkan| vulkan.status) - } - - /// The `vulkan` line: the status, and the evidence behind it. - fn vulkan_line(&self) -> String { - let Some(vulkan) = self.vulkan.as_ref() else { - return "".to_owned(); - }; - let status = match vulkan.status { - VulkanStatus::Available => "available", - VulkanStatus::DriverPresentDevicesUnverified => "driver_present_devices_unverified", - VulkanStatus::Absent => "absent", - VulkanStatus::Unknown => "unknown", - }; - vulkan.detail.as_deref().map_or_else( - || status.to_owned(), - |detail| format!("{status} ({detail})"), - ) - } - /// The `rocm_support` line of the examine text. /// /// ROCMAI-550 asks `rocm examine` to "report the support level with one @@ -67,49 +34,14 @@ impl ExamineSummary { /// blank line. fn rocm_support_line(&self) -> String { let detected = self.detected_gfx_target.as_deref(); - let vulkan = self.vulkan_status(); - match gpu_support_level(detected, self.compatible_therock_family.as_deref(), vulkan) { + match gpu_support_level(detected, self.compatible_therock_family.as_deref()) { // `VulkanOnly` is only ever returned for a detected target, so the // fallback below is unreachable; it keeps the arm total without a // panic path. - // - // The claim is graded by what the probe actually established. An - // enumerated device is named, because naming it is what lets the - // reader check the claim rather than take it; an installed-but- - // unenumerated driver says so instead of borrowing the stronger - // sentence; and a platform that could not be probed admits it. All - // three still point at the same command, because all three are - // hosts worth trying. GpuSupportLevel::VulkanOnly => format!( - " rocm_support: GPU detected ({}) but not supported by ROCm; {} - serve on it \ - with `rocm serve --engine lemonade`\n", - detected.unwrap_or_default(), - match vulkan { - VulkanStatus::Available => self.vulkan.as_ref().map_or_else( - || "the Vulkan backend can be used".to_owned(), - |support| support.driver_label().map_or_else( - || "the Vulkan backend can be used".to_owned(), - |driver| format!("the Vulkan backend can be used ({driver})"), - ), - ), - VulkanStatus::DriverPresentDevicesUnverified => - "a Vulkan driver is installed, so the Vulkan backend is expected to work" - .to_owned(), - VulkanStatus::Absent | VulkanStatus::Unknown => - "Vulkan was not probed on this platform, so the Vulkan backend is \ - unconfirmed" - .to_owned(), - } - ), - // Both halves are established facts: the catalog confirmed there is - // no ROCm build, and the probe looked for a Vulkan driver and found - // none. Unlike `Unknown` below, there is nothing left to go and find - // out on this host, so the advice is to install a driver rather than - // to serve a model and read back what happened. - GpuSupportLevel::NoGpuBackend => format!( - " rocm_support: GPU detected ({}) but not supported by ROCm, and no Vulkan GPU \ - driver was found - install one (Mesa RADV on Linux, the AMD display driver on \ - Windows) to run models on this GPU\n", + " rocm_support: GPU detected ({}) but not supported by ROCm; \ + the Vulkan backend can still be used - serve on it with \ + `rocm serve --engine lemonade`\n", detected.unwrap_or_default() ), // The covering family is the fact; it is already printed on its own @@ -205,7 +137,7 @@ impl ExamineSummary { "false (this run's output is captured, so the CLI will not prompt)" }; format!( - "rocm examine\n os: {}\n arch: {}\n kernel: {}\n distro: {}\n cpu: {}\n system_ram: {}\n interactive_terminal: {}\n default_engine: {}\n detected_gfx_target: {}\n compatible_therock_family: {}\n{} detected_therock_family: {}\n vulkan: {}\n driver_policy: {}\n driver_status: {}\n driver_detail: {}\n{} legacy_rocm_status: {}\n legacy_rocm_paths: {}\n legacy_rocm_version: {}\n legacy_rocm_detail: {}\n legacy_rocm_guidance: {}\n wsl: {}\n wsl_dxg_device: {}\n wsl_dxcore: {}\n wsl_librocdxg: {}\n wsl_rocdxg_dids: {}\n wsl_ldconfig_librocdxg: {}\n wsl_librocdxg_on_ld_library_path: {}\n wsl_librocdxg_beside_runtime: {}\n wsl_global_rocminfo: {}\n wsl_cargo: {}\n wsl_detail: {}\n managed_runtimes: {}\n managed_services: {}\n model_cache_entries: {}\n config_dir: {}\n data_dir: {}\n cache_dir: {}\n", + "rocm examine\n os: {}\n arch: {}\n kernel: {}\n distro: {}\n cpu: {}\n system_ram: {}\n interactive_terminal: {}\n default_engine: {}\n detected_gfx_target: {}\n compatible_therock_family: {}\n{} detected_therock_family: {}\n driver_policy: {}\n driver_status: {}\n driver_detail: {}\n{} legacy_rocm_status: {}\n legacy_rocm_paths: {}\n legacy_rocm_version: {}\n legacy_rocm_detail: {}\n legacy_rocm_guidance: {}\n wsl: {}\n wsl_dxg_device: {}\n wsl_dxcore: {}\n wsl_librocdxg: {}\n wsl_rocdxg_dids: {}\n wsl_ldconfig_librocdxg: {}\n wsl_librocdxg_on_ld_library_path: {}\n wsl_librocdxg_beside_runtime: {}\n wsl_global_rocminfo: {}\n wsl_cargo: {}\n wsl_detail: {}\n managed_runtimes: {}\n managed_services: {}\n model_cache_entries: {}\n config_dir: {}\n data_dir: {}\n cache_dir: {}\n", self.os, self.arch, self.kernel.as_deref().unwrap_or(""), @@ -223,7 +155,6 @@ impl ExamineSummary { self.detected_therock_family .as_deref() .unwrap_or(""), - self.vulkan_line(), self.driver.policy, self.driver.status, self.driver.detail.as_deref().unwrap_or(""), @@ -286,8 +217,8 @@ mod tests { use super::*; use crate::LegacyRocmSummary; use crate::gfx_target::normalize_therock_family; + use crate::host_gpu::DriverSummary; use crate::host_gpu::tests::windows_summary_with_driver; - use crate::host_gpu::{DriverSummary, VulkanDevice, VulkanSupport}; use std::path::PathBuf; #[test] @@ -304,7 +235,6 @@ mod tests { detected_gfx_target: None, compatible_therock_family: Some("gfx120X-all".to_owned()), detected_therock_family: None, - vulkan: None, driver: DriverSummary { policy: "windows_validate_only".to_owned(), status: "amd_display_driver_detected".to_owned(), @@ -384,7 +314,6 @@ mod tests { detected_gfx_target: detected_gfx_target.map(str::to_owned), compatible_therock_family: compatible_therock_family.map(str::to_owned), detected_therock_family: None, - vulkan: None, driver: DriverSummary { policy: "windows_validate_only".to_owned(), status: "amd_display_driver_detected".to_owned(), @@ -407,41 +336,12 @@ mod tests { } } - /// A gfx90c summary carrying the given Vulkan evidence. - /// - /// gfx90c is detected from its PCI ID but has no ROCm package family, so it - /// is the target that reaches the Vulkan-graded arms at all. - fn gfx90c_summary_with_vulkan(vulkan: Option) -> ExamineSummary { - let family = normalize_therock_family("gfx90c"); - let mut summary = examine_summary_with_gpu(Some("gfx90c"), family.as_deref()); - summary.vulkan = vulkan; - summary - } - - fn vulkan_support(status: VulkanStatus, detail: &str) -> Option { - Some(VulkanSupport { - status, - detail: Some(detail.to_owned()), - devices: vec![VulkanDevice { - name: "AMD Radeon Graphics (RADV RENOIR)".to_owned(), - driver_name: Some("radv".to_owned()), - vendor_id: Some("0x1002".to_owned()), - device_type: Some("PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU".to_owned()), - ..VulkanDevice::default() - }], - ..VulkanSupport::default() - }) - } - #[test] fn examine_render_says_a_gpu_without_a_rocm_family_is_unsupported() { - let summary = gfx90c_summary_with_vulkan(vulkan_support( - VulkanStatus::Available, - "AMD Radeon Graphics (RADV RENOIR) through radv", - )); - + // gfx90c is detected from its PCI ID but has no ROCm package family. + let family = normalize_therock_family("gfx90c"); + let summary = examine_summary_with_gpu(Some("gfx90c"), family.as_deref()); let rendered = summary.render_text(); - assert!( rendered.contains("detected_gfx_target: gfx90c"), "{rendered}" @@ -451,92 +351,9 @@ mod tests { "{rendered}" ); assert!( - rendered.contains("the Vulkan backend can be used (radv)"), - "a confirmed device must be named, so the claim can be checked: {rendered}" - ); - assert!( - rendered.contains("vulkan: available"), - "the evidence line must carry the verdict too: {rendered}" - ); - } - - /// The defect this guards against: the `VulkanOnly` line used to promise - /// "the Vulkan backend can still be used" on every gfx90c host, derived from - /// the gfx table alone. A Renoir laptop with no Vulkan driver installed was - /// told to go and serve a model on a backend that cannot start — and under - /// the GPU-required policy there is nothing for it to fall back to. - #[test] - fn examine_render_does_not_promise_vulkan_on_a_host_that_has_none() { - let summary = gfx90c_summary_with_vulkan(Some(VulkanSupport { - status: VulkanStatus::Absent, - detail: Some("no Vulkan driver found".to_owned()), - ..VulkanSupport::default() - })); - - let rendered = summary.render_text(); - - assert!( - !rendered.contains("the Vulkan backend can be used"), - "a host with no Vulkan driver must not be promised one: {rendered}" - ); - assert!( - rendered.contains("no Vulkan GPU driver was found"), + rendered.contains("the Vulkan backend can still be used"), "{rendered}" ); - assert!( - rendered.contains("install one"), - "the advice must be able to clear the condition it is printed for: {rendered}" - ); - assert!(rendered.contains("vulkan: absent"), "{rendered}"); - } - - /// An installed driver that enumerated no device is a weaker claim than a - /// device that answered, and says so rather than borrowing the stronger - /// sentence. - #[test] - fn examine_render_separates_an_installed_driver_from_a_confirmed_device() { - let summary = gfx90c_summary_with_vulkan(Some(VulkanSupport { - status: VulkanStatus::DriverPresentDevicesUnverified, - detail: Some("Mesa RADV is installed, but no device was enumerated".to_owned()), - ..VulkanSupport::default() - })); - - let rendered = summary.render_text(); - - assert!( - rendered.contains("a Vulkan driver is installed, so the Vulkan backend is expected"), - "{rendered}" - ); - assert!( - rendered.contains("vulkan: driver_present_devices_unverified"), - "{rendered}" - ); - } - - /// A summary with no probe result at all — an older host's JSON read back - /// through `rocm remote doctor` — must not read as "no Vulkan". It keeps - /// the Vulkan advice, because withholding the only path such a host has on - /// the strength of a probe that never ran would be worse than the - /// assumption this change replaces. - #[test] - fn examine_render_treats_an_unprobed_summary_as_unconfirmed_not_absent() { - let summary = gfx90c_summary_with_vulkan(None); - - let rendered = summary.render_text(); - - assert!(rendered.contains("vulkan: "), "{rendered}"); - assert!( - rendered.contains("the Vulkan backend is unconfirmed"), - "{rendered}" - ); - assert!( - !rendered.contains("no Vulkan GPU driver was found"), - "an unprobed host is not a host without Vulkan: {rendered}" - ); - assert!( - rendered.contains("rocm serve --engine lemonade"), - "it is still worth trying, so the command stays: {rendered}" - ); } /// An unmapped target must not be *claimed* unsupported, even though every @@ -644,7 +461,6 @@ mod tests { detected_gfx_target: None, compatible_therock_family: None, detected_therock_family: None, - vulkan: None, driver: DriverSummary { policy: "linux_official_amd_dkms_wrapper".to_owned(), status: "amdgpu_available".to_owned(), @@ -699,7 +515,6 @@ mod tests { detected_gfx_target: None, compatible_therock_family: None, detected_therock_family: None, - vulkan: None, driver: DriverSummary { policy: "linux_official_amd_dkms_wrapper".to_owned(), status: "amdgpu_available".to_owned(), @@ -795,7 +610,7 @@ mod tests { .find(|l| l.trim_start().starts_with("rocm_support:")) .expect("every host gets a support line"); assert_eq!( - gpu_support_level(None, None, VulkanStatus::Unknown), + gpu_support_level(None, None), GpuSupportLevel::Unknown, "precondition: nothing detected is the Unknown tier" ); diff --git a/crates/rocm-core/src/host_gpu/vulkan.rs b/crates/rocm-core/src/host_gpu/vulkan.rs deleted file mode 100644 index b1e2b08913..0000000000 --- a/crates/rocm-core/src/host_gpu/vulkan.rs +++ /dev/null @@ -1,726 +0,0 @@ -// Copyright © Advanced Micro Devices, Inc., or its affiliates. -// -// SPDX-License-Identifier: MIT - -//! Whether this host can run a model through Vulkan. -//! -//! The CLI has always *asserted* this for a GPU with no ROCm build -- `examine` -//! told a gfx90c user "the Vulkan backend can still be used" without ever -//! looking. This probe is what makes that sentence evidence rather than a -//! promise, and it is what lets a host outside the ROCm compatibility matrix be -//! described by what it can do instead of only by what it cannot. -//! -//! Two sources, in increasing order of strength: -//! -//! 1. An **installable driver**: an ICD manifest on Linux (`/usr/share/vulkan/ -//! icd.d` and friends), or the Khronos registry key on Windows. This proves a -//! driver is *installed*, not that a device enumerates. -//! 2. An **enumerated device**: `vulkaninfo --summary`, when it is on PATH. -//! This proves a device actually answers. -//! -//! The two are kept apart by [`VulkanStatus`] rather than collapsed, because -//! collapsing them would replace one unverified promise with another. -//! -//! **A software rasteriser is not a GPU.** Mesa's lavapipe (`lvp_icd.*.json`, -//! `PHYSICAL_DEVICE_TYPE_CPU`) presents a complete Vulkan implementation that -//! runs entirely on the CPU. Counting it here would route a user onto CPU -//! execution while reporting a GPU path -- precisely the silent CPU fallback -//! `AGENTS.md` §6 forbids -- so it is recognised in order to be excluded, and -//! named in `detail` so the exclusion is visible rather than mysterious. - -use super::capture::capture_optional_command; -#[cfg(target_os = "linux")] -use crate::host_path; -use serde::{Deserialize, Serialize}; - -/// The directories a Linux Vulkan loader reads ICD manifests from. -/// -/// Read through [`host_path`] so a simulated host root can plant them, which is -/// what lets the no-GPU CI lane cover this probe at all. -#[cfg(target_os = "linux")] -const LINUX_ICD_DIRS: [&str; 4] = [ - "/usr/share/vulkan/icd.d", - "/usr/local/share/vulkan/icd.d", - "/etc/vulkan/icd.d", - "/usr/lib/x86_64-linux-gnu/vulkan/icd.d", -]; - -/// How sure the CLI is that a model can run through Vulkan here. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(rename_all = "snake_case")] -pub enum VulkanStatus { - /// A Vulkan device answered and it is a real GPU. - Available, - /// A driver is installed, but no device was enumerated -- `vulkaninfo` is - /// not on PATH, so nothing confirmed a device answers. Likely to work; - /// not established. - DriverPresentDevicesUnverified, - /// The probe looked and found no GPU Vulkan driver. A software rasteriser - /// alone lands here, not in [`Self::Available`]. - Absent, - /// The probe could not look (an unsupported platform). - /// - /// The default: a value nobody has filled in is one nobody has probed, and - /// defaulting to `Absent` would let an unset field read as a fact about the - /// machine. - #[default] - Unknown, -} - -impl VulkanStatus { - /// Whether a model may be routed through Vulkan on the strength of this. - #[must_use] - pub const fn is_usable(self) -> bool { - matches!(self, Self::Available | Self::DriverPresentDevicesUnverified) - } -} - -/// One physical device as `vulkaninfo --summary` reported it. -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct VulkanDevice { - pub name: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub driver_name: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub driver_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub api_version: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub vendor_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub device_id: Option, - /// The raw `deviceType`, e.g. `PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU`. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub device_type: Option, -} - -impl VulkanDevice { - /// AMD's PCI vendor id, as `vulkaninfo` prints it. - const AMD_VENDOR_ID: &'static str = "0x1002"; - - #[must_use] - pub fn is_amd(&self) -> bool { - self.vendor_id - .as_deref() - .is_some_and(|id| id.eq_ignore_ascii_case(Self::AMD_VENDOR_ID)) - } - - /// Whether this device is real hardware rather than a software rasteriser. - /// - /// A missing `deviceType` reads as hardware: the field is optional in the - /// summary output, and treating an unreported type as CPU would discard a - /// working GPU. Lavapipe always reports its type, so the only entries this - /// lets through unverified are ones that named no type at all. - #[must_use] - pub fn is_hardware(&self) -> bool { - !self - .device_type - .as_deref() - .is_some_and(|kind| kind.to_ascii_uppercase().contains("TYPE_CPU")) - } -} - -/// What the host offers for Vulkan, and the evidence behind it. -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct VulkanSupport { - pub status: VulkanStatus, - /// The ICD manifests or registry entries found, as the loader names them. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub icds: Vec, - /// The devices `vulkaninfo` enumerated, when it was available. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub devices: Vec, - /// A one-line human summary, including why anything was excluded. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub detail: Option, -} - -impl VulkanSupport { - /// The driver this host would use, for the `examine` line. - /// - /// Prefers what a device actually reported (`AMD proprietary driver`, - /// `radv`) over what an ICD filename implies, because the filename is a - /// guess about a vendor and the device's own `driverName` is not. - #[must_use] - pub fn driver_label(&self) -> Option { - self.devices - .iter() - .find(|device| device.is_amd() && device.is_hardware()) - .and_then(|device| device.driver_name.clone()) - .or_else(|| { - self.icds - .iter() - .find_map(|icd| icd_driver_label(icd)) - .map(str::to_owned) - }) - } -} - -/// The driver an ICD manifest's file name implies, or `None` when the name is -/// not one this table knows. -/// -/// Deliberately includes the software rasterisers so a caller can *exclude* -/// them by name; [`icd_is_gpu_driver`] is the predicate that does so. -#[must_use] -pub fn icd_driver_label(icd: &str) -> Option<&'static str> { - let name = icd - .rsplit(['/', '\\']) - .next() - .unwrap_or(icd) - .to_ascii_lowercase(); - if name.starts_with("radeon_icd") { - return Some("Mesa RADV"); - } - if name.starts_with("amd_icd") || name.starts_with("amdvlk") { - return Some("AMDVLK"); - } - if name.starts_with("lvp_icd") { - return Some("Mesa lavapipe (software)"); - } - if name.starts_with("vk_swiftshader") { - return Some("SwiftShader (software)"); - } - if name.starts_with("nvidia_icd") { - return Some("NVIDIA"); - } - if name.starts_with("intel_icd") || name.starts_with("intel_hasvk") { - return Some("Intel"); - } - None -} - -/// Whether an ICD manifest names a driver that runs on a GPU. -/// -/// An unrecognised manifest counts as a GPU driver: a vendor the table does not -/// list yet is far more likely to be real hardware than a new software -/// rasteriser, and the cost of the two errors is not symmetric -- missing a -/// working GPU path sends the user to CPU, while admitting an unknown one is -/// corrected by the device enumeration that follows. -#[must_use] -pub fn icd_is_gpu_driver(icd: &str) -> bool { - !icd_driver_label(icd).is_some_and(|label| label.contains("(software)")) -} - -/// Parse `vulkaninfo --summary` output into its physical devices. -/// -/// Tolerates the loader warnings `vulkaninfo` prints to stdout before the -/// report, and the `Instance Extensions` / `Instance Layers` sections ahead of -/// `Devices:` -- only `GPUn:` blocks and their indented `key = value` lines are -/// read. -#[must_use] -pub fn parse_vulkaninfo_summary(output: &str) -> Vec { - let mut devices = Vec::new(); - let mut current: Option = None; - for line in output.lines() { - let trimmed = line.trim(); - if trimmed.starts_with("GPU") && trimmed.ends_with(':') { - if let Some(device) = current.take() { - devices.push(device); - } - current = Some(VulkanDevice::default()); - continue; - } - let Some(device) = current.as_mut() else { - continue; - }; - let Some((key, value)) = trimmed.split_once('=') else { - // A non-indented, non-`key = value` line ends the block: the - // summary's device section is the last thing printed, but a caller - // may hand us trailing output. - if !trimmed.is_empty() && !line.starts_with([' ', '\t']) { - devices.push(current.take().unwrap_or_default()); - } - continue; - }; - let value = value.trim().to_owned(); - if value.is_empty() { - continue; - } - match key.trim() { - "deviceName" => device.name = value, - "driverName" => device.driver_name = Some(value), - "driverID" => device.driver_id = Some(value), - "apiVersion" => device.api_version = Some(value), - "vendorID" => device.vendor_id = Some(value), - "deviceID" => device.device_id = Some(value), - "deviceType" => device.device_type = Some(value), - _ => {} - } - } - if let Some(device) = current.take() { - devices.push(device); - } - devices.retain(|device| !device.name.is_empty()); - devices -} - -/// The ICD manifest paths listed under the Windows Khronos registry key, as -/// [`WINDOWS_VULKAN_ICD_SCRIPT`] prints them (one `ICD\t` row each). -#[must_use] -pub fn parse_windows_vulkan_icd_listing(output: &str) -> Vec { - output - .lines() - .filter_map(|line| line.trim().strip_prefix("ICD\t")) - .map(str::trim) - .filter(|path| !path.is_empty()) - .map(str::to_owned) - .collect() -} - -/// Enumerate both Khronos ICD keys. The WOW6432Node key carries the 32-bit -/// drivers; a 64-bit-only install lists nothing there, which is not an error. -#[cfg(any(windows, test))] -pub(super) const WINDOWS_VULKAN_ICD_SCRIPT: &str = r#"foreach ($key in 'HKLM:\SOFTWARE\Khronos\Vulkan\Drivers','HKLM:\SOFTWARE\WOW6432Node\Khronos\Vulkan\Drivers') { if (Test-Path $key) { foreach ($name in (Get-Item $key).GetValueNames()) { "ICD`t$name" } } }"#; - -/// Every Vulkan ICD manifest this host's loader would read. -#[cfg(target_os = "linux")] -fn detect_icds() -> Vec { - let mut icds = Vec::new(); - for dir in LINUX_ICD_DIRS { - let Ok(entries) = std::fs::read_dir(host_path(dir)) else { - continue; - }; - for entry in entries.flatten() { - let path = entry.path(); - if path.extension().is_some_and(|ext| ext == "json") - && let Some(name) = path.file_name().and_then(|name| name.to_str()) - { - icds.push(name.to_owned()); - } - } - } - icds.sort(); - icds.dedup(); - icds -} - -#[cfg(windows)] -fn detect_icds() -> Vec { - capture_optional_command( - "powershell", - &[ - "-NoLogo", - "-NoProfile", - "-NonInteractive", - "-Command", - WINDOWS_VULKAN_ICD_SCRIPT, - ], - ) - .map(|output| parse_windows_vulkan_icd_listing(&output)) - .unwrap_or_default() -} - -#[cfg(not(any(target_os = "linux", windows)))] -const fn detect_icds() -> Vec { - Vec::new() -} - -/// The devices `vulkaninfo --summary` reports, or an empty list when the tool -/// is not installed. Absence of the tool is not absence of a driver, which is -/// why this cannot decide [`VulkanStatus`] on its own. -fn detect_vulkan_devices() -> Vec { - capture_optional_command("vulkaninfo", &["--summary"]) - .map(|output| parse_vulkaninfo_summary(&output)) - .unwrap_or_default() -} - -/// Whether this platform's Vulkan surfaces can be read at all. -const fn vulkan_is_probeable() -> bool { - cfg!(any(target_os = "linux", windows)) -} - -/// What this host offers for Vulkan. -#[must_use] -pub fn detect_vulkan_support() -> VulkanSupport { - classify_vulkan_support( - detect_icds(), - detect_vulkan_devices(), - vulkan_is_probeable(), - ) -} - -/// The pure decision behind [`detect_vulkan_support`], split out so every -/// combination of evidence can be tested without a Vulkan install. -#[must_use] -pub fn classify_vulkan_support( - icds: Vec, - devices: Vec, - probeable: bool, -) -> VulkanSupport { - if !probeable { - return VulkanSupport { - status: VulkanStatus::Unknown, - icds, - devices, - detail: Some("Vulkan support is not probed on this platform".to_owned()), - }; - } - - let amd_gpu = devices - .iter() - .find(|device| device.is_amd() && device.is_hardware()); - if let Some(device) = amd_gpu { - let driver = device.driver_name.as_deref().unwrap_or("an unnamed driver"); - return VulkanSupport { - detail: Some(format!("{} through {driver}", device.name)), - status: VulkanStatus::Available, - icds, - devices, - }; - } - - let software_only = !devices.is_empty() && devices.iter().all(|device| !device.is_hardware()); - if software_only { - return VulkanSupport { - status: VulkanStatus::Absent, - detail: Some( - "only a software (CPU) Vulkan device was found; that is not a GPU path".to_owned(), - ), - icds, - devices, - }; - } - - let gpu_icds: Vec<&String> = icds.iter().filter(|icd| icd_is_gpu_driver(icd)).collect(); - if gpu_icds.is_empty() { - // No ICD *and* no enumerated device is an absence of evidence, not - // evidence of absence: neither source is exhaustive. The Linux ICD - // search is a fixed directory list that ignores `VK_ICD_FILENAMES` and - // `VK_DRIVER_FILES`; the Windows one reads only the legacy Khronos - // registry key, which current AMD drivers need not populate; and - // `vulkaninfo` is frequently not installed on either. A working driver - // can therefore leave both empty, and `Absent` is rendered to the user - // as a confirmed fact ("no Vulkan GPU driver was found - install one"). - // Say we could not tell instead, and keep `Absent` for the cases below - // where something was actually found and ruled out. - if icds.is_empty() && devices.is_empty() { - return VulkanSupport { - status: VulkanStatus::Unknown, - detail: Some( - "no Vulkan driver was found, but this host exposes nothing that confirms one is absent either (no ICD manifest, and `vulkaninfo` did not enumerate)" - .to_owned(), - ), - icds, - devices, - }; - } - return VulkanSupport { - status: VulkanStatus::Absent, - detail: Some( - "only a software (CPU) Vulkan driver is installed; that is not a GPU path" - .to_owned(), - ), - icds, - devices, - }; - } - - let named = gpu_icds - .iter() - .find_map(|icd| icd_driver_label(icd)) - .map_or_else( - || "a Vulkan driver is installed".to_owned(), - |label| format!("{label} is installed"), - ); - VulkanSupport { - status: VulkanStatus::DriverPresentDevicesUnverified, - detail: Some(format!( - "{named}, but no device was enumerated (install `vulkaninfo` to confirm)" - )), - icds, - devices, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// The real `vulkaninfo --summary` output from a Ryzen 7 4700U (gfx90c, - /// Renoir) on Windows 11 with Adrenalin 31.0.21925.1001 -- the machine - /// ROCMAI-517 is about. Note the loader warnings ahead of the report and - /// the instance sections before `Devices:`; both must be skipped. - const RENOIR_WINDOWS_SUMMARY: &str = "\ -WARNING: [Loader Message] Code 0 : windows_read_data_files_in_registry: Registry lookup failed to get layer manifest files. -========== -VULKANINFO -========== - -Vulkan Instance Version: 1.3.301 - - -Instance Extensions: count = 13 -------------------------------- -VK_EXT_debug_report : extension revision 10 -VK_KHR_surface : extension revision 25 - -Instance Layers: count = 1 --------------------------- -VK_LAYER_AMD_switchable_graphics AMD switchable graphics layer 1.3.260 version 1 - -Devices: -======== -GPU0: -\tapiVersion = 1.3.260 -\tdriverVersion = 2.0.279 -\tvendorID = 0x1002 -\tdeviceID = 0x1636 -\tdeviceType = PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU -\tdeviceName = AMD Radeon(TM) Graphics -\tdriverID = DRIVER_ID_AMD_PROPRIETARY -\tdriverName = AMD proprietary driver -\tdriverInfo = 26.5.2 (AMD proprietary shader compiler) -\tconformanceVersion = 1.3.3.1 -"; - - #[test] - fn the_renoir_igpu_is_read_out_of_a_real_vulkaninfo_summary() { - let devices = parse_vulkaninfo_summary(RENOIR_WINDOWS_SUMMARY); - - assert_eq!(devices.len(), 1, "{devices:?}"); - let gpu = &devices[0]; - assert_eq!(gpu.name, "AMD Radeon(TM) Graphics"); - assert_eq!(gpu.vendor_id.as_deref(), Some("0x1002")); - // The same device id the gfx-target table maps to gfx90c. - assert_eq!(gpu.device_id.as_deref(), Some("0x1636")); - assert_eq!(gpu.driver_name.as_deref(), Some("AMD proprietary driver")); - assert_eq!(gpu.driver_id.as_deref(), Some("DRIVER_ID_AMD_PROPRIETARY")); - assert_eq!(gpu.api_version.as_deref(), Some("1.3.260")); - assert!(gpu.is_amd(), "vendor 0x1002 is AMD"); - assert!(gpu.is_hardware(), "an integrated GPU is hardware"); - } - - #[test] - fn a_real_renoir_host_reports_vulkan_available_and_names_its_driver() { - let devices = parse_vulkaninfo_summary(RENOIR_WINDOWS_SUMMARY); - - let support = classify_vulkan_support(Vec::new(), devices, true); - - assert_eq!(support.status, VulkanStatus::Available); - assert!(support.status.is_usable()); - assert_eq!( - support.driver_label().as_deref(), - Some("AMD proprietary driver"), - "the device's own driverName must win over any ICD filename guess" - ); - } - - /// The defect this guards against: lavapipe is a complete Vulkan - /// implementation that runs on the CPU. Reporting it as a usable Vulkan - /// path would route a GPU-required launch onto CPU execution while telling - /// the user a GPU path was chosen -- the silent CPU fallback AGENTS.md §6 - /// forbids, reached through a backend rather than a policy bug. - #[test] - fn a_software_rasteriser_is_never_a_usable_vulkan_path() { - let lavapipe = "\ -GPU0: -\tapiVersion = 1.3.255 -\tvendorID = 0x10005 -\tdeviceType = PHYSICAL_DEVICE_TYPE_CPU -\tdeviceName = llvmpipe (LLVM 15.0.7, 256 bits) -\tdriverName = llvmpipe -"; - let devices = parse_vulkaninfo_summary(lavapipe); - assert_eq!(devices.len(), 1); - assert!(!devices[0].is_hardware()); - - let support = - classify_vulkan_support(vec!["lvp_icd.x86_64.json".to_owned()], devices, true); - - assert_eq!( - support.status, - VulkanStatus::Absent, - "a CPU rasteriser must not read as a Vulkan GPU path: {support:?}" - ); - assert!(!support.status.is_usable()); - assert!( - support - .detail - .as_deref() - .is_some_and(|detail| detail.contains("not a GPU path")), - "{support:?}" - ); - } - - #[test] - fn a_lavapipe_only_install_is_absent_even_before_any_device_is_enumerated() { - let support = - classify_vulkan_support(vec!["lvp_icd.x86_64.json".to_owned()], Vec::new(), true); - - assert_eq!(support.status, VulkanStatus::Absent); - assert!(!icd_is_gpu_driver("lvp_icd.x86_64.json")); - assert!(icd_is_gpu_driver("radeon_icd.x86_64.json")); - } - - #[test] - fn an_installed_driver_with_no_enumerated_device_stays_unverified() { - let support = - classify_vulkan_support(vec!["radeon_icd.x86_64.json".to_owned()], Vec::new(), true); - - assert_eq!(support.status, VulkanStatus::DriverPresentDevicesUnverified); - assert!( - support.status.is_usable(), - "an installed GPU driver is still worth routing to" - ); - assert_eq!(support.driver_label().as_deref(), Some("Mesa RADV")); - assert!( - support - .detail - .as_deref() - .is_some_and(|detail| detail.contains("no device was enumerated")), - "the gap must be stated, not papered over: {support:?}" - ); - } - - /// No ICD and no enumerated device is an absence of evidence, not evidence - /// of absence — neither source is exhaustive (the Linux ICD search ignores - /// `VK_ICD_FILENAMES`, the Windows one reads only the legacy registry key, - /// and `vulkaninfo` is often not installed). `Absent` renders as a - /// confirmed "no Vulkan GPU driver was found - install one", so a host with - /// a working driver would be told to install one it already has. - #[test] - fn no_evidence_either_way_is_not_reported_as_a_confirmed_absence() { - let nothing = classify_vulkan_support(Vec::new(), Vec::new(), true); - assert_eq!( - nothing.status, - VulkanStatus::Unknown, - "an empty probe must not claim the driver is absent" - ); - assert!( - nothing - .detail - .as_deref() - .is_some_and(|detail| detail.contains("confirms")), - "the detail must say the absence is unconfirmed: {nothing:?}" - ); - assert!(nothing.driver_label().is_none()); - - // What `Absent` is now reserved for: something WAS found and ruled out. - let software_only = - classify_vulkan_support(vec!["lvp_icd.x86_64.json".to_owned()], Vec::new(), true); - assert_eq!(software_only.status, VulkanStatus::Absent); - - // macOS and friends: no evidence either way, which must not read as - // "no Vulkan" any more than it reads as "Vulkan works". - let unprobeable = classify_vulkan_support(Vec::new(), Vec::new(), false); - assert_eq!(unprobeable.status, VulkanStatus::Unknown); - assert!(!unprobeable.status.is_usable()); - } - - #[test] - fn an_unknown_vendors_icd_is_assumed_to_be_hardware() { - // A vendor the table does not list yet must not be mistaken for a - // software rasteriser; the device enumeration corrects it if wrong. - assert!(icd_is_gpu_driver("some_new_vendor_icd.x86_64.json")); - assert_eq!(icd_driver_label("some_new_vendor_icd.x86_64.json"), None); - - let support = classify_vulkan_support( - vec!["some_new_vendor_icd.x86_64.json".to_owned()], - Vec::new(), - true, - ); - assert_eq!(support.status, VulkanStatus::DriverPresentDevicesUnverified); - } - - #[test] - fn a_discrete_amd_card_is_preferred_over_a_software_device_in_the_same_report() { - let mixed = "\ -GPU0: -\tvendorID = 0x10005 -\tdeviceType = PHYSICAL_DEVICE_TYPE_CPU -\tdeviceName = llvmpipe -\tdriverName = llvmpipe -GPU1: -\tvendorID = 0x1002 -\tdeviceType = PHYSICAL_DEVICE_TYPE_DISCRETE_GPU -\tdeviceName = AMD Radeon RX 7900 XTX -\tdriverName = radv -"; - let support = classify_vulkan_support(Vec::new(), parse_vulkaninfo_summary(mixed), true); - - assert_eq!(support.status, VulkanStatus::Available); - assert_eq!(support.driver_label().as_deref(), Some("radv")); - assert!( - support - .detail - .as_deref() - .is_some_and(|detail| detail.contains("AMD Radeon RX 7900 XTX")), - "{support:?}" - ); - } - - #[test] - fn a_non_amd_gpu_does_not_make_vulkan_an_amd_path() { - let nvidia = "\ -GPU0: -\tvendorID = 0x10de -\tdeviceType = PHYSICAL_DEVICE_TYPE_DISCRETE_GPU -\tdeviceName = NVIDIA GeForce GTX 1050 Ti -\tdriverName = NVIDIA -"; - let support = classify_vulkan_support( - vec!["nvidia_icd.json".to_owned()], - parse_vulkaninfo_summary(nvidia), - true, - ); - - // The driver is installed and is not software, so the honest answer is - // "unverified for AMD", not "available". - assert_eq!(support.status, VulkanStatus::DriverPresentDevicesUnverified); - } - - #[test] - fn the_windows_registry_listing_is_read_into_icd_paths() { - let output = "ICD\tC:\\Windows\\System32\\DriverStore\\FileRepository\\u0397342.inf_amd64_x\\amdvlk64.json\nICD\tC:\\Windows\\SysWOW64\\amdvlk32.json\n"; - - let icds = parse_windows_vulkan_icd_listing(output); - - assert_eq!(icds.len(), 2, "{icds:?}"); - assert!(icds[0].ends_with("amdvlk64.json")); - assert_eq!(icd_driver_label(&icds[0]), Some("AMDVLK")); - assert!( - icd_is_gpu_driver(&icds[0]), - "a Windows path separator must not defeat the filename table" - ); - } - - /// Both Khronos keys must be queried. A 32-bit AMDVLK registration lives - /// only under `WOW6432Node`, so reading the 64-bit key alone would report - /// "no Vulkan driver" on a host that has one. - #[test] - fn the_windows_icd_query_reads_both_the_native_and_the_wow6432_key() { - assert!( - WINDOWS_VULKAN_ICD_SCRIPT.contains(r"HKLM:\SOFTWARE\Khronos\Vulkan\Drivers"), - "{WINDOWS_VULKAN_ICD_SCRIPT}" - ); - assert!( - WINDOWS_VULKAN_ICD_SCRIPT - .contains(r"HKLM:\SOFTWARE\WOW6432Node\Khronos\Vulkan\Drivers"), - "{WINDOWS_VULKAN_ICD_SCRIPT}" - ); - // A missing key is ordinary on a host with no Vulkan at all, so the - // script must test for it rather than let PowerShell error out. - assert!( - WINDOWS_VULKAN_ICD_SCRIPT.contains("Test-Path"), - "{WINDOWS_VULKAN_ICD_SCRIPT}" - ); - // The rows the parser consumes. - assert!( - WINDOWS_VULKAN_ICD_SCRIPT.contains("\"ICD`t$name\""), - "{WINDOWS_VULKAN_ICD_SCRIPT}" - ); - } - - #[test] - fn an_empty_registry_key_yields_no_icds_rather_than_a_blank_entry() { - assert!(parse_windows_vulkan_icd_listing("").is_empty()); - assert!(parse_windows_vulkan_icd_listing("ICD\t\n").is_empty()); - assert!(parse_windows_vulkan_icd_listing("unrelated output\n").is_empty()); - } - - #[test] - fn a_vulkaninfo_report_with_no_devices_yields_none() { - assert!(parse_vulkaninfo_summary("Vulkan Instance Version: 1.3.301\n").is_empty()); - assert!(parse_vulkaninfo_summary("").is_empty()); - } -} diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index aa33884fee..c5ea6662a8 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -139,12 +139,12 @@ pub use gfx_target::{ }; pub use hardware_root::host_path; pub use host_gpu::{ - DriverSummary, ExamineSummary, HostGpuSummary, VulkanDevice, VulkanStatus, VulkanSupport, - WSL_ROCDXG_READY, WindowsGpuListing, WslHostDriverProbe, WslSummary, default_engine_for_host, - default_engine_for_platform, detect_gpu_driver_version, detect_host_gfx_target, - detect_host_gpu_diagnostics, detect_host_gpu_summary, detect_system_ram_gib, - detect_vulkan_support, has_usable_amd_gpu, is_amdgpu_device, is_wsl_host, pinned_gpu_name, - preferred_serve_engine_for_host_gpu_summary, usable_amd_gpu_indices, windows_amd_gpu_listing, + DriverSummary, ExamineSummary, HostGpuSummary, WSL_ROCDXG_READY, WindowsGpuListing, + WslHostDriverProbe, WslSummary, default_engine_for_host, default_engine_for_platform, + detect_gpu_driver_version, detect_host_gfx_target, detect_host_gpu_diagnostics, + detect_host_gpu_summary, detect_system_ram_gib, has_usable_amd_gpu, is_amdgpu_device, + is_wsl_host, pinned_gpu_name, preferred_serve_engine_for_host_gpu_summary, + usable_amd_gpu_indices, windows_amd_gpu_listing, }; #[cfg(any(target_os = "linux", test))] use host_gpu::{KfdGpuNode, kfd_gpu_nodes_in}; diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 2dd776d15b..67511ad01d 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -108,15 +108,6 @@ pub struct EngineStdioOptions<'a> { /// its HTTP probe of a protected endpoint authenticates. The caller decides /// whether a file holds a usable key; this only carries it. pub endpoint_key_file: Option<&'a Path>, - /// With no `endpoint_key_file`, also CLEAR [`ENDPOINT_API_KEY_FILE_ENV`] - /// rather than leave whatever this process inherited to pass through. - /// - /// For a child that must not see a credential path at all — `rocm`'s - /// external engine plugins are third-party binaries. Off by default, - /// because "no key file for this request" and "this child may not have one" - /// are different statements: `rocmd` passes `None` for a loopback service - /// that simply needs no key, and its children are its own. - pub withhold_inherited_key_file: bool, /// Ask the engine for progress on stderr ([`ENGINE_PROGRESS_STDERR_ENV`]). pub request_progress: bool, /// Write the child's stderr to this process's stderr as it arrives, so a @@ -164,12 +155,13 @@ where } if let Some(key_file) = options.endpoint_key_file { command.env(ENDPOINT_API_KEY_FILE_ENV, key_file); - } else if options.withhold_inherited_key_file { - // Cleared, not merely left unset: a value this process inherited must - // not reach a third-party plugin binary just because nothing - // overwrote it. Only for callers that ask — `rocmd` passes `None` for - // a loopback service that needs no key, and clearing there would - // change what its own children see. See + } else { + // Cleared, not merely left unset. The caller said this child gets no + // endpoint key, and an engine plugin is a third-party binary: a value + // this process happens to have inherited must not reach it just + // because nothing overwrote it. `rocm`'s plugin calls pass `None` + // here, so this is what keeps a 0600 key-file path off their + // environment. See // `an_external_plugin_child_is_not_handed_the_endpoint_key_file`. command.env_remove(ENDPOINT_API_KEY_FILE_ENV); } @@ -714,7 +706,6 @@ mod tests { app_paths: None, env_root: None, endpoint_key_file: None, - withhold_inherited_key_file: true, request_progress: false, forward_stderr: false, note_failing_exit_after_answer: false, @@ -731,27 +722,6 @@ mod tests { "the key-file variable must be explicitly CLEARED when no key file \ is passed, not merely left unset" ); - - // The other direction, which is what keeps this scoped: a caller that - // did not ask (`rocmd`, for a loopback service that simply needs no - // key) must leave the child's inherited value alone. - let kept = engine_stdio_command( - Path::new("/an/engine"), - ["stdio"], - &EngineStdioOptions { - app_paths: None, - env_root: None, - endpoint_key_file: None, - withhold_inherited_key_file: false, - request_progress: false, - forward_stderr: false, - note_failing_exit_after_answer: false, - }, - ); - assert!( - kept.get_envs().all(|(key, _)| key != key_var), - "a caller that did not ask to withhold must not touch the variable" - ); } #[test] @@ -768,7 +738,6 @@ mod tests { app_paths: Some(&paths), env_root: Some(Path::new("/envs")), endpoint_key_file: Some(Path::new("/keys/svc.endpoint-key")), - withhold_inherited_key_file: false, request_progress: true, forward_stderr: true, note_failing_exit_after_answer: false, @@ -802,7 +771,6 @@ mod tests { "ROCM_SERVE_API_KEY_FILE", EngineStdioOptions { endpoint_key_file: Some(Path::new("/keys/k")), - withhold_inherited_key_file: false, ..EngineStdioOptions::default() }, ), @@ -1224,7 +1192,6 @@ mod tests { app_paths: Some(&paths), env_root: Some(Path::new("/envs")), endpoint_key_file: Some(Path::new("/keys/k")), - withhold_inherited_key_file: false, request_progress: true, forward_stderr: false, note_failing_exit_after_answer: false, diff --git a/docs/architecture.md b/docs/architecture.md index e33121035f..7270fd1b87 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -203,8 +203,8 @@ Already-extracted subsystem modules, following the same full-domain-extraction p - [`gguf.rs`](../crates/rocm-core/src/gguf.rs) — reads the shape of a GGUF model (layers, KV heads, head width, parameter count, data offset) from its header over any `Read + Seek`, never touching the weights. - [`gguf_fit.rs`](../crates/rocm-core/src/gguf_fit.rs) — pure arithmetic over that header: weights + KV cache at a context size and KV type + compute buffer + runtime reserve against dedicated VRAM, the largest context that fits, and what would make a model that does not fit, fit. - [`gguf_source.rs`](../crates/rocm-core/src/gguf_source.rs) — where a header comes from: a local file, or a Hugging Face file, found by one repository-listing request (and a `HEAD` request for a file the listing does not size) and read by HTTP range requests (`HF_ENDPOINT` selects a mirror; redirects are followed by hand, up to five, and never from https to plain http). The only code in the estimate that reads a file or the network. -- [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) — (ROCMAI-85, Phase 7a of ROCMAI-27's sequencing) GPU/host hardware detection — see the module's own doc comment for what it owns. Its leaf concerns are child modules under `host_gpu/`, with `host_gpu.rs` kept as the module root: [`capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs) (running an optional tool under a timeout, trying each spelling [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs) gives for it), [`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) (kernel version, distro name, installed RAM, and the CPU model and vendor, which one reader returns together for both the summary and `Examination::probe`: Linux from `/proc/cpuinfo`, Windows from the CPU row of the system inventory), [`linux.rs`](../crates/rocm-core/src/host_gpu/linux.rs) (the sysfs, KFD and DRM probes), [`gpu_visibility.rs`](../crates/rocm-core/src/host_gpu/gpu_visibility.rs) (which AMD GPUs this process may use, from the KFD/DRM counts and the `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` masks, plus `windows_amd_gpu_listing`, the native-Windows adapter bound `rocm serve --gpu ` is checked against), [`windows_inventory.rs`](../crates/rocm-core/src/host_gpu/windows_inventory.rs) (the Windows display-adapter inventory and its parsers, including the AMD-adapter count behind `windows_amd_gpu_listing` and the system query whose CPU row the CPU reader in `os_facts.rs` takes), [`wsl.rs`](../crates/rocm-core/src/host_gpu/wsl.rs) (WSL detection, the librocdxg chain and the Windows host driver as seen from the guest), [`render.rs`](../crates/rocm-core/src/host_gpu/render.rs) (the `rocm examine` text of an `ExamineSummary`, as a second `impl` block), [`vulkan.rs`](../crates/rocm-core/src/host_gpu/vulkan.rs) (whether a model can run through Vulkan here, read from ICD manifests, the Windows Khronos registry and `vulkaninfo --summary` — the evidence `gpu_support_level` in [`gfx_target.rs`](../crates/rocm-core/src/gfx_target.rs) needs to tell a GPU with no ROCm build that still has a Vulkan driver from one with no GPU path at all, and where a software rasteriser is recognised in order to be excluded), [`diagnostics.rs`](../crates/rocm-core/src/host_gpu/diagnostics.rs) (the GPU detection diagnostics text) and [`engine_default.rs`](../crates/rocm-core/src/host_gpu/engine_default.rs) (the default serve engine, and the vLLM preference for the TheRock families that serve through it). -- [`host-probing.md`](host-probing.md) — a design note, not a module: [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) and [`examine.rs`](../crates/rocm-core/src/examine.rs) are two separate host-probing stacks that read some of the same facts twice (the Windows GPU query, the marketing-name table and the subprocess runners). It proposes which of each pair should survive, how the two differ and the order in which they could converge, and records what each convergence decided; it changes no code itself. Three pairs are converged: the `PATH` lookup ([`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs)'s `which` is now the only copy) and the kernel release ([`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs)'s `detect_kernel_version` is now the only `uname -r` reader, which [`host_env.rs`](../crates/rocm-core/src/examine/host_env.rs)'s `probe_os` calls for `kernel_release`, and `rocm examine --json` runs it once: [`examine.rs`](../crates/rocm-core/src/examine.rs)'s `Examination::probe_with_summary` probes the host and hands the examination's kernel release to the summary), and the CPU model ([`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) has one reader per platform for the model and vendor, which `probe_cpu_linux` and `probe_cpu_windows` record and the summary reads). The first step of the subprocess runners is delivered too: [`process/bounded.rs`](../crates/rocm-core/src/process/bounded.rs)'s `run_bounded` is the executor behind `run`, `capture_optional_command*` and the Windows diagnostic capture. It is a snapshot, to be retired when the convergence tickets close. +- [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) — (ROCMAI-85, Phase 7a of ROCMAI-27's sequencing) GPU/host hardware detection — see the module's own doc comment for what it owns. Its leaf concerns are child modules under `host_gpu/`, with `host_gpu.rs` kept as the module root: [`capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs) (running an optional tool under a timeout, trying each spelling [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs) gives for it), [`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) (kernel version, distro name, installed RAM, and the CPU model and vendor, which one reader returns together for both the summary and `Examination::probe`: Linux from `/proc/cpuinfo`, Windows from the CPU row of the system inventory), [`linux.rs`](../crates/rocm-core/src/host_gpu/linux.rs) (the sysfs, KFD and DRM probes), [`gpu_visibility.rs`](../crates/rocm-core/src/host_gpu/gpu_visibility.rs) (which AMD GPUs this process may use, from the KFD/DRM counts and the `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` masks, plus `windows_amd_gpu_listing`, the native-Windows adapter bound `rocm serve --gpu ` is checked against), [`windows_inventory.rs`](../crates/rocm-core/src/host_gpu/windows_inventory.rs) (the Windows display-adapter inventory and its parsers, including the AMD-adapter count behind `windows_amd_gpu_listing` and the system query whose CPU row the CPU reader in `os_facts.rs` takes), [`wsl.rs`](../crates/rocm-core/src/host_gpu/wsl.rs) (WSL detection, the librocdxg chain and the Windows host driver as seen from the guest), [`render.rs`](../crates/rocm-core/src/host_gpu/render.rs) (the `rocm examine` text of an `ExamineSummary`, as a second `impl` block), [`diagnostics.rs`](../crates/rocm-core/src/host_gpu/diagnostics.rs) (the GPU detection diagnostics text) and [`engine_default.rs`](../crates/rocm-core/src/host_gpu/engine_default.rs) (the default serve engine, and the vLLM preference for the TheRock families that serve through it). +- [`host-probing.md`](host-probing.md) — a design note, not a module: [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) and [`examine.rs`](../crates/rocm-core/src/examine.rs) are two separate host-probing stacks that read some of the same facts twice (the Windows GPU query, the marketing-name table and the subprocess runners). It proposes which of each pair should survive, how the two differ and the order in which they could converge, and records what each convergence decided; it changes no code itself. Three pairs are converged: the `PATH` lookup ([`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs)'s `which` is now the only copy) and the kernel release ([`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs)'s `detect_kernel_version` is now the only `uname -r` reader, which [`host_env.rs`](../crates/rocm-core/src/examine/host_env.rs)'s `probe_os` calls for `kernel_release`, and `rocm examine --json` runs it once: [`examine.rs`](../crates/rocm-core/src/examine.rs)'s `Examination::probe_with_summary` probes the host and hands the examination's kernel release to the summary), and the CPU model ([`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) has one reader per platform for the model and vendor, which `probe_cpu_linux` and `probe_cpu_windows` record and the summary reads). It is a snapshot, to be retired when the convergence tickets close. - [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs) — the JSONL store engine behind the automation stores: the sibling `.lock` file (`jsonl_lock_path`, re-exported at the crate root), single-line appends, torn-tail repair, size-based rotation (`JsonlRotation`) and the tail-first readers that span rotated generations. It is generic over the record type; [`automation.rs`](../crates/rocm-core/src/automation.rs) supplies the record types and each store's policy, and nothing in this module depends on it, in production or in its own tests: those drive the engine with a small record type of their own. The module is private; its engine items are `pub(crate)`. - [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) — (ROCMAI-85, Phase 7a) the TheRock managed-runtime-environment surface — see the module's own doc comment for what it owns. - [`model_catalog.rs`](../crates/rocm-core/src/model_catalog.rs) — (ROCMAI-152, Phase 7c) the curated model catalog's types, built-in catalog JSON loading, signed external recipe indexes, and artifact cache-marker bookkeeping — see the module's own doc comment for what it owns. diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index e64ae34929..11e49f5f4d 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -752,40 +752,3 @@ Feature: GPU detection and system inspection Then the inspection states this host's ROCm support level And the support level agrees with the GPU the inspection detected And the support level leaves a command that fits this host - - # The three below close the gap this PR's own notes call unclosable: the - # `VulkanOnly` door needs a Renoir/Cezanne Vega APU, and no runner has one. - # It does not need the hardware, only the two signals the CLI reads for it -- - # a KFD node naming the target, and the host's Vulkan driver state -- both of - # which the harness can plant. So these run on the ungated mock lane, which is - # the lane that has neither. - # - # They exist because the support level used to be inferred: "no ROCm build" - # was taken to mean "Vulkan then", on a machine nobody had looked at. Each - # scenario pins one of the three answers that inference could not tell apart. - @id:examine-names-the-vulkan-driver-an-unsupported-gpu-can-use @requires-os:linux - Scenario: examine-50 - A GPU the official stack rejects is told which way it can still run models - Given a machine whose GPU has no official support but has a working Vulkan driver - When the user inspects that machine - Then the inspection reports the GPU as unsupported by the official stack - And the inspection reports that models can still run through Vulkan - And the inspection names the Vulkan driver it found - - @id:examine-does-not-promise-vulkan-without-a-driver @requires-os:linux - Scenario: examine-51 - A machine with no way to run models is not told it has one - Given a machine whose GPU has no official support and no Vulkan driver - When the user inspects that machine - Then the inspection reports the GPU as unsupported by the official stack - And the inspection does not claim models can run through Vulkan - And the inspection says no Vulkan driver was found - - # A software renderer is a complete Vulkan implementation that runs on the - # CPU. Counting it would put a GPU-required run on the processor while - # reporting a GPU path - the one outcome the no-fallback rule exists to - # prevent, reached through a backend rather than a policy change. - @id:examine-does-not-count-a-software-renderer-as-a-gpu-path @requires-os:linux - Scenario: examine-52 - A software renderer does not count as a way to run on the GPU - Given a machine whose GPU has no official support and only a software renderer - When the user inspects that machine - Then the inspection does not claim models can run through Vulkan - And the inspection says the only renderer it found runs on the CPU diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index a5bc2b38c3..8fbe54b3cf 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -1704,25 +1704,6 @@ async fn assert_no_amdgpu_fault_evidence(world: &mut E2eWorld) { } } -/// The human-readable twin of the machine-readable planted-machine step, for -/// scenarios whose subject is a sentence the report prints rather than a field -/// it serialises. -/// -/// It has to be its own step rather than reusing "the user inspects the -/// system", because that one runs through `run_rocm`, which does not consume -/// `World::command_env` -- so a planted host root never reaches the child and -/// the CLI silently describes the *runner* instead. Not hypothetical: it is -/// what made the first draft of examine-50/51/52 report the lane's own MI300X -/// as `gfx942` while claiming to be about a Renoir APU. -#[when("the user inspects that machine")] -async fn user_inspects_planted_machine_text(world: &mut E2eWorld) { - let (stdout, stderr, rc) = - crate::run_rocm_with_scenario_env(world, &["examine", "--framework", "skip"]); - world.cli_output = Some(stdout); - world.cli_stderr = Some(stderr); - world.cli_rc = Some(rc); -} - #[when("the user inspects that machine in machine-readable form")] async fn user_inspects_planted_machine(world: &mut E2eWorld) { let (stdout, stderr, rc) = @@ -3366,208 +3347,3 @@ async fn assert_support_level_advice_fits(world: &mut E2eWorld) { ); } } - -/// A KFD `gfx_target_version` naming gfx90c, the Vega iGPU in Renoir / -/// Lucienne / Cezanne / Barcelo APUs. -/// -/// The kernel packs the target as `major * 10000 + minor * 100 + step`, so -/// `9.0.12` is `90012` and decodes to `gfx90c` (step 12 is the hex `c`). This -/// is the one target the CLI recognizes *and* refuses to map to a ROCm package -/// family, so it is what reaches the support level these scenarios are about. -const KFD_GFX90C_TARGET_VERSION: &str = "90012"; - -/// `vulkaninfo --summary` on a host whose Vega iGPU is driven by Mesa RADV. -const RADV_VULKANINFO_SUMMARY: &str = "\ -Devices: -======== -GPU0: -\tapiVersion = 1.3.274 -\tvendorID = 0x1002 -\tdeviceID = 0x1636 -\tdeviceType = PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU -\tdeviceName = AMD Radeon Graphics (RADV RENOIR) -\tdriverID = DRIVER_ID_MESA_RADV -\tdriverName = radv -"; - -/// `vulkaninfo --summary` on a host with only Mesa's CPU rasteriser: a complete -/// Vulkan implementation, and not a GPU. -const LAVAPIPE_VULKANINFO_SUMMARY: &str = "\ -Devices: -======== -GPU0: -\tapiVersion = 1.3.255 -\tvendorID = 0x10005 -\tdeviceType = PHYSICAL_DEVICE_TYPE_CPU -\tdeviceName = llvmpipe (LLVM 15.0.7, 256 bits) -\tdriverName = llvmpipe -"; - -/// Plant a host whose GPU is identified but has no ROCm package family, -/// carrying the Vulkan state the caller describes. -/// -/// `icd_manifest` is the ICD file name the loader would find and -/// `vulkaninfo_summary` what `vulkaninfo --summary` prints; `None` for either -/// means that half of the evidence is absent. -/// -/// The `vulkaninfo` stub is planted in **both** directions. When a scenario -/// wants the tool absent it is planted as a failing command rather than left -/// out, because `PATH` still reaches the rest of the machine: a runner with a -/// real `vulkaninfo` installed would otherwise answer for it, and a scenario -/// about a host with no Vulkan driver has to be hermetic about the host and not -/// only about the planted root. -fn plant_unsupported_gpu( - world: &mut E2eWorld, - icd_manifest: Option<&str>, - vulkaninfo_summary: Option<&str>, -) { - let root = world - .isolated_root - .as_ref() - .expect("no isolated root") - .path() - .to_owned(); - let host_root = root.join("host-root"); - - let node = host_root.join("sys/class/kfd/kfd/topology/nodes/1"); - std::fs::create_dir_all(&node).expect("failed to create planted KFD node"); - std::fs::write( - node.join("properties"), - format!( - "cpu_cores_count 0\nsimd_count 56\ngfx_target_version {KFD_GFX90C_TARGET_VERSION}\n" - ), - ) - .expect("failed to write planted KFD node properties"); - - if let Some(manifest) = icd_manifest { - let icd_dir = host_root.join("usr/share/vulkan/icd.d"); - std::fs::create_dir_all(&icd_dir).expect("failed to create planted ICD dir"); - std::fs::write( - icd_dir.join(manifest), - "{\"file_format_version\":\"1.0.0\",\"ICD\":{\"library_path\":\"libvulkan_fake.so\"}}\n", - ) - .expect("failed to write planted ICD manifest"); - } - - let bin = root.join("fake-bin"); - std::fs::create_dir_all(&bin).expect("failed to create the fake PATH dir"); - // The same empty-bus `lspci` the other GPU fixtures plant: the probe must - // succeed without describing the runner's real cards. - rocm_core::fake_executable::write_fake_executable(&bin.join("lspci"), "#!/bin/sh\nexit 0\n") - .expect("failed to write the fake lspci"); - let vulkaninfo = match vulkaninfo_summary { - Some(summary) => format!("#!/bin/sh\ncat <<'ROCM_E2E_EOF'\n{summary}ROCM_E2E_EOF\n"), - None => "#!/bin/sh\nexit 1\n".to_owned(), - }; - rocm_core::fake_executable::write_fake_executable(&bin.join("vulkaninfo"), &vulkaninfo) - .expect("failed to write the fake vulkaninfo"); - - let path = std::env::join_paths(std::iter::once(bin).chain(std::env::split_paths( - &std::env::var_os("PATH").unwrap_or_default(), - ))) - .expect("PATH entries cannot hold a separator"); - world.command_env.push(("PATH", path)); - world - .command_env - .push(("ROCM_CLI_TEST_HOST_ROOT", host_root.into_os_string())); -} - -#[given("a machine whose GPU has no official support but has a working Vulkan driver")] -async fn unsupported_gpu_with_vulkan(world: &mut E2eWorld) { - plant_unsupported_gpu( - world, - Some("radeon_icd.x86_64.json"), - Some(RADV_VULKANINFO_SUMMARY), - ); -} - -#[given("a machine whose GPU has no official support and no Vulkan driver")] -async fn unsupported_gpu_without_vulkan(world: &mut E2eWorld) { - plant_unsupported_gpu(world, None, None); -} - -#[given("a machine whose GPU has no official support and only a software renderer")] -async fn unsupported_gpu_with_software_renderer(world: &mut E2eWorld) { - plant_unsupported_gpu( - world, - Some("lvp_icd.x86_64.json"), - Some(LAVAPIPE_VULKANINFO_SUMMARY), - ); -} - -#[then("the inspection reports the GPU as unsupported by the official stack")] -async fn assert_gpu_unsupported_by_rocm(world: &mut E2eWorld) { - let output = world.cli_output.as_ref().expect("no command was run"); - assert!( - output.contains("rocm_support:") && output.contains("not supported by ROCm"), - "the inspection must say the GPU was identified and has no ROCm build:\n{output}" - ); -} - -#[then("the inspection reports that models can still run through Vulkan")] -async fn assert_vulkan_offered(world: &mut E2eWorld) { - let output = world.cli_output.as_ref().expect("no command was run"); - assert!( - output.contains("the Vulkan backend can be used"), - "a host with a working Vulkan device must be told so:\n{output}" - ); - assert!( - output.contains("vulkan: available"), - "the machine-readable line must agree with the sentence:\n{output}" - ); -} - -#[then("the inspection names the Vulkan driver it found")] -async fn assert_vulkan_driver_named(world: &mut E2eWorld) { - let output = world.cli_output.as_ref().expect("no command was run"); - // The driver the planted device reported, not a guess from the ICD file - // name: naming it is what lets a reader check the claim. - assert!( - output.contains("radv"), - "the inspection must name the driver it found:\n{output}" - ); -} - -#[then("the inspection does not claim models can run through Vulkan")] -async fn assert_vulkan_not_promised(world: &mut E2eWorld) { - let output = world.cli_output.as_ref().expect("no command was run"); - assert!( - !output.contains("the Vulkan backend can be used"), - "a host with no usable Vulkan GPU must not be promised one:\n{output}" - ); - assert!( - !output.contains("Vulkan backend is expected to work"), - "a host with no usable Vulkan GPU must not be promised one:\n{output}" - ); -} - -#[then("the inspection says no Vulkan driver was found")] -async fn assert_no_vulkan_driver_reported(world: &mut E2eWorld) { - let output = world.cli_output.as_ref().expect("no command was run"); - assert!( - output.contains("no Vulkan GPU driver was found"), - "the inspection must say why no GPU path is open:\n{output}" - ); - assert!( - output.contains("install one"), - "the advice must be able to clear the condition it is printed for:\n{output}" - ); -} - -#[then("the inspection says the only renderer it found runs on the CPU")] -async fn assert_software_renderer_reported(world: &mut E2eWorld) { - let output = world.cli_output.as_ref().expect("no command was run"); - assert!( - output.contains("not a GPU path"), - "a CPU rasteriser must be named as such, not counted as a GPU:\n{output}" - ); - // The support level must agree, not just the evidence line. Without this the - // scenario still passed when `gpu_support_level` ignored the probe entirely: - // the absence of a Vulkan promise was coming from the status wording alone, - // so nothing here held the classification itself. - assert!( - output.contains("no Vulkan GPU driver was found"), - "a host whose only renderer is the CPU has no GPU backend, and the support \ - line must say so:\n{output}" - ); -} From 773129e859b4197664f06d0fdbc66bd0abbd8f87 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 15:26:49 +0200 Subject: [PATCH 20/59] feat(examine): probe Vulkan instead of inferring it from the gfx table MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Split out of #678 on review, and stacked on it: the classifier this extends is that PR's. `rocm examine` told a gfx90c APU it could run models through Lemonade's Vulkan backend on the strength of the gfx table alone — nothing had looked for a driver. On a host with none that is an instruction the user cannot carry out. Probe instead: ICD manifests, the Windows Khronos registry, and `vulkaninfo --summary` where it exists. A fourth support level, `NoGpuBackend`, carries the confirmed answer, and a `vulkan:` line prints what was found. `Absent` is reserved for a probe that found something and ruled it out — a software-only ICD, or an enumeration containing only software devices. No ICD *and* no enumerated device is `Unknown`, because neither source is exhaustive: the Linux ICD search is a fixed directory list that ignores `VK_ICD_FILENAMES`/`VK_DRIVER_FILES`, Windows reads only the legacy Khronos key, and `vulkaninfo` is frequently not installed. Inferring absence from that would tell a host with a working driver to install one. A software rasteriser never counts as a GPU path. examine-50/51/52 cover the three answers on the ungated mock lane, which needs neither a Renoir APU nor a Vulkan driver — only the KFD node and ICD state the harness can plant. Falsified: both negative scenarios fail when `gpu_support_level` is made to ignore the probe. Signed-off-by: Roman Sirokov --- apps/rocm/src/bridge_snapshot.rs | 1 + apps/rocm/src/driver_plan.rs | 1 + apps/rocm/src/examine_report_text.rs | 10 +- apps/rocm/src/main.rs | 24 +- crates/rocm-core/src/gfx_target.rs | 132 +++- crates/rocm-core/src/host_gpu.rs | 17 + crates/rocm-core/src/host_gpu/render.rs | 207 ++++- crates/rocm-core/src/host_gpu/vulkan.rs | 726 ++++++++++++++++++ crates/rocm-core/src/lib.rs | 12 +- .../rocm-engine-protocol/src/stdio_client.rs | 49 -- docs/architecture.md | 4 +- tests/e2e-cucumber/features/examine.feature | 37 + tests/e2e-cucumber/tests/e2e/examine_steps.rs | 224 ++++++ 13 files changed, 1359 insertions(+), 85 deletions(-) create mode 100644 crates/rocm-core/src/host_gpu/vulkan.rs diff --git a/apps/rocm/src/bridge_snapshot.rs b/apps/rocm/src/bridge_snapshot.rs index 741c18ede7..115279e42a 100644 --- a/apps/rocm/src/bridge_snapshot.rs +++ b/apps/rocm/src/bridge_snapshot.rs @@ -40,6 +40,7 @@ pub(crate) fn build_codex_bridge_snapshot(paths: &AppPaths) -> Result) render_engine_inventory_text_for_host( paths, rocm_core::default_engine_for_host(&host_gpu), - rocm_core::gpu_support_level_for_host_gpu(&host_gpu), + // No Vulkan probe on this listing path, for the reason given on + // `host_gpu_support_level`: `rocm engines` is cheap by design, and + // `rocm examine` is where the verified answer lives. + rocm_core::gpu_support_level_for_host_gpu(&host_gpu, rocm_core::VulkanStatus::Unknown), ) } @@ -634,6 +641,7 @@ mod tests { detected_gfx_target: None, compatible_therock_family: None, detected_therock_family: None, + vulkan: None, driver: rocm_core::DriverSummary { policy: "linux_official_amd_dkms_wrapper".to_owned(), status: "amdgpu_available".to_owned(), diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index c18432bc61..3887afd02e 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -1081,6 +1081,15 @@ const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str rocm_core::GpuSupportLevel::RocmHip => { "default embedded Lemonade server with ROCm llama.cpp backend" } + // Only `rocm examine` reaches this level — the cheap probe behind + // `host_gpu_support_level` passes `VulkanStatus::Unknown` and so lands + // on `VulkanOnly` — but the arm is spelled out rather than folded into + // `Unknown`, because what it means is the opposite: `Unknown` promises + // that a command can still answer, and here the question is already + // answered and the answer is no. + rocm_core::GpuSupportLevel::NoGpuBackend => { + "default embedded Lemonade server; this GPU has no ROCm/HIP support and no Vulkan driver was found, so no GPU backend can run here" + } rocm_core::GpuSupportLevel::Unknown => { "default embedded Lemonade server; run `rocm examine` for this host's GPU support level, which says how to find the backend" } @@ -1093,8 +1102,21 @@ const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str /// The same two inputs `rocm examine` derives it from, read through /// `detect_host_gpu_summary` rather than the full examine gather — exactly what /// `render_engine_inventory_text_with_paths` (i.e. `rocm engines`) already does. +/// +/// **The Vulkan probe is deliberately not run here**, so this stays the cheap +/// probe its callers chose it for: the probe spawns `vulkaninfo` on Linux and +/// PowerShell on Windows, and `rocm engines` is a listing command that would +/// pay that on every invocation. Passing [`VulkanStatus::Unknown`] keeps these +/// surfaces answering exactly as they did before this probe existed — a +/// Vulkan-capable-by-the-table host still reads `VulkanOnly` — while `rocm +/// examine`, which already gathers the probe, is the surface that can tell a +/// host with a Vulkan driver from one without. That split is why +/// `lemonade_engine_note` sends the user to `rocm examine` for a real answer. pub(crate) fn host_gpu_support_level(paths: Option<&AppPaths>) -> rocm_core::GpuSupportLevel { - rocm_core::gpu_support_level_for_host_gpu(&rocm_core::detect_host_gpu_summary(paths)) + rocm_core::gpu_support_level_for_host_gpu( + &rocm_core::detect_host_gpu_summary(paths), + rocm_core::VulkanStatus::Unknown, + ) } fn refresh_startup_update_check_quietly() { diff --git a/crates/rocm-core/src/gfx_target.rs b/crates/rocm-core/src/gfx_target.rs index 092c288a46..8b843bb6b0 100644 --- a/crates/rocm-core/src/gfx_target.rs +++ b/crates/rocm-core/src/gfx_target.rs @@ -566,23 +566,63 @@ pub(crate) fn gfx_target_from_gc_version(major: u32, minor: u32, revision: u32) pub enum GpuSupportLevel { /// A TheRock package family covers this GPU: full ROCm/HIP support. RocmHip, - /// Recognized, confirmed to have no ROCm/HIP build, but Vulkan-based - /// engines (the default Lemonade backend) still work on it. + /// Recognized, confirmed to have no ROCm/HIP build, and a Vulkan GPU driver + /// was **found on this host**, so Vulkan-based engines (the default Lemonade + /// backend) run on it. + /// + /// The second half of that sentence is evidence, not inference. This variant + /// used to be returned from the gfx table alone — "no ROCm build" was taken + /// to mean "Vulkan then", on a machine nobody had looked at. Every surface + /// that renders this level tells the user to go and serve a model on it, so + /// a host with no Vulkan driver installed, or with only a software + /// rasteriser, was sent to a backend that cannot start. VulkanOnly, + /// Recognized, confirmed to have no ROCm/HIP build, **and** no Vulkan GPU + /// driver was found: this host has no GPU path at all. + /// + /// Deliberately not folded into [`Self::Unknown`]. Both halves here are + /// confirmed — the catalog establishes the missing ROCm build, and the + /// Vulkan probe looked and found nothing — whereas `Unknown` means the CLI + /// could not establish either. Collapsing them would commit, in the other + /// direction, the exact conflation this enum's `Unknown` exists to prevent: + /// a fact about the machine would be reported as a gap in what this build + /// knows, and the user would be sent to probe a question already answered. + NoGpuBackend, /// No GPU detected yet, or a gfx target this catalog doesn't recognize. Unknown, } -/// Classify the host's GPU support level from the same two facts -/// `ExamineSummary` and `resolve_family` already compute: the detected gfx -/// target and whichever TheRock family (if any) covers it. +/// Classify the host's GPU support level. +/// +/// Reads the two facts `ExamineSummary` and `resolve_family` already compute — +/// the detected gfx target and whichever TheRock family (if any) covers it — +/// plus what the Vulkan probe found. +/// +/// `vulkan` only ever decides between [`GpuSupportLevel::VulkanOnly`] and +/// [`GpuSupportLevel::NoGpuBackend`], and is consulted only once the catalog has +/// confirmed there is no ROCm build. A host with a covering family is `RocmHip` +/// whatever Vulkan reports, and an unrecognized target stays `Unknown`: finding +/// a Vulkan driver says nothing about whether ROCm would also work there, so it +/// must not be allowed to turn a detection gap into a support claim. +/// +/// [`VulkanStatus::Unknown`] — the platform could not be probed at all — keeps +/// `VulkanOnly`, because withholding the one path such a host has on the +/// strength of a probe that never ran would be worse than the old assumption, +/// not better. pub fn gpu_support_level( detected_gfx_target: Option<&str>, compatible_therock_family: Option<&str>, + vulkan: crate::host_gpu::VulkanStatus, ) -> GpuSupportLevel { match (detected_gfx_target, compatible_therock_family) { (_, Some(_)) => GpuSupportLevel::RocmHip, - (Some(gfx), None) if gfx_target_has_no_rocm_build(gfx) => GpuSupportLevel::VulkanOnly, + (Some(gfx), None) if gfx_target_has_no_rocm_build(gfx) => { + if vulkan.is_usable() || vulkan == crate::host_gpu::VulkanStatus::Unknown { + GpuSupportLevel::VulkanOnly + } else { + GpuSupportLevel::NoGpuBackend + } + } _ => GpuSupportLevel::Unknown, } } @@ -594,17 +634,27 @@ pub fn gpu_support_level( /// site that transposed them would classify a Vulkan-only host as `RocmHip` /// and still compile. `rocm engines` and the codex-bridge `engines` tool both /// derive the level from this struct, so both go through this. +/// +/// `vulkan` is passed in rather than probed here so this stays a pure function +/// of what the caller already holds: the probe runs a subprocess and reads the +/// filesystem, and burying that in a helper this cheap would make every caller +/// pay for it without saying so. #[must_use] -pub fn gpu_support_level_for_host_gpu(host_gpu: &HostGpuSummary) -> GpuSupportLevel { +pub fn gpu_support_level_for_host_gpu( + host_gpu: &HostGpuSummary, + vulkan: crate::host_gpu::VulkanStatus, +) -> GpuSupportLevel { gpu_support_level( host_gpu.gfx_target.as_deref(), host_gpu.therock_family.as_deref(), + vulkan, ) } #[cfg(test)] mod tests { use super::*; + use crate::host_gpu::VulkanStatus; #[test] fn normalize_therock_family_maps_gfx1101_to_gfx110x_all() { @@ -997,7 +1047,7 @@ mod tests { therock_family: Some("gfx94X-dcgpu".to_owned()), }; assert_eq!( - gpu_support_level_for_host_gpu(&host_gpu), + gpu_support_level_for_host_gpu(&host_gpu, VulkanStatus::Available), GpuSupportLevel::RocmHip ); } @@ -1010,7 +1060,7 @@ mod tests { therock_family: None, }; assert_eq!( - gpu_support_level_for_host_gpu(&host_gpu), + gpu_support_level_for_host_gpu(&host_gpu, VulkanStatus::Available), GpuSupportLevel::VulkanOnly ); } @@ -1018,7 +1068,7 @@ mod tests { #[test] fn gpu_support_level_for_host_gpu_reads_an_undetected_summary_as_unknown() { assert_eq!( - gpu_support_level_for_host_gpu(&HostGpuSummary::default()), + gpu_support_level_for_host_gpu(&HostGpuSummary::default(), VulkanStatus::Available), GpuSupportLevel::Unknown ); } @@ -1026,24 +1076,76 @@ mod tests { #[test] fn gpu_support_level_is_rocm_hip_when_a_family_covers_it() { assert_eq!( - gpu_support_level(Some("gfx90a"), Some("gfx90a")), + gpu_support_level(Some("gfx90a"), Some("gfx90a"), VulkanStatus::Available), + GpuSupportLevel::RocmHip + ); + // A covering family wins outright: a host whose Vulkan probe found + // nothing still has ROCm, and must not be downgraded for it. + assert_eq!( + gpu_support_level(Some("gfx90a"), Some("gfx90a"), VulkanStatus::Absent), GpuSupportLevel::RocmHip ); } #[test] fn gpu_support_level_is_unknown_without_detection_or_for_an_unrecognized_target() { - assert_eq!(gpu_support_level(None, None), GpuSupportLevel::Unknown); assert_eq!( - gpu_support_level(Some("gfx_totally_unrecognized"), None), + gpu_support_level(None, None, VulkanStatus::Available), + GpuSupportLevel::Unknown + ); + // Finding a Vulkan driver says nothing about whether ROCm would work on + // an unrecognized target, so it must not promote a detection gap into a + // support claim. + assert_eq!( + gpu_support_level( + Some("gfx_totally_unrecognized"), + None, + VulkanStatus::Available + ), GpuSupportLevel::Unknown ); } #[test] - fn gpu_support_level_is_vulkan_only_for_gfx90c() { + fn gpu_support_level_is_vulkan_only_for_gfx90c_with_a_vulkan_driver() { + assert_eq!( + gpu_support_level(Some("gfx90c"), None, VulkanStatus::Available), + GpuSupportLevel::VulkanOnly + ); + // An installed driver that enumerated no device is still worth routing + // to; the level says "Vulkan", and `examine`'s own line carries how sure + // the probe was. + assert_eq!( + gpu_support_level( + Some("gfx90c"), + None, + VulkanStatus::DriverPresentDevicesUnverified + ), + GpuSupportLevel::VulkanOnly + ); + } + + /// The defect this guards against: `VulkanOnly` used to be inferred from the + /// gfx table alone, so a Renoir laptop with no Vulkan driver installed — or + /// with only a CPU rasteriser — was told to go and serve a model on a + /// backend that cannot start. + #[test] + fn gpu_support_level_reports_no_gpu_backend_when_the_vulkan_probe_found_none() { + assert_eq!( + gpu_support_level(Some("gfx90c"), None, VulkanStatus::Absent), + GpuSupportLevel::NoGpuBackend, + "a confirmed-absent Vulkan driver must not read as a usable Vulkan path" + ); + } + + /// A platform the probe cannot read is not a platform without Vulkan. + /// Withholding the only path such a host has, on the strength of a probe + /// that never ran, would be a worse answer than the assumption this change + /// replaces. + #[test] + fn gpu_support_level_keeps_vulkan_when_the_probe_could_not_run() { assert_eq!( - gpu_support_level(Some("gfx90c"), None), + gpu_support_level(Some("gfx90c"), None, VulkanStatus::Unknown), GpuSupportLevel::VulkanOnly ); } diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index 87b6ad185f..db45107033 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -54,6 +54,7 @@ mod gpu_visibility; pub(crate) mod linux; mod os_facts; mod render; +mod vulkan; mod windows_inventory; mod wsl; @@ -73,6 +74,11 @@ pub(crate) use linux::detect_linux_sysfs_gfx_target; pub use linux::is_amdgpu_device; #[cfg(any(target_os = "linux", test))] pub(crate) use linux::{KfdGpuNode, kfd_gpu_nodes_in}; +pub use vulkan::{ + VulkanDevice, VulkanStatus, VulkanSupport, classify_vulkan_support, detect_vulkan_support, + icd_driver_label, icd_is_gpu_driver, parse_vulkaninfo_summary, + parse_windows_vulkan_icd_listing, +}; pub(crate) use windows_inventory::detect_windows_amd_driver_info; #[cfg(test)] use windows_inventory::parse_windows_examine_inventory; @@ -133,6 +139,15 @@ pub struct ExamineSummary { pub compatible_therock_family: Option, #[serde(default)] pub detected_therock_family: Option, + /// What the Vulkan probe found, when it could run. + /// + /// `#[serde(default)]` is load-bearing in the *reverse* direction: `rocm + /// remote doctor` deserializes summaries produced by other hosts, which may + /// be running a build from before this field existed. `None` means "not + /// probed", which [`ExamineSummary::vulkan_status`] maps to + /// [`VulkanStatus::Unknown`] — never to "no Vulkan". + #[serde(default)] + pub vulkan: Option, pub driver: DriverSummary, pub legacy_rocm: LegacyRocmSummary, #[serde(default)] @@ -284,6 +299,7 @@ impl ExamineSummary { detected_gfx_target, compatible_therock_family, detected_therock_family, + vulkan: Some(detect_vulkan_support()), driver: detect_driver_summary_with_windows_inventory( windows_inventory.as_ref(), wsl.as_ref(), @@ -730,6 +746,7 @@ mod tests { detected_gfx_target: None, compatible_therock_family: None, detected_therock_family: None, + vulkan: None, driver: windows_driver_summary( Some("AMD Radeon(TM) Graphics driver 30.0.13014.8".to_owned()), info, diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index e6f0f479f2..721b6ed6a3 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -6,9 +6,42 @@ use super::ExamineSummary; use super::os_facts::format_gib_value; +use super::vulkan::VulkanStatus; use crate::gfx_target::{GpuSupportLevel, gpu_support_level}; impl ExamineSummary { + /// What the Vulkan probe found, or [`VulkanStatus::Unknown`] when this + /// summary carries no probe result — an older host's JSON read back by + /// `rocm remote doctor`, or a caller that built the struct by hand. + /// + /// Public because the `rocm` binary classifies its own support level from a + /// summary it already gathered, and must read the probe result the same way + /// this module does rather than reaching into the field and inventing its + /// own `None` handling. + #[must_use] + pub fn vulkan_status(&self) -> VulkanStatus { + self.vulkan + .as_ref() + .map_or(VulkanStatus::Unknown, |vulkan| vulkan.status) + } + + /// The `vulkan` line: the status, and the evidence behind it. + fn vulkan_line(&self) -> String { + let Some(vulkan) = self.vulkan.as_ref() else { + return "".to_owned(); + }; + let status = match vulkan.status { + VulkanStatus::Available => "available", + VulkanStatus::DriverPresentDevicesUnverified => "driver_present_devices_unverified", + VulkanStatus::Absent => "absent", + VulkanStatus::Unknown => "unknown", + }; + vulkan.detail.as_deref().map_or_else( + || status.to_owned(), + |detail| format!("{status} ({detail})"), + ) + } + /// The `rocm_support` line of the examine text. /// /// ROCMAI-550 asks `rocm examine` to "report the support level with one @@ -34,14 +67,49 @@ impl ExamineSummary { /// blank line. fn rocm_support_line(&self) -> String { let detected = self.detected_gfx_target.as_deref(); - match gpu_support_level(detected, self.compatible_therock_family.as_deref()) { + let vulkan = self.vulkan_status(); + match gpu_support_level(detected, self.compatible_therock_family.as_deref(), vulkan) { // `VulkanOnly` is only ever returned for a detected target, so the // fallback below is unreachable; it keeps the arm total without a // panic path. + // + // The claim is graded by what the probe actually established. An + // enumerated device is named, because naming it is what lets the + // reader check the claim rather than take it; an installed-but- + // unenumerated driver says so instead of borrowing the stronger + // sentence; and a platform that could not be probed admits it. All + // three still point at the same command, because all three are + // hosts worth trying. GpuSupportLevel::VulkanOnly => format!( - " rocm_support: GPU detected ({}) but not supported by ROCm; \ - the Vulkan backend can still be used - serve on it with \ - `rocm serve --engine lemonade`\n", + " rocm_support: GPU detected ({}) but not supported by ROCm; {} - serve on it \ + with `rocm serve --engine lemonade`\n", + detected.unwrap_or_default(), + match vulkan { + VulkanStatus::Available => self.vulkan.as_ref().map_or_else( + || "the Vulkan backend can be used".to_owned(), + |support| support.driver_label().map_or_else( + || "the Vulkan backend can be used".to_owned(), + |driver| format!("the Vulkan backend can be used ({driver})"), + ), + ), + VulkanStatus::DriverPresentDevicesUnverified => + "a Vulkan driver is installed, so the Vulkan backend is expected to work" + .to_owned(), + VulkanStatus::Absent | VulkanStatus::Unknown => + "Vulkan was not probed on this platform, so the Vulkan backend is \ + unconfirmed" + .to_owned(), + } + ), + // Both halves are established facts: the catalog confirmed there is + // no ROCm build, and the probe looked for a Vulkan driver and found + // none. Unlike `Unknown` below, there is nothing left to go and find + // out on this host, so the advice is to install a driver rather than + // to serve a model and read back what happened. + GpuSupportLevel::NoGpuBackend => format!( + " rocm_support: GPU detected ({}) but not supported by ROCm, and no Vulkan GPU \ + driver was found - install one (Mesa RADV on Linux, the AMD display driver on \ + Windows) to run models on this GPU\n", detected.unwrap_or_default() ), // The covering family is the fact; it is already printed on its own @@ -137,7 +205,7 @@ impl ExamineSummary { "false (this run's output is captured, so the CLI will not prompt)" }; format!( - "rocm examine\n os: {}\n arch: {}\n kernel: {}\n distro: {}\n cpu: {}\n system_ram: {}\n interactive_terminal: {}\n default_engine: {}\n detected_gfx_target: {}\n compatible_therock_family: {}\n{} detected_therock_family: {}\n driver_policy: {}\n driver_status: {}\n driver_detail: {}\n{} legacy_rocm_status: {}\n legacy_rocm_paths: {}\n legacy_rocm_version: {}\n legacy_rocm_detail: {}\n legacy_rocm_guidance: {}\n wsl: {}\n wsl_dxg_device: {}\n wsl_dxcore: {}\n wsl_librocdxg: {}\n wsl_rocdxg_dids: {}\n wsl_ldconfig_librocdxg: {}\n wsl_librocdxg_on_ld_library_path: {}\n wsl_librocdxg_beside_runtime: {}\n wsl_global_rocminfo: {}\n wsl_cargo: {}\n wsl_detail: {}\n managed_runtimes: {}\n managed_services: {}\n model_cache_entries: {}\n config_dir: {}\n data_dir: {}\n cache_dir: {}\n", + "rocm examine\n os: {}\n arch: {}\n kernel: {}\n distro: {}\n cpu: {}\n system_ram: {}\n interactive_terminal: {}\n default_engine: {}\n detected_gfx_target: {}\n compatible_therock_family: {}\n{} detected_therock_family: {}\n vulkan: {}\n driver_policy: {}\n driver_status: {}\n driver_detail: {}\n{} legacy_rocm_status: {}\n legacy_rocm_paths: {}\n legacy_rocm_version: {}\n legacy_rocm_detail: {}\n legacy_rocm_guidance: {}\n wsl: {}\n wsl_dxg_device: {}\n wsl_dxcore: {}\n wsl_librocdxg: {}\n wsl_rocdxg_dids: {}\n wsl_ldconfig_librocdxg: {}\n wsl_librocdxg_on_ld_library_path: {}\n wsl_librocdxg_beside_runtime: {}\n wsl_global_rocminfo: {}\n wsl_cargo: {}\n wsl_detail: {}\n managed_runtimes: {}\n managed_services: {}\n model_cache_entries: {}\n config_dir: {}\n data_dir: {}\n cache_dir: {}\n", self.os, self.arch, self.kernel.as_deref().unwrap_or(""), @@ -155,6 +223,7 @@ impl ExamineSummary { self.detected_therock_family .as_deref() .unwrap_or(""), + self.vulkan_line(), self.driver.policy, self.driver.status, self.driver.detail.as_deref().unwrap_or(""), @@ -217,8 +286,8 @@ mod tests { use super::*; use crate::LegacyRocmSummary; use crate::gfx_target::normalize_therock_family; - use crate::host_gpu::DriverSummary; use crate::host_gpu::tests::windows_summary_with_driver; + use crate::host_gpu::{DriverSummary, VulkanDevice, VulkanSupport}; use std::path::PathBuf; #[test] @@ -235,6 +304,7 @@ mod tests { detected_gfx_target: None, compatible_therock_family: Some("gfx120X-all".to_owned()), detected_therock_family: None, + vulkan: None, driver: DriverSummary { policy: "windows_validate_only".to_owned(), status: "amd_display_driver_detected".to_owned(), @@ -314,6 +384,7 @@ mod tests { detected_gfx_target: detected_gfx_target.map(str::to_owned), compatible_therock_family: compatible_therock_family.map(str::to_owned), detected_therock_family: None, + vulkan: None, driver: DriverSummary { policy: "windows_validate_only".to_owned(), status: "amd_display_driver_detected".to_owned(), @@ -336,12 +407,41 @@ mod tests { } } + /// A gfx90c summary carrying the given Vulkan evidence. + /// + /// gfx90c is detected from its PCI ID but has no ROCm package family, so it + /// is the target that reaches the Vulkan-graded arms at all. + fn gfx90c_summary_with_vulkan(vulkan: Option) -> ExamineSummary { + let family = normalize_therock_family("gfx90c"); + let mut summary = examine_summary_with_gpu(Some("gfx90c"), family.as_deref()); + summary.vulkan = vulkan; + summary + } + + fn vulkan_support(status: VulkanStatus, detail: &str) -> Option { + Some(VulkanSupport { + status, + detail: Some(detail.to_owned()), + devices: vec![VulkanDevice { + name: "AMD Radeon Graphics (RADV RENOIR)".to_owned(), + driver_name: Some("radv".to_owned()), + vendor_id: Some("0x1002".to_owned()), + device_type: Some("PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU".to_owned()), + ..VulkanDevice::default() + }], + ..VulkanSupport::default() + }) + } + #[test] fn examine_render_says_a_gpu_without_a_rocm_family_is_unsupported() { - // gfx90c is detected from its PCI ID but has no ROCm package family. - let family = normalize_therock_family("gfx90c"); - let summary = examine_summary_with_gpu(Some("gfx90c"), family.as_deref()); + let summary = gfx90c_summary_with_vulkan(vulkan_support( + VulkanStatus::Available, + "AMD Radeon Graphics (RADV RENOIR) through radv", + )); + let rendered = summary.render_text(); + assert!( rendered.contains("detected_gfx_target: gfx90c"), "{rendered}" @@ -351,9 +451,92 @@ mod tests { "{rendered}" ); assert!( - rendered.contains("the Vulkan backend can still be used"), + rendered.contains("the Vulkan backend can be used (radv)"), + "a confirmed device must be named, so the claim can be checked: {rendered}" + ); + assert!( + rendered.contains("vulkan: available"), + "the evidence line must carry the verdict too: {rendered}" + ); + } + + /// The defect this guards against: the `VulkanOnly` line used to promise + /// "the Vulkan backend can still be used" on every gfx90c host, derived from + /// the gfx table alone. A Renoir laptop with no Vulkan driver installed was + /// told to go and serve a model on a backend that cannot start — and under + /// the GPU-required policy there is nothing for it to fall back to. + #[test] + fn examine_render_does_not_promise_vulkan_on_a_host_that_has_none() { + let summary = gfx90c_summary_with_vulkan(Some(VulkanSupport { + status: VulkanStatus::Absent, + detail: Some("no Vulkan driver found".to_owned()), + ..VulkanSupport::default() + })); + + let rendered = summary.render_text(); + + assert!( + !rendered.contains("the Vulkan backend can be used"), + "a host with no Vulkan driver must not be promised one: {rendered}" + ); + assert!( + rendered.contains("no Vulkan GPU driver was found"), "{rendered}" ); + assert!( + rendered.contains("install one"), + "the advice must be able to clear the condition it is printed for: {rendered}" + ); + assert!(rendered.contains("vulkan: absent"), "{rendered}"); + } + + /// An installed driver that enumerated no device is a weaker claim than a + /// device that answered, and says so rather than borrowing the stronger + /// sentence. + #[test] + fn examine_render_separates_an_installed_driver_from_a_confirmed_device() { + let summary = gfx90c_summary_with_vulkan(Some(VulkanSupport { + status: VulkanStatus::DriverPresentDevicesUnverified, + detail: Some("Mesa RADV is installed, but no device was enumerated".to_owned()), + ..VulkanSupport::default() + })); + + let rendered = summary.render_text(); + + assert!( + rendered.contains("a Vulkan driver is installed, so the Vulkan backend is expected"), + "{rendered}" + ); + assert!( + rendered.contains("vulkan: driver_present_devices_unverified"), + "{rendered}" + ); + } + + /// A summary with no probe result at all — an older host's JSON read back + /// through `rocm remote doctor` — must not read as "no Vulkan". It keeps + /// the Vulkan advice, because withholding the only path such a host has on + /// the strength of a probe that never ran would be worse than the + /// assumption this change replaces. + #[test] + fn examine_render_treats_an_unprobed_summary_as_unconfirmed_not_absent() { + let summary = gfx90c_summary_with_vulkan(None); + + let rendered = summary.render_text(); + + assert!(rendered.contains("vulkan: "), "{rendered}"); + assert!( + rendered.contains("the Vulkan backend is unconfirmed"), + "{rendered}" + ); + assert!( + !rendered.contains("no Vulkan GPU driver was found"), + "an unprobed host is not a host without Vulkan: {rendered}" + ); + assert!( + rendered.contains("rocm serve --engine lemonade"), + "it is still worth trying, so the command stays: {rendered}" + ); } /// An unmapped target must not be *claimed* unsupported, even though every @@ -461,6 +644,7 @@ mod tests { detected_gfx_target: None, compatible_therock_family: None, detected_therock_family: None, + vulkan: None, driver: DriverSummary { policy: "linux_official_amd_dkms_wrapper".to_owned(), status: "amdgpu_available".to_owned(), @@ -515,6 +699,7 @@ mod tests { detected_gfx_target: None, compatible_therock_family: None, detected_therock_family: None, + vulkan: None, driver: DriverSummary { policy: "linux_official_amd_dkms_wrapper".to_owned(), status: "amdgpu_available".to_owned(), @@ -610,7 +795,7 @@ mod tests { .find(|l| l.trim_start().starts_with("rocm_support:")) .expect("every host gets a support line"); assert_eq!( - gpu_support_level(None, None), + gpu_support_level(None, None, VulkanStatus::Unknown), GpuSupportLevel::Unknown, "precondition: nothing detected is the Unknown tier" ); diff --git a/crates/rocm-core/src/host_gpu/vulkan.rs b/crates/rocm-core/src/host_gpu/vulkan.rs new file mode 100644 index 0000000000..b1e2b08913 --- /dev/null +++ b/crates/rocm-core/src/host_gpu/vulkan.rs @@ -0,0 +1,726 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Whether this host can run a model through Vulkan. +//! +//! The CLI has always *asserted* this for a GPU with no ROCm build -- `examine` +//! told a gfx90c user "the Vulkan backend can still be used" without ever +//! looking. This probe is what makes that sentence evidence rather than a +//! promise, and it is what lets a host outside the ROCm compatibility matrix be +//! described by what it can do instead of only by what it cannot. +//! +//! Two sources, in increasing order of strength: +//! +//! 1. An **installable driver**: an ICD manifest on Linux (`/usr/share/vulkan/ +//! icd.d` and friends), or the Khronos registry key on Windows. This proves a +//! driver is *installed*, not that a device enumerates. +//! 2. An **enumerated device**: `vulkaninfo --summary`, when it is on PATH. +//! This proves a device actually answers. +//! +//! The two are kept apart by [`VulkanStatus`] rather than collapsed, because +//! collapsing them would replace one unverified promise with another. +//! +//! **A software rasteriser is not a GPU.** Mesa's lavapipe (`lvp_icd.*.json`, +//! `PHYSICAL_DEVICE_TYPE_CPU`) presents a complete Vulkan implementation that +//! runs entirely on the CPU. Counting it here would route a user onto CPU +//! execution while reporting a GPU path -- precisely the silent CPU fallback +//! `AGENTS.md` §6 forbids -- so it is recognised in order to be excluded, and +//! named in `detail` so the exclusion is visible rather than mysterious. + +use super::capture::capture_optional_command; +#[cfg(target_os = "linux")] +use crate::host_path; +use serde::{Deserialize, Serialize}; + +/// The directories a Linux Vulkan loader reads ICD manifests from. +/// +/// Read through [`host_path`] so a simulated host root can plant them, which is +/// what lets the no-GPU CI lane cover this probe at all. +#[cfg(target_os = "linux")] +const LINUX_ICD_DIRS: [&str; 4] = [ + "/usr/share/vulkan/icd.d", + "/usr/local/share/vulkan/icd.d", + "/etc/vulkan/icd.d", + "/usr/lib/x86_64-linux-gnu/vulkan/icd.d", +]; + +/// How sure the CLI is that a model can run through Vulkan here. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum VulkanStatus { + /// A Vulkan device answered and it is a real GPU. + Available, + /// A driver is installed, but no device was enumerated -- `vulkaninfo` is + /// not on PATH, so nothing confirmed a device answers. Likely to work; + /// not established. + DriverPresentDevicesUnverified, + /// The probe looked and found no GPU Vulkan driver. A software rasteriser + /// alone lands here, not in [`Self::Available`]. + Absent, + /// The probe could not look (an unsupported platform). + /// + /// The default: a value nobody has filled in is one nobody has probed, and + /// defaulting to `Absent` would let an unset field read as a fact about the + /// machine. + #[default] + Unknown, +} + +impl VulkanStatus { + /// Whether a model may be routed through Vulkan on the strength of this. + #[must_use] + pub const fn is_usable(self) -> bool { + matches!(self, Self::Available | Self::DriverPresentDevicesUnverified) + } +} + +/// One physical device as `vulkaninfo --summary` reported it. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct VulkanDevice { + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub driver_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub driver_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub api_version: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vendor_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub device_id: Option, + /// The raw `deviceType`, e.g. `PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub device_type: Option, +} + +impl VulkanDevice { + /// AMD's PCI vendor id, as `vulkaninfo` prints it. + const AMD_VENDOR_ID: &'static str = "0x1002"; + + #[must_use] + pub fn is_amd(&self) -> bool { + self.vendor_id + .as_deref() + .is_some_and(|id| id.eq_ignore_ascii_case(Self::AMD_VENDOR_ID)) + } + + /// Whether this device is real hardware rather than a software rasteriser. + /// + /// A missing `deviceType` reads as hardware: the field is optional in the + /// summary output, and treating an unreported type as CPU would discard a + /// working GPU. Lavapipe always reports its type, so the only entries this + /// lets through unverified are ones that named no type at all. + #[must_use] + pub fn is_hardware(&self) -> bool { + !self + .device_type + .as_deref() + .is_some_and(|kind| kind.to_ascii_uppercase().contains("TYPE_CPU")) + } +} + +/// What the host offers for Vulkan, and the evidence behind it. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct VulkanSupport { + pub status: VulkanStatus, + /// The ICD manifests or registry entries found, as the loader names them. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub icds: Vec, + /// The devices `vulkaninfo` enumerated, when it was available. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub devices: Vec, + /// A one-line human summary, including why anything was excluded. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +impl VulkanSupport { + /// The driver this host would use, for the `examine` line. + /// + /// Prefers what a device actually reported (`AMD proprietary driver`, + /// `radv`) over what an ICD filename implies, because the filename is a + /// guess about a vendor and the device's own `driverName` is not. + #[must_use] + pub fn driver_label(&self) -> Option { + self.devices + .iter() + .find(|device| device.is_amd() && device.is_hardware()) + .and_then(|device| device.driver_name.clone()) + .or_else(|| { + self.icds + .iter() + .find_map(|icd| icd_driver_label(icd)) + .map(str::to_owned) + }) + } +} + +/// The driver an ICD manifest's file name implies, or `None` when the name is +/// not one this table knows. +/// +/// Deliberately includes the software rasterisers so a caller can *exclude* +/// them by name; [`icd_is_gpu_driver`] is the predicate that does so. +#[must_use] +pub fn icd_driver_label(icd: &str) -> Option<&'static str> { + let name = icd + .rsplit(['/', '\\']) + .next() + .unwrap_or(icd) + .to_ascii_lowercase(); + if name.starts_with("radeon_icd") { + return Some("Mesa RADV"); + } + if name.starts_with("amd_icd") || name.starts_with("amdvlk") { + return Some("AMDVLK"); + } + if name.starts_with("lvp_icd") { + return Some("Mesa lavapipe (software)"); + } + if name.starts_with("vk_swiftshader") { + return Some("SwiftShader (software)"); + } + if name.starts_with("nvidia_icd") { + return Some("NVIDIA"); + } + if name.starts_with("intel_icd") || name.starts_with("intel_hasvk") { + return Some("Intel"); + } + None +} + +/// Whether an ICD manifest names a driver that runs on a GPU. +/// +/// An unrecognised manifest counts as a GPU driver: a vendor the table does not +/// list yet is far more likely to be real hardware than a new software +/// rasteriser, and the cost of the two errors is not symmetric -- missing a +/// working GPU path sends the user to CPU, while admitting an unknown one is +/// corrected by the device enumeration that follows. +#[must_use] +pub fn icd_is_gpu_driver(icd: &str) -> bool { + !icd_driver_label(icd).is_some_and(|label| label.contains("(software)")) +} + +/// Parse `vulkaninfo --summary` output into its physical devices. +/// +/// Tolerates the loader warnings `vulkaninfo` prints to stdout before the +/// report, and the `Instance Extensions` / `Instance Layers` sections ahead of +/// `Devices:` -- only `GPUn:` blocks and their indented `key = value` lines are +/// read. +#[must_use] +pub fn parse_vulkaninfo_summary(output: &str) -> Vec { + let mut devices = Vec::new(); + let mut current: Option = None; + for line in output.lines() { + let trimmed = line.trim(); + if trimmed.starts_with("GPU") && trimmed.ends_with(':') { + if let Some(device) = current.take() { + devices.push(device); + } + current = Some(VulkanDevice::default()); + continue; + } + let Some(device) = current.as_mut() else { + continue; + }; + let Some((key, value)) = trimmed.split_once('=') else { + // A non-indented, non-`key = value` line ends the block: the + // summary's device section is the last thing printed, but a caller + // may hand us trailing output. + if !trimmed.is_empty() && !line.starts_with([' ', '\t']) { + devices.push(current.take().unwrap_or_default()); + } + continue; + }; + let value = value.trim().to_owned(); + if value.is_empty() { + continue; + } + match key.trim() { + "deviceName" => device.name = value, + "driverName" => device.driver_name = Some(value), + "driverID" => device.driver_id = Some(value), + "apiVersion" => device.api_version = Some(value), + "vendorID" => device.vendor_id = Some(value), + "deviceID" => device.device_id = Some(value), + "deviceType" => device.device_type = Some(value), + _ => {} + } + } + if let Some(device) = current.take() { + devices.push(device); + } + devices.retain(|device| !device.name.is_empty()); + devices +} + +/// The ICD manifest paths listed under the Windows Khronos registry key, as +/// [`WINDOWS_VULKAN_ICD_SCRIPT`] prints them (one `ICD\t` row each). +#[must_use] +pub fn parse_windows_vulkan_icd_listing(output: &str) -> Vec { + output + .lines() + .filter_map(|line| line.trim().strip_prefix("ICD\t")) + .map(str::trim) + .filter(|path| !path.is_empty()) + .map(str::to_owned) + .collect() +} + +/// Enumerate both Khronos ICD keys. The WOW6432Node key carries the 32-bit +/// drivers; a 64-bit-only install lists nothing there, which is not an error. +#[cfg(any(windows, test))] +pub(super) const WINDOWS_VULKAN_ICD_SCRIPT: &str = r#"foreach ($key in 'HKLM:\SOFTWARE\Khronos\Vulkan\Drivers','HKLM:\SOFTWARE\WOW6432Node\Khronos\Vulkan\Drivers') { if (Test-Path $key) { foreach ($name in (Get-Item $key).GetValueNames()) { "ICD`t$name" } } }"#; + +/// Every Vulkan ICD manifest this host's loader would read. +#[cfg(target_os = "linux")] +fn detect_icds() -> Vec { + let mut icds = Vec::new(); + for dir in LINUX_ICD_DIRS { + let Ok(entries) = std::fs::read_dir(host_path(dir)) else { + continue; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.extension().is_some_and(|ext| ext == "json") + && let Some(name) = path.file_name().and_then(|name| name.to_str()) + { + icds.push(name.to_owned()); + } + } + } + icds.sort(); + icds.dedup(); + icds +} + +#[cfg(windows)] +fn detect_icds() -> Vec { + capture_optional_command( + "powershell", + &[ + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-Command", + WINDOWS_VULKAN_ICD_SCRIPT, + ], + ) + .map(|output| parse_windows_vulkan_icd_listing(&output)) + .unwrap_or_default() +} + +#[cfg(not(any(target_os = "linux", windows)))] +const fn detect_icds() -> Vec { + Vec::new() +} + +/// The devices `vulkaninfo --summary` reports, or an empty list when the tool +/// is not installed. Absence of the tool is not absence of a driver, which is +/// why this cannot decide [`VulkanStatus`] on its own. +fn detect_vulkan_devices() -> Vec { + capture_optional_command("vulkaninfo", &["--summary"]) + .map(|output| parse_vulkaninfo_summary(&output)) + .unwrap_or_default() +} + +/// Whether this platform's Vulkan surfaces can be read at all. +const fn vulkan_is_probeable() -> bool { + cfg!(any(target_os = "linux", windows)) +} + +/// What this host offers for Vulkan. +#[must_use] +pub fn detect_vulkan_support() -> VulkanSupport { + classify_vulkan_support( + detect_icds(), + detect_vulkan_devices(), + vulkan_is_probeable(), + ) +} + +/// The pure decision behind [`detect_vulkan_support`], split out so every +/// combination of evidence can be tested without a Vulkan install. +#[must_use] +pub fn classify_vulkan_support( + icds: Vec, + devices: Vec, + probeable: bool, +) -> VulkanSupport { + if !probeable { + return VulkanSupport { + status: VulkanStatus::Unknown, + icds, + devices, + detail: Some("Vulkan support is not probed on this platform".to_owned()), + }; + } + + let amd_gpu = devices + .iter() + .find(|device| device.is_amd() && device.is_hardware()); + if let Some(device) = amd_gpu { + let driver = device.driver_name.as_deref().unwrap_or("an unnamed driver"); + return VulkanSupport { + detail: Some(format!("{} through {driver}", device.name)), + status: VulkanStatus::Available, + icds, + devices, + }; + } + + let software_only = !devices.is_empty() && devices.iter().all(|device| !device.is_hardware()); + if software_only { + return VulkanSupport { + status: VulkanStatus::Absent, + detail: Some( + "only a software (CPU) Vulkan device was found; that is not a GPU path".to_owned(), + ), + icds, + devices, + }; + } + + let gpu_icds: Vec<&String> = icds.iter().filter(|icd| icd_is_gpu_driver(icd)).collect(); + if gpu_icds.is_empty() { + // No ICD *and* no enumerated device is an absence of evidence, not + // evidence of absence: neither source is exhaustive. The Linux ICD + // search is a fixed directory list that ignores `VK_ICD_FILENAMES` and + // `VK_DRIVER_FILES`; the Windows one reads only the legacy Khronos + // registry key, which current AMD drivers need not populate; and + // `vulkaninfo` is frequently not installed on either. A working driver + // can therefore leave both empty, and `Absent` is rendered to the user + // as a confirmed fact ("no Vulkan GPU driver was found - install one"). + // Say we could not tell instead, and keep `Absent` for the cases below + // where something was actually found and ruled out. + if icds.is_empty() && devices.is_empty() { + return VulkanSupport { + status: VulkanStatus::Unknown, + detail: Some( + "no Vulkan driver was found, but this host exposes nothing that confirms one is absent either (no ICD manifest, and `vulkaninfo` did not enumerate)" + .to_owned(), + ), + icds, + devices, + }; + } + return VulkanSupport { + status: VulkanStatus::Absent, + detail: Some( + "only a software (CPU) Vulkan driver is installed; that is not a GPU path" + .to_owned(), + ), + icds, + devices, + }; + } + + let named = gpu_icds + .iter() + .find_map(|icd| icd_driver_label(icd)) + .map_or_else( + || "a Vulkan driver is installed".to_owned(), + |label| format!("{label} is installed"), + ); + VulkanSupport { + status: VulkanStatus::DriverPresentDevicesUnverified, + detail: Some(format!( + "{named}, but no device was enumerated (install `vulkaninfo` to confirm)" + )), + icds, + devices, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The real `vulkaninfo --summary` output from a Ryzen 7 4700U (gfx90c, + /// Renoir) on Windows 11 with Adrenalin 31.0.21925.1001 -- the machine + /// ROCMAI-517 is about. Note the loader warnings ahead of the report and + /// the instance sections before `Devices:`; both must be skipped. + const RENOIR_WINDOWS_SUMMARY: &str = "\ +WARNING: [Loader Message] Code 0 : windows_read_data_files_in_registry: Registry lookup failed to get layer manifest files. +========== +VULKANINFO +========== + +Vulkan Instance Version: 1.3.301 + + +Instance Extensions: count = 13 +------------------------------- +VK_EXT_debug_report : extension revision 10 +VK_KHR_surface : extension revision 25 + +Instance Layers: count = 1 +-------------------------- +VK_LAYER_AMD_switchable_graphics AMD switchable graphics layer 1.3.260 version 1 + +Devices: +======== +GPU0: +\tapiVersion = 1.3.260 +\tdriverVersion = 2.0.279 +\tvendorID = 0x1002 +\tdeviceID = 0x1636 +\tdeviceType = PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU +\tdeviceName = AMD Radeon(TM) Graphics +\tdriverID = DRIVER_ID_AMD_PROPRIETARY +\tdriverName = AMD proprietary driver +\tdriverInfo = 26.5.2 (AMD proprietary shader compiler) +\tconformanceVersion = 1.3.3.1 +"; + + #[test] + fn the_renoir_igpu_is_read_out_of_a_real_vulkaninfo_summary() { + let devices = parse_vulkaninfo_summary(RENOIR_WINDOWS_SUMMARY); + + assert_eq!(devices.len(), 1, "{devices:?}"); + let gpu = &devices[0]; + assert_eq!(gpu.name, "AMD Radeon(TM) Graphics"); + assert_eq!(gpu.vendor_id.as_deref(), Some("0x1002")); + // The same device id the gfx-target table maps to gfx90c. + assert_eq!(gpu.device_id.as_deref(), Some("0x1636")); + assert_eq!(gpu.driver_name.as_deref(), Some("AMD proprietary driver")); + assert_eq!(gpu.driver_id.as_deref(), Some("DRIVER_ID_AMD_PROPRIETARY")); + assert_eq!(gpu.api_version.as_deref(), Some("1.3.260")); + assert!(gpu.is_amd(), "vendor 0x1002 is AMD"); + assert!(gpu.is_hardware(), "an integrated GPU is hardware"); + } + + #[test] + fn a_real_renoir_host_reports_vulkan_available_and_names_its_driver() { + let devices = parse_vulkaninfo_summary(RENOIR_WINDOWS_SUMMARY); + + let support = classify_vulkan_support(Vec::new(), devices, true); + + assert_eq!(support.status, VulkanStatus::Available); + assert!(support.status.is_usable()); + assert_eq!( + support.driver_label().as_deref(), + Some("AMD proprietary driver"), + "the device's own driverName must win over any ICD filename guess" + ); + } + + /// The defect this guards against: lavapipe is a complete Vulkan + /// implementation that runs on the CPU. Reporting it as a usable Vulkan + /// path would route a GPU-required launch onto CPU execution while telling + /// the user a GPU path was chosen -- the silent CPU fallback AGENTS.md §6 + /// forbids, reached through a backend rather than a policy bug. + #[test] + fn a_software_rasteriser_is_never_a_usable_vulkan_path() { + let lavapipe = "\ +GPU0: +\tapiVersion = 1.3.255 +\tvendorID = 0x10005 +\tdeviceType = PHYSICAL_DEVICE_TYPE_CPU +\tdeviceName = llvmpipe (LLVM 15.0.7, 256 bits) +\tdriverName = llvmpipe +"; + let devices = parse_vulkaninfo_summary(lavapipe); + assert_eq!(devices.len(), 1); + assert!(!devices[0].is_hardware()); + + let support = + classify_vulkan_support(vec!["lvp_icd.x86_64.json".to_owned()], devices, true); + + assert_eq!( + support.status, + VulkanStatus::Absent, + "a CPU rasteriser must not read as a Vulkan GPU path: {support:?}" + ); + assert!(!support.status.is_usable()); + assert!( + support + .detail + .as_deref() + .is_some_and(|detail| detail.contains("not a GPU path")), + "{support:?}" + ); + } + + #[test] + fn a_lavapipe_only_install_is_absent_even_before_any_device_is_enumerated() { + let support = + classify_vulkan_support(vec!["lvp_icd.x86_64.json".to_owned()], Vec::new(), true); + + assert_eq!(support.status, VulkanStatus::Absent); + assert!(!icd_is_gpu_driver("lvp_icd.x86_64.json")); + assert!(icd_is_gpu_driver("radeon_icd.x86_64.json")); + } + + #[test] + fn an_installed_driver_with_no_enumerated_device_stays_unverified() { + let support = + classify_vulkan_support(vec!["radeon_icd.x86_64.json".to_owned()], Vec::new(), true); + + assert_eq!(support.status, VulkanStatus::DriverPresentDevicesUnverified); + assert!( + support.status.is_usable(), + "an installed GPU driver is still worth routing to" + ); + assert_eq!(support.driver_label().as_deref(), Some("Mesa RADV")); + assert!( + support + .detail + .as_deref() + .is_some_and(|detail| detail.contains("no device was enumerated")), + "the gap must be stated, not papered over: {support:?}" + ); + } + + /// No ICD and no enumerated device is an absence of evidence, not evidence + /// of absence — neither source is exhaustive (the Linux ICD search ignores + /// `VK_ICD_FILENAMES`, the Windows one reads only the legacy registry key, + /// and `vulkaninfo` is often not installed). `Absent` renders as a + /// confirmed "no Vulkan GPU driver was found - install one", so a host with + /// a working driver would be told to install one it already has. + #[test] + fn no_evidence_either_way_is_not_reported_as_a_confirmed_absence() { + let nothing = classify_vulkan_support(Vec::new(), Vec::new(), true); + assert_eq!( + nothing.status, + VulkanStatus::Unknown, + "an empty probe must not claim the driver is absent" + ); + assert!( + nothing + .detail + .as_deref() + .is_some_and(|detail| detail.contains("confirms")), + "the detail must say the absence is unconfirmed: {nothing:?}" + ); + assert!(nothing.driver_label().is_none()); + + // What `Absent` is now reserved for: something WAS found and ruled out. + let software_only = + classify_vulkan_support(vec!["lvp_icd.x86_64.json".to_owned()], Vec::new(), true); + assert_eq!(software_only.status, VulkanStatus::Absent); + + // macOS and friends: no evidence either way, which must not read as + // "no Vulkan" any more than it reads as "Vulkan works". + let unprobeable = classify_vulkan_support(Vec::new(), Vec::new(), false); + assert_eq!(unprobeable.status, VulkanStatus::Unknown); + assert!(!unprobeable.status.is_usable()); + } + + #[test] + fn an_unknown_vendors_icd_is_assumed_to_be_hardware() { + // A vendor the table does not list yet must not be mistaken for a + // software rasteriser; the device enumeration corrects it if wrong. + assert!(icd_is_gpu_driver("some_new_vendor_icd.x86_64.json")); + assert_eq!(icd_driver_label("some_new_vendor_icd.x86_64.json"), None); + + let support = classify_vulkan_support( + vec!["some_new_vendor_icd.x86_64.json".to_owned()], + Vec::new(), + true, + ); + assert_eq!(support.status, VulkanStatus::DriverPresentDevicesUnverified); + } + + #[test] + fn a_discrete_amd_card_is_preferred_over_a_software_device_in_the_same_report() { + let mixed = "\ +GPU0: +\tvendorID = 0x10005 +\tdeviceType = PHYSICAL_DEVICE_TYPE_CPU +\tdeviceName = llvmpipe +\tdriverName = llvmpipe +GPU1: +\tvendorID = 0x1002 +\tdeviceType = PHYSICAL_DEVICE_TYPE_DISCRETE_GPU +\tdeviceName = AMD Radeon RX 7900 XTX +\tdriverName = radv +"; + let support = classify_vulkan_support(Vec::new(), parse_vulkaninfo_summary(mixed), true); + + assert_eq!(support.status, VulkanStatus::Available); + assert_eq!(support.driver_label().as_deref(), Some("radv")); + assert!( + support + .detail + .as_deref() + .is_some_and(|detail| detail.contains("AMD Radeon RX 7900 XTX")), + "{support:?}" + ); + } + + #[test] + fn a_non_amd_gpu_does_not_make_vulkan_an_amd_path() { + let nvidia = "\ +GPU0: +\tvendorID = 0x10de +\tdeviceType = PHYSICAL_DEVICE_TYPE_DISCRETE_GPU +\tdeviceName = NVIDIA GeForce GTX 1050 Ti +\tdriverName = NVIDIA +"; + let support = classify_vulkan_support( + vec!["nvidia_icd.json".to_owned()], + parse_vulkaninfo_summary(nvidia), + true, + ); + + // The driver is installed and is not software, so the honest answer is + // "unverified for AMD", not "available". + assert_eq!(support.status, VulkanStatus::DriverPresentDevicesUnverified); + } + + #[test] + fn the_windows_registry_listing_is_read_into_icd_paths() { + let output = "ICD\tC:\\Windows\\System32\\DriverStore\\FileRepository\\u0397342.inf_amd64_x\\amdvlk64.json\nICD\tC:\\Windows\\SysWOW64\\amdvlk32.json\n"; + + let icds = parse_windows_vulkan_icd_listing(output); + + assert_eq!(icds.len(), 2, "{icds:?}"); + assert!(icds[0].ends_with("amdvlk64.json")); + assert_eq!(icd_driver_label(&icds[0]), Some("AMDVLK")); + assert!( + icd_is_gpu_driver(&icds[0]), + "a Windows path separator must not defeat the filename table" + ); + } + + /// Both Khronos keys must be queried. A 32-bit AMDVLK registration lives + /// only under `WOW6432Node`, so reading the 64-bit key alone would report + /// "no Vulkan driver" on a host that has one. + #[test] + fn the_windows_icd_query_reads_both_the_native_and_the_wow6432_key() { + assert!( + WINDOWS_VULKAN_ICD_SCRIPT.contains(r"HKLM:\SOFTWARE\Khronos\Vulkan\Drivers"), + "{WINDOWS_VULKAN_ICD_SCRIPT}" + ); + assert!( + WINDOWS_VULKAN_ICD_SCRIPT + .contains(r"HKLM:\SOFTWARE\WOW6432Node\Khronos\Vulkan\Drivers"), + "{WINDOWS_VULKAN_ICD_SCRIPT}" + ); + // A missing key is ordinary on a host with no Vulkan at all, so the + // script must test for it rather than let PowerShell error out. + assert!( + WINDOWS_VULKAN_ICD_SCRIPT.contains("Test-Path"), + "{WINDOWS_VULKAN_ICD_SCRIPT}" + ); + // The rows the parser consumes. + assert!( + WINDOWS_VULKAN_ICD_SCRIPT.contains("\"ICD`t$name\""), + "{WINDOWS_VULKAN_ICD_SCRIPT}" + ); + } + + #[test] + fn an_empty_registry_key_yields_no_icds_rather_than_a_blank_entry() { + assert!(parse_windows_vulkan_icd_listing("").is_empty()); + assert!(parse_windows_vulkan_icd_listing("ICD\t\n").is_empty()); + assert!(parse_windows_vulkan_icd_listing("unrelated output\n").is_empty()); + } + + #[test] + fn a_vulkaninfo_report_with_no_devices_yields_none() { + assert!(parse_vulkaninfo_summary("Vulkan Instance Version: 1.3.301\n").is_empty()); + assert!(parse_vulkaninfo_summary("").is_empty()); + } +} diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index c5ea6662a8..aa33884fee 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -139,12 +139,12 @@ pub use gfx_target::{ }; pub use hardware_root::host_path; pub use host_gpu::{ - DriverSummary, ExamineSummary, HostGpuSummary, WSL_ROCDXG_READY, WindowsGpuListing, - WslHostDriverProbe, WslSummary, default_engine_for_host, default_engine_for_platform, - detect_gpu_driver_version, detect_host_gfx_target, detect_host_gpu_diagnostics, - detect_host_gpu_summary, detect_system_ram_gib, has_usable_amd_gpu, is_amdgpu_device, - is_wsl_host, pinned_gpu_name, preferred_serve_engine_for_host_gpu_summary, - usable_amd_gpu_indices, windows_amd_gpu_listing, + DriverSummary, ExamineSummary, HostGpuSummary, VulkanDevice, VulkanStatus, VulkanSupport, + WSL_ROCDXG_READY, WindowsGpuListing, WslHostDriverProbe, WslSummary, default_engine_for_host, + default_engine_for_platform, detect_gpu_driver_version, detect_host_gfx_target, + detect_host_gpu_diagnostics, detect_host_gpu_summary, detect_system_ram_gib, + detect_vulkan_support, has_usable_amd_gpu, is_amdgpu_device, is_wsl_host, pinned_gpu_name, + preferred_serve_engine_for_host_gpu_summary, usable_amd_gpu_indices, windows_amd_gpu_listing, }; #[cfg(any(target_os = "linux", test))] use host_gpu::{KfdGpuNode, kfd_gpu_nodes_in}; diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 67511ad01d..5a8989e5a7 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -155,15 +155,6 @@ where } if let Some(key_file) = options.endpoint_key_file { command.env(ENDPOINT_API_KEY_FILE_ENV, key_file); - } else { - // Cleared, not merely left unset. The caller said this child gets no - // endpoint key, and an engine plugin is a third-party binary: a value - // this process happens to have inherited must not reach it just - // because nothing overwrote it. `rocm`'s plugin calls pass `None` - // here, so this is what keeps a 0600 key-file path off their - // environment. See - // `an_external_plugin_child_is_not_handed_the_endpoint_key_file`. - command.env_remove(ENDPOINT_API_KEY_FILE_ENV); } if options.request_progress { command.env(ENGINE_PROGRESS_STDERR_ENV, "1"); @@ -684,46 +675,6 @@ mod tests { assert!(added_env(&command).is_empty(), "{:?}", added_env(&command)); } - /// A child told it gets no endpoint key must have the variable CLEARED, - /// not merely left unset. - /// - /// `rocm` passes `endpoint_key_file: None` for every external engine - /// plugin — a third-party binary — and the path it would otherwise carry - /// points at a mode-0600 file holding a live endpoint credential. Merely - /// not setting it would still hand the value through whenever this process - /// inherited one. - /// - /// Asserts the distinction directly: `Command` records `env_remove` as an - /// explicit `(key, None)` entry, so a plain "is it absent" check would pass - /// even with the clearing deleted. - #[test] - fn an_external_plugin_child_is_not_handed_the_endpoint_key_file() { - let key_var = std::ffi::OsStr::new(ENDPOINT_API_KEY_FILE_ENV); - let command = engine_stdio_command( - Path::new("/an/engine"), - ["stdio"], - &EngineStdioOptions { - app_paths: None, - env_root: None, - endpoint_key_file: None, - request_progress: false, - forward_stderr: false, - note_failing_exit_after_answer: false, - }, - ); - - let entry = command - .get_envs() - .find(|(key, _)| *key == key_var) - .map(|(_, value)| value); - assert_eq!( - entry, - Some(None), - "the key-file variable must be explicitly CLEARED when no key file \ - is passed, not merely left unset" - ); - } - #[test] fn each_option_sets_exactly_its_own_variable() { let paths = AppPaths { diff --git a/docs/architecture.md b/docs/architecture.md index 7270fd1b87..e33121035f 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -203,8 +203,8 @@ Already-extracted subsystem modules, following the same full-domain-extraction p - [`gguf.rs`](../crates/rocm-core/src/gguf.rs) — reads the shape of a GGUF model (layers, KV heads, head width, parameter count, data offset) from its header over any `Read + Seek`, never touching the weights. - [`gguf_fit.rs`](../crates/rocm-core/src/gguf_fit.rs) — pure arithmetic over that header: weights + KV cache at a context size and KV type + compute buffer + runtime reserve against dedicated VRAM, the largest context that fits, and what would make a model that does not fit, fit. - [`gguf_source.rs`](../crates/rocm-core/src/gguf_source.rs) — where a header comes from: a local file, or a Hugging Face file, found by one repository-listing request (and a `HEAD` request for a file the listing does not size) and read by HTTP range requests (`HF_ENDPOINT` selects a mirror; redirects are followed by hand, up to five, and never from https to plain http). The only code in the estimate that reads a file or the network. -- [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) — (ROCMAI-85, Phase 7a of ROCMAI-27's sequencing) GPU/host hardware detection — see the module's own doc comment for what it owns. Its leaf concerns are child modules under `host_gpu/`, with `host_gpu.rs` kept as the module root: [`capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs) (running an optional tool under a timeout, trying each spelling [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs) gives for it), [`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) (kernel version, distro name, installed RAM, and the CPU model and vendor, which one reader returns together for both the summary and `Examination::probe`: Linux from `/proc/cpuinfo`, Windows from the CPU row of the system inventory), [`linux.rs`](../crates/rocm-core/src/host_gpu/linux.rs) (the sysfs, KFD and DRM probes), [`gpu_visibility.rs`](../crates/rocm-core/src/host_gpu/gpu_visibility.rs) (which AMD GPUs this process may use, from the KFD/DRM counts and the `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` masks, plus `windows_amd_gpu_listing`, the native-Windows adapter bound `rocm serve --gpu ` is checked against), [`windows_inventory.rs`](../crates/rocm-core/src/host_gpu/windows_inventory.rs) (the Windows display-adapter inventory and its parsers, including the AMD-adapter count behind `windows_amd_gpu_listing` and the system query whose CPU row the CPU reader in `os_facts.rs` takes), [`wsl.rs`](../crates/rocm-core/src/host_gpu/wsl.rs) (WSL detection, the librocdxg chain and the Windows host driver as seen from the guest), [`render.rs`](../crates/rocm-core/src/host_gpu/render.rs) (the `rocm examine` text of an `ExamineSummary`, as a second `impl` block), [`diagnostics.rs`](../crates/rocm-core/src/host_gpu/diagnostics.rs) (the GPU detection diagnostics text) and [`engine_default.rs`](../crates/rocm-core/src/host_gpu/engine_default.rs) (the default serve engine, and the vLLM preference for the TheRock families that serve through it). -- [`host-probing.md`](host-probing.md) — a design note, not a module: [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) and [`examine.rs`](../crates/rocm-core/src/examine.rs) are two separate host-probing stacks that read some of the same facts twice (the Windows GPU query, the marketing-name table and the subprocess runners). It proposes which of each pair should survive, how the two differ and the order in which they could converge, and records what each convergence decided; it changes no code itself. Three pairs are converged: the `PATH` lookup ([`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs)'s `which` is now the only copy) and the kernel release ([`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs)'s `detect_kernel_version` is now the only `uname -r` reader, which [`host_env.rs`](../crates/rocm-core/src/examine/host_env.rs)'s `probe_os` calls for `kernel_release`, and `rocm examine --json` runs it once: [`examine.rs`](../crates/rocm-core/src/examine.rs)'s `Examination::probe_with_summary` probes the host and hands the examination's kernel release to the summary), and the CPU model ([`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) has one reader per platform for the model and vendor, which `probe_cpu_linux` and `probe_cpu_windows` record and the summary reads). It is a snapshot, to be retired when the convergence tickets close. +- [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) — (ROCMAI-85, Phase 7a of ROCMAI-27's sequencing) GPU/host hardware detection — see the module's own doc comment for what it owns. Its leaf concerns are child modules under `host_gpu/`, with `host_gpu.rs` kept as the module root: [`capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs) (running an optional tool under a timeout, trying each spelling [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs) gives for it), [`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) (kernel version, distro name, installed RAM, and the CPU model and vendor, which one reader returns together for both the summary and `Examination::probe`: Linux from `/proc/cpuinfo`, Windows from the CPU row of the system inventory), [`linux.rs`](../crates/rocm-core/src/host_gpu/linux.rs) (the sysfs, KFD and DRM probes), [`gpu_visibility.rs`](../crates/rocm-core/src/host_gpu/gpu_visibility.rs) (which AMD GPUs this process may use, from the KFD/DRM counts and the `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` masks, plus `windows_amd_gpu_listing`, the native-Windows adapter bound `rocm serve --gpu ` is checked against), [`windows_inventory.rs`](../crates/rocm-core/src/host_gpu/windows_inventory.rs) (the Windows display-adapter inventory and its parsers, including the AMD-adapter count behind `windows_amd_gpu_listing` and the system query whose CPU row the CPU reader in `os_facts.rs` takes), [`wsl.rs`](../crates/rocm-core/src/host_gpu/wsl.rs) (WSL detection, the librocdxg chain and the Windows host driver as seen from the guest), [`render.rs`](../crates/rocm-core/src/host_gpu/render.rs) (the `rocm examine` text of an `ExamineSummary`, as a second `impl` block), [`vulkan.rs`](../crates/rocm-core/src/host_gpu/vulkan.rs) (whether a model can run through Vulkan here, read from ICD manifests, the Windows Khronos registry and `vulkaninfo --summary` — the evidence `gpu_support_level` in [`gfx_target.rs`](../crates/rocm-core/src/gfx_target.rs) needs to tell a GPU with no ROCm build that still has a Vulkan driver from one with no GPU path at all, and where a software rasteriser is recognised in order to be excluded), [`diagnostics.rs`](../crates/rocm-core/src/host_gpu/diagnostics.rs) (the GPU detection diagnostics text) and [`engine_default.rs`](../crates/rocm-core/src/host_gpu/engine_default.rs) (the default serve engine, and the vLLM preference for the TheRock families that serve through it). +- [`host-probing.md`](host-probing.md) — a design note, not a module: [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) and [`examine.rs`](../crates/rocm-core/src/examine.rs) are two separate host-probing stacks that read some of the same facts twice (the Windows GPU query, the marketing-name table and the subprocess runners). It proposes which of each pair should survive, how the two differ and the order in which they could converge, and records what each convergence decided; it changes no code itself. Three pairs are converged: the `PATH` lookup ([`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs)'s `which` is now the only copy) and the kernel release ([`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs)'s `detect_kernel_version` is now the only `uname -r` reader, which [`host_env.rs`](../crates/rocm-core/src/examine/host_env.rs)'s `probe_os` calls for `kernel_release`, and `rocm examine --json` runs it once: [`examine.rs`](../crates/rocm-core/src/examine.rs)'s `Examination::probe_with_summary` probes the host and hands the examination's kernel release to the summary), and the CPU model ([`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) has one reader per platform for the model and vendor, which `probe_cpu_linux` and `probe_cpu_windows` record and the summary reads). The first step of the subprocess runners is delivered too: [`process/bounded.rs`](../crates/rocm-core/src/process/bounded.rs)'s `run_bounded` is the executor behind `run`, `capture_optional_command*` and the Windows diagnostic capture. It is a snapshot, to be retired when the convergence tickets close. - [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs) — the JSONL store engine behind the automation stores: the sibling `.lock` file (`jsonl_lock_path`, re-exported at the crate root), single-line appends, torn-tail repair, size-based rotation (`JsonlRotation`) and the tail-first readers that span rotated generations. It is generic over the record type; [`automation.rs`](../crates/rocm-core/src/automation.rs) supplies the record types and each store's policy, and nothing in this module depends on it, in production or in its own tests: those drive the engine with a small record type of their own. The module is private; its engine items are `pub(crate)`. - [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) — (ROCMAI-85, Phase 7a) the TheRock managed-runtime-environment surface — see the module's own doc comment for what it owns. - [`model_catalog.rs`](../crates/rocm-core/src/model_catalog.rs) — (ROCMAI-152, Phase 7c) the curated model catalog's types, built-in catalog JSON loading, signed external recipe indexes, and artifact cache-marker bookkeeping — see the module's own doc comment for what it owns. diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index 11e49f5f4d..e64ae34929 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -752,3 +752,40 @@ Feature: GPU detection and system inspection Then the inspection states this host's ROCm support level And the support level agrees with the GPU the inspection detected And the support level leaves a command that fits this host + + # The three below close the gap this PR's own notes call unclosable: the + # `VulkanOnly` door needs a Renoir/Cezanne Vega APU, and no runner has one. + # It does not need the hardware, only the two signals the CLI reads for it -- + # a KFD node naming the target, and the host's Vulkan driver state -- both of + # which the harness can plant. So these run on the ungated mock lane, which is + # the lane that has neither. + # + # They exist because the support level used to be inferred: "no ROCm build" + # was taken to mean "Vulkan then", on a machine nobody had looked at. Each + # scenario pins one of the three answers that inference could not tell apart. + @id:examine-names-the-vulkan-driver-an-unsupported-gpu-can-use @requires-os:linux + Scenario: examine-50 - A GPU the official stack rejects is told which way it can still run models + Given a machine whose GPU has no official support but has a working Vulkan driver + When the user inspects that machine + Then the inspection reports the GPU as unsupported by the official stack + And the inspection reports that models can still run through Vulkan + And the inspection names the Vulkan driver it found + + @id:examine-does-not-promise-vulkan-without-a-driver @requires-os:linux + Scenario: examine-51 - A machine with no way to run models is not told it has one + Given a machine whose GPU has no official support and no Vulkan driver + When the user inspects that machine + Then the inspection reports the GPU as unsupported by the official stack + And the inspection does not claim models can run through Vulkan + And the inspection says no Vulkan driver was found + + # A software renderer is a complete Vulkan implementation that runs on the + # CPU. Counting it would put a GPU-required run on the processor while + # reporting a GPU path - the one outcome the no-fallback rule exists to + # prevent, reached through a backend rather than a policy change. + @id:examine-does-not-count-a-software-renderer-as-a-gpu-path @requires-os:linux + Scenario: examine-52 - A software renderer does not count as a way to run on the GPU + Given a machine whose GPU has no official support and only a software renderer + When the user inspects that machine + Then the inspection does not claim models can run through Vulkan + And the inspection says the only renderer it found runs on the CPU diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 8fbe54b3cf..a5bc2b38c3 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -1704,6 +1704,25 @@ async fn assert_no_amdgpu_fault_evidence(world: &mut E2eWorld) { } } +/// The human-readable twin of the machine-readable planted-machine step, for +/// scenarios whose subject is a sentence the report prints rather than a field +/// it serialises. +/// +/// It has to be its own step rather than reusing "the user inspects the +/// system", because that one runs through `run_rocm`, which does not consume +/// `World::command_env` -- so a planted host root never reaches the child and +/// the CLI silently describes the *runner* instead. Not hypothetical: it is +/// what made the first draft of examine-50/51/52 report the lane's own MI300X +/// as `gfx942` while claiming to be about a Renoir APU. +#[when("the user inspects that machine")] +async fn user_inspects_planted_machine_text(world: &mut E2eWorld) { + let (stdout, stderr, rc) = + crate::run_rocm_with_scenario_env(world, &["examine", "--framework", "skip"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + #[when("the user inspects that machine in machine-readable form")] async fn user_inspects_planted_machine(world: &mut E2eWorld) { let (stdout, stderr, rc) = @@ -3347,3 +3366,208 @@ async fn assert_support_level_advice_fits(world: &mut E2eWorld) { ); } } + +/// A KFD `gfx_target_version` naming gfx90c, the Vega iGPU in Renoir / +/// Lucienne / Cezanne / Barcelo APUs. +/// +/// The kernel packs the target as `major * 10000 + minor * 100 + step`, so +/// `9.0.12` is `90012` and decodes to `gfx90c` (step 12 is the hex `c`). This +/// is the one target the CLI recognizes *and* refuses to map to a ROCm package +/// family, so it is what reaches the support level these scenarios are about. +const KFD_GFX90C_TARGET_VERSION: &str = "90012"; + +/// `vulkaninfo --summary` on a host whose Vega iGPU is driven by Mesa RADV. +const RADV_VULKANINFO_SUMMARY: &str = "\ +Devices: +======== +GPU0: +\tapiVersion = 1.3.274 +\tvendorID = 0x1002 +\tdeviceID = 0x1636 +\tdeviceType = PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU +\tdeviceName = AMD Radeon Graphics (RADV RENOIR) +\tdriverID = DRIVER_ID_MESA_RADV +\tdriverName = radv +"; + +/// `vulkaninfo --summary` on a host with only Mesa's CPU rasteriser: a complete +/// Vulkan implementation, and not a GPU. +const LAVAPIPE_VULKANINFO_SUMMARY: &str = "\ +Devices: +======== +GPU0: +\tapiVersion = 1.3.255 +\tvendorID = 0x10005 +\tdeviceType = PHYSICAL_DEVICE_TYPE_CPU +\tdeviceName = llvmpipe (LLVM 15.0.7, 256 bits) +\tdriverName = llvmpipe +"; + +/// Plant a host whose GPU is identified but has no ROCm package family, +/// carrying the Vulkan state the caller describes. +/// +/// `icd_manifest` is the ICD file name the loader would find and +/// `vulkaninfo_summary` what `vulkaninfo --summary` prints; `None` for either +/// means that half of the evidence is absent. +/// +/// The `vulkaninfo` stub is planted in **both** directions. When a scenario +/// wants the tool absent it is planted as a failing command rather than left +/// out, because `PATH` still reaches the rest of the machine: a runner with a +/// real `vulkaninfo` installed would otherwise answer for it, and a scenario +/// about a host with no Vulkan driver has to be hermetic about the host and not +/// only about the planted root. +fn plant_unsupported_gpu( + world: &mut E2eWorld, + icd_manifest: Option<&str>, + vulkaninfo_summary: Option<&str>, +) { + let root = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .to_owned(); + let host_root = root.join("host-root"); + + let node = host_root.join("sys/class/kfd/kfd/topology/nodes/1"); + std::fs::create_dir_all(&node).expect("failed to create planted KFD node"); + std::fs::write( + node.join("properties"), + format!( + "cpu_cores_count 0\nsimd_count 56\ngfx_target_version {KFD_GFX90C_TARGET_VERSION}\n" + ), + ) + .expect("failed to write planted KFD node properties"); + + if let Some(manifest) = icd_manifest { + let icd_dir = host_root.join("usr/share/vulkan/icd.d"); + std::fs::create_dir_all(&icd_dir).expect("failed to create planted ICD dir"); + std::fs::write( + icd_dir.join(manifest), + "{\"file_format_version\":\"1.0.0\",\"ICD\":{\"library_path\":\"libvulkan_fake.so\"}}\n", + ) + .expect("failed to write planted ICD manifest"); + } + + let bin = root.join("fake-bin"); + std::fs::create_dir_all(&bin).expect("failed to create the fake PATH dir"); + // The same empty-bus `lspci` the other GPU fixtures plant: the probe must + // succeed without describing the runner's real cards. + rocm_core::fake_executable::write_fake_executable(&bin.join("lspci"), "#!/bin/sh\nexit 0\n") + .expect("failed to write the fake lspci"); + let vulkaninfo = match vulkaninfo_summary { + Some(summary) => format!("#!/bin/sh\ncat <<'ROCM_E2E_EOF'\n{summary}ROCM_E2E_EOF\n"), + None => "#!/bin/sh\nexit 1\n".to_owned(), + }; + rocm_core::fake_executable::write_fake_executable(&bin.join("vulkaninfo"), &vulkaninfo) + .expect("failed to write the fake vulkaninfo"); + + let path = std::env::join_paths(std::iter::once(bin).chain(std::env::split_paths( + &std::env::var_os("PATH").unwrap_or_default(), + ))) + .expect("PATH entries cannot hold a separator"); + world.command_env.push(("PATH", path)); + world + .command_env + .push(("ROCM_CLI_TEST_HOST_ROOT", host_root.into_os_string())); +} + +#[given("a machine whose GPU has no official support but has a working Vulkan driver")] +async fn unsupported_gpu_with_vulkan(world: &mut E2eWorld) { + plant_unsupported_gpu( + world, + Some("radeon_icd.x86_64.json"), + Some(RADV_VULKANINFO_SUMMARY), + ); +} + +#[given("a machine whose GPU has no official support and no Vulkan driver")] +async fn unsupported_gpu_without_vulkan(world: &mut E2eWorld) { + plant_unsupported_gpu(world, None, None); +} + +#[given("a machine whose GPU has no official support and only a software renderer")] +async fn unsupported_gpu_with_software_renderer(world: &mut E2eWorld) { + plant_unsupported_gpu( + world, + Some("lvp_icd.x86_64.json"), + Some(LAVAPIPE_VULKANINFO_SUMMARY), + ); +} + +#[then("the inspection reports the GPU as unsupported by the official stack")] +async fn assert_gpu_unsupported_by_rocm(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + assert!( + output.contains("rocm_support:") && output.contains("not supported by ROCm"), + "the inspection must say the GPU was identified and has no ROCm build:\n{output}" + ); +} + +#[then("the inspection reports that models can still run through Vulkan")] +async fn assert_vulkan_offered(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + assert!( + output.contains("the Vulkan backend can be used"), + "a host with a working Vulkan device must be told so:\n{output}" + ); + assert!( + output.contains("vulkan: available"), + "the machine-readable line must agree with the sentence:\n{output}" + ); +} + +#[then("the inspection names the Vulkan driver it found")] +async fn assert_vulkan_driver_named(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + // The driver the planted device reported, not a guess from the ICD file + // name: naming it is what lets a reader check the claim. + assert!( + output.contains("radv"), + "the inspection must name the driver it found:\n{output}" + ); +} + +#[then("the inspection does not claim models can run through Vulkan")] +async fn assert_vulkan_not_promised(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + assert!( + !output.contains("the Vulkan backend can be used"), + "a host with no usable Vulkan GPU must not be promised one:\n{output}" + ); + assert!( + !output.contains("Vulkan backend is expected to work"), + "a host with no usable Vulkan GPU must not be promised one:\n{output}" + ); +} + +#[then("the inspection says no Vulkan driver was found")] +async fn assert_no_vulkan_driver_reported(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + assert!( + output.contains("no Vulkan GPU driver was found"), + "the inspection must say why no GPU path is open:\n{output}" + ); + assert!( + output.contains("install one"), + "the advice must be able to clear the condition it is printed for:\n{output}" + ); +} + +#[then("the inspection says the only renderer it found runs on the CPU")] +async fn assert_software_renderer_reported(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + assert!( + output.contains("not a GPU path"), + "a CPU rasteriser must be named as such, not counted as a GPU:\n{output}" + ); + // The support level must agree, not just the evidence line. Without this the + // scenario still passed when `gpu_support_level` ignored the probe entirely: + // the absence of a Vulkan promise was coming from the status wording alone, + // so nothing here held the classification itself. + assert!( + output.contains("no Vulkan GPU driver was found"), + "a host whose only renderer is the CPU has no GPU backend, and the support \ + line must say so:\n{output}" + ); +} From 90a31c3d6d8393d5429dd753e1ee973546b6382e Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 16:14:19 +0200 Subject: [PATCH 21/59] fix(examine): decide absence on whether the probe could look, not on what it found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit examine-51 went red: a planted host with no Vulkan driver was reported `unknown` instead of "no Vulkan driver was found". My own regression, from hedging `Absent` on an empty result. The hedge was right in intent and wrong in test. An empty device list does not distinguish "`vulkaninfo` is not installed" from "`vulkaninfo` ran and found nothing" — and those are opposite answers. The first is a gap in the probe; the second is exactly what a host without a driver looks like, which is why the fixture plants a `vulkaninfo` that exits 1. Decide on whether the tool was there to ask. `Absent` needs an ICD that was found and ruled out, or an enumeration that could have happened and came back empty. `Unknown` is kept for no ICD, no devices, and no `vulkaninfo` to ask — where claiming absence would tell a host with a working driver to install one. The availability check searches PATH itself: `tool_path_candidates` returns a bare program name off Windows, so the previous `is_file()` on it was always false and every host read as unable to enumerate. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/host_gpu/vulkan.rs | 114 +++++++++++++++++------- 1 file changed, 82 insertions(+), 32 deletions(-) diff --git a/crates/rocm-core/src/host_gpu/vulkan.rs b/crates/rocm-core/src/host_gpu/vulkan.rs index b1e2b08913..fc211a9283 100644 --- a/crates/rocm-core/src/host_gpu/vulkan.rs +++ b/crates/rocm-core/src/host_gpu/vulkan.rs @@ -324,6 +324,24 @@ fn detect_vulkan_devices() -> Vec { .unwrap_or_default() } +/// Whether `vulkaninfo` is on this host at all, regardless of what it said. +/// +/// The distinction the classification turns on is "could we ask" versus "we +/// asked and nothing was there" — and an empty device list cannot tell those +/// apart, because a missing binary and one that exits non-zero both yield it. +/// A host that genuinely has no Vulkan driver is the second case: `vulkaninfo` +/// is installed and fails. Only the first is a gap in the probe. +fn vulkan_enumeration_available() -> bool { + let Some(path) = std::env::var_os("PATH") else { + return false; + }; + std::env::split_paths(&path).any(|dir| { + crate::probe_exec::tool_path_candidates("vulkaninfo") + .iter() + .any(|name| dir.join(name).is_file()) + }) +} + /// Whether this platform's Vulkan surfaces can be read at all. const fn vulkan_is_probeable() -> bool { cfg!(any(target_os = "linux", windows)) @@ -336,6 +354,7 @@ pub fn detect_vulkan_support() -> VulkanSupport { detect_icds(), detect_vulkan_devices(), vulkan_is_probeable(), + vulkan_enumeration_available(), ) } @@ -346,6 +365,7 @@ pub fn classify_vulkan_support( icds: Vec, devices: Vec, probeable: bool, + enumerated: bool, ) -> VulkanSupport { if !probeable { return VulkanSupport { @@ -383,33 +403,37 @@ pub fn classify_vulkan_support( let gpu_icds: Vec<&String> = icds.iter().filter(|icd| icd_is_gpu_driver(icd)).collect(); if gpu_icds.is_empty() { - // No ICD *and* no enumerated device is an absence of evidence, not - // evidence of absence: neither source is exhaustive. The Linux ICD - // search is a fixed directory list that ignores `VK_ICD_FILENAMES` and - // `VK_DRIVER_FILES`; the Windows one reads only the legacy Khronos - // registry key, which current AMD drivers need not populate; and - // `vulkaninfo` is frequently not installed on either. A working driver - // can therefore leave both empty, and `Absent` is rendered to the user - // as a confirmed fact ("no Vulkan GPU driver was found - install one"). - // Say we could not tell instead, and keep `Absent` for the cases below - // where something was actually found and ruled out. - if icds.is_empty() && devices.is_empty() { + // Nothing found at all, and nothing that could have found it: the ICD + // search is not exhaustive (a fixed directory list on Linux that + // ignores `VK_ICD_FILENAMES`/`VK_DRIVER_FILES`; only the legacy + // Khronos key on Windows), so without `vulkaninfo` to ask, an empty + // result is a gap in the probe rather than a fact about the host. + // `Absent` renders as "no Vulkan GPU driver was found - install one", + // which would then be told to a host that has one. + // + // `enumerated` is deliberately "the tool is installed", not "the tool + // returned devices": a host genuinely without a driver HAS + // `vulkaninfo` and gets nothing from it, and that is a real answer. + if icds.is_empty() && devices.is_empty() && !enumerated { return VulkanSupport { status: VulkanStatus::Unknown, detail: Some( - "no Vulkan driver was found, but this host exposes nothing that confirms one is absent either (no ICD manifest, and `vulkaninfo` did not enumerate)" + "no Vulkan driver was found, but `vulkaninfo` is not installed either, so \ + nothing here confirms one is absent" .to_owned(), ), icds, devices, }; } + let detail = if icds.is_empty() { + "no Vulkan driver found".to_owned() + } else { + "only a software (CPU) Vulkan driver is installed; that is not a GPU path".to_owned() + }; return VulkanSupport { status: VulkanStatus::Absent, - detail: Some( - "only a software (CPU) Vulkan driver is installed; that is not a GPU path" - .to_owned(), - ), + detail: Some(detail), icds, devices, }; @@ -494,7 +518,7 @@ GPU0: fn a_real_renoir_host_reports_vulkan_available_and_names_its_driver() { let devices = parse_vulkaninfo_summary(RENOIR_WINDOWS_SUMMARY); - let support = classify_vulkan_support(Vec::new(), devices, true); + let support = classify_vulkan_support(Vec::new(), devices, true, true); assert_eq!(support.status, VulkanStatus::Available); assert!(support.status.is_usable()); @@ -525,7 +549,7 @@ GPU0: assert!(!devices[0].is_hardware()); let support = - classify_vulkan_support(vec!["lvp_icd.x86_64.json".to_owned()], devices, true); + classify_vulkan_support(vec!["lvp_icd.x86_64.json".to_owned()], devices, true, true); assert_eq!( support.status, @@ -544,8 +568,12 @@ GPU0: #[test] fn a_lavapipe_only_install_is_absent_even_before_any_device_is_enumerated() { - let support = - classify_vulkan_support(vec!["lvp_icd.x86_64.json".to_owned()], Vec::new(), true); + let support = classify_vulkan_support( + vec!["lvp_icd.x86_64.json".to_owned()], + Vec::new(), + true, + true, + ); assert_eq!(support.status, VulkanStatus::Absent); assert!(!icd_is_gpu_driver("lvp_icd.x86_64.json")); @@ -554,8 +582,12 @@ GPU0: #[test] fn an_installed_driver_with_no_enumerated_device_stays_unverified() { - let support = - classify_vulkan_support(vec!["radeon_icd.x86_64.json".to_owned()], Vec::new(), true); + let support = classify_vulkan_support( + vec!["radeon_icd.x86_64.json".to_owned()], + Vec::new(), + true, + true, + ); assert_eq!(support.status, VulkanStatus::DriverPresentDevicesUnverified); assert!( @@ -580,29 +612,44 @@ GPU0: /// a working driver would be told to install one it already has. #[test] fn no_evidence_either_way_is_not_reported_as_a_confirmed_absence() { - let nothing = classify_vulkan_support(Vec::new(), Vec::new(), true); + // `vulkaninfo` absent: nothing could have found a driver, so nothing + // may claim there is none. + let unasked = classify_vulkan_support(Vec::new(), Vec::new(), true, false); assert_eq!( - nothing.status, + unasked.status, VulkanStatus::Unknown, - "an empty probe must not claim the driver is absent" + "with no ICD and no way to enumerate, absence is unproven" ); assert!( - nothing + unasked .detail .as_deref() - .is_some_and(|detail| detail.contains("confirms")), - "the detail must say the absence is unconfirmed: {nothing:?}" + .is_some_and(|detail| detail.contains("not installed")), + "the detail must name the missing tool: {unasked:?}" + ); + + // `vulkaninfo` present and it found nothing: that IS the answer, and + // it is the shape a host genuinely without a driver takes. + let nothing = classify_vulkan_support(Vec::new(), Vec::new(), true, true); + assert_eq!( + nothing.status, + VulkanStatus::Absent, + "a probe that could look and found nothing is a real absence" ); assert!(nothing.driver_label().is_none()); // What `Absent` is now reserved for: something WAS found and ruled out. - let software_only = - classify_vulkan_support(vec!["lvp_icd.x86_64.json".to_owned()], Vec::new(), true); + let software_only = classify_vulkan_support( + vec!["lvp_icd.x86_64.json".to_owned()], + Vec::new(), + true, + true, + ); assert_eq!(software_only.status, VulkanStatus::Absent); // macOS and friends: no evidence either way, which must not read as // "no Vulkan" any more than it reads as "Vulkan works". - let unprobeable = classify_vulkan_support(Vec::new(), Vec::new(), false); + let unprobeable = classify_vulkan_support(Vec::new(), Vec::new(), false, false); assert_eq!(unprobeable.status, VulkanStatus::Unknown); assert!(!unprobeable.status.is_usable()); } @@ -618,6 +665,7 @@ GPU0: vec!["some_new_vendor_icd.x86_64.json".to_owned()], Vec::new(), true, + true, ); assert_eq!(support.status, VulkanStatus::DriverPresentDevicesUnverified); } @@ -636,7 +684,8 @@ GPU1: \tdeviceName = AMD Radeon RX 7900 XTX \tdriverName = radv "; - let support = classify_vulkan_support(Vec::new(), parse_vulkaninfo_summary(mixed), true); + let support = + classify_vulkan_support(Vec::new(), parse_vulkaninfo_summary(mixed), true, true); assert_eq!(support.status, VulkanStatus::Available); assert_eq!(support.driver_label().as_deref(), Some("radv")); @@ -662,6 +711,7 @@ GPU0: vec!["nvidia_icd.json".to_owned()], parse_vulkaninfo_summary(nvidia), true, + true, ); // The driver is installed and is not software, so the honest answer is From cc3dc5a59222d1d17d60cb8e71e11d6f2ff72762 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 17:01:18 +0200 Subject: [PATCH 22/59] fix(examine): stop promising Vulkan to a host that was probed and showed nothing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The hedge added for a false confirmed-absence left the common case wrong. A host with no ICD, no enumerated device and no `vulkaninfo` became `VulkanStatus::Unknown`, which `gpu_support_level` maps to `VulkanOnly` — so the usual no-driver host was still told to serve through Vulkan, which is the symptom this whole change exists to remove. `NoGpuBackend` was reached only when `vulkaninfo` existed and failed, or when a software-only ICD was present: the rarer shapes. `Unknown` was carrying two incompatible meanings. It now means only what its doc said — a platform the probe cannot look at, which keeps the benefit of the doubt. The new `Inconclusive` is a host that WAS looked at and showed nothing, and it maps to `NoGpuBackend`. The two reach that level with different evidence and owe different sentences, so the message is graded: `Absent` found something and ruled it out, so "install a driver" is the whole story; `Inconclusive` names the gap and both ways to close it, because claiming a confirmed absence there would tell a host with a working driver to install one it already has. `render.rs` also claimed "Vulkan was not probed on this platform" for `Absent`, which is false — that wording is now only for the unprobed platform it describes. examine-53 covers the no-`vulkaninfo` row, which examine-51 does not reach: it plants a failing `vulkaninfo`, a confirmed absence, while most hosts without a driver have no `vulkaninfo` at all. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/gfx_target.rs | 32 +++++++++++++++++++ crates/rocm-core/src/host_gpu/render.rs | 31 +++++++++++++++--- crates/rocm-core/src/host_gpu/vulkan.rs | 17 ++++++++-- tests/e2e-cucumber/features/examine.feature | 12 +++++++ tests/e2e-cucumber/tests/e2e/examine_steps.rs | 31 ++++++++++++++++++ 5 files changed, 115 insertions(+), 8 deletions(-) diff --git a/crates/rocm-core/src/gfx_target.rs b/crates/rocm-core/src/gfx_target.rs index 8b843bb6b0..157049d16e 100644 --- a/crates/rocm-core/src/gfx_target.rs +++ b/crates/rocm-core/src/gfx_target.rs @@ -617,6 +617,13 @@ pub fn gpu_support_level( match (detected_gfx_target, compatible_therock_family) { (_, Some(_)) => GpuSupportLevel::RocmHip, (Some(gfx), None) if gfx_target_has_no_rocm_build(gfx) => { + // `Unknown` only — a platform the probe cannot look at, which keeps + // the benefit of the doubt. `Inconclusive` is a host that WAS + // looked at and showed nothing, and promising it Vulkan is the + // symptom this change exists to remove; it is the common + // no-driver shape, since most such hosts have no `vulkaninfo` + // either. The wording it gets is hedged rather than a confirmed + // absence — see `render.rs`. if vulkan.is_usable() || vulkan == crate::host_gpu::VulkanStatus::Unknown { GpuSupportLevel::VulkanOnly } else { @@ -1129,6 +1136,31 @@ mod tests { /// gfx table alone, so a Renoir laptop with no Vulkan driver installed — or /// with only a CPU rasteriser — was told to go and serve a model on a /// backend that cannot start. + /// The common no-driver host: no ICD, no device, and no `vulkaninfo` to + /// ask. It must not be promised a Vulkan path — that is the symptom this + /// change exists to remove, and it is the usual shape, since a host with no + /// driver generally has no `vulkaninfo` either. + /// + /// `Unknown` (a platform the probe cannot look at) keeps the benefit of the + /// doubt; `Inconclusive` has been looked at and showed nothing. + #[test] + fn a_probed_host_with_no_evidence_is_not_promised_vulkan() { + assert_eq!( + gpu_support_level( + Some("gfx90c"), + None, + crate::host_gpu::VulkanStatus::Inconclusive + ), + GpuSupportLevel::NoGpuBackend, + "a host that was probed and showed nothing must not be told it runs on Vulkan" + ); + assert_eq!( + gpu_support_level(Some("gfx90c"), None, crate::host_gpu::VulkanStatus::Unknown), + GpuSupportLevel::VulkanOnly, + "a platform that cannot be probed keeps the benefit of the doubt" + ); + } + #[test] fn gpu_support_level_reports_no_gpu_backend_when_the_vulkan_probe_found_none() { assert_eq!( diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index 721b6ed6a3..d655b4b1e8 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -34,6 +34,7 @@ impl ExamineSummary { VulkanStatus::Available => "available", VulkanStatus::DriverPresentDevicesUnverified => "driver_present_devices_unverified", VulkanStatus::Absent => "absent", + VulkanStatus::Inconclusive => "inconclusive", VulkanStatus::Unknown => "unknown", }; vulkan.detail.as_deref().map_or_else( @@ -95,10 +96,15 @@ impl ExamineSummary { VulkanStatus::DriverPresentDevicesUnverified => "a Vulkan driver is installed, so the Vulkan backend is expected to work" .to_owned(), - VulkanStatus::Absent | VulkanStatus::Unknown => + VulkanStatus::Unknown => "Vulkan was not probed on this platform, so the Vulkan backend is \ unconfirmed" .to_owned(), + VulkanStatus::Inconclusive => + "no Vulkan driver was found, and `vulkaninfo` is not installed to \ + confirm it" + .to_owned(), + VulkanStatus::Absent => "no Vulkan GPU driver was found".to_owned(), } ), // Both halves are established facts: the catalog confirmed there is @@ -107,10 +113,25 @@ impl ExamineSummary { // out on this host, so the advice is to install a driver rather than // to serve a model and read back what happened. GpuSupportLevel::NoGpuBackend => format!( - " rocm_support: GPU detected ({}) but not supported by ROCm, and no Vulkan GPU \ - driver was found - install one (Mesa RADV on Linux, the AMD display driver on \ - Windows) to run models on this GPU\n", - detected.unwrap_or_default() + " rocm_support: GPU detected ({}) but not supported by ROCm, and {}\n", + detected.unwrap_or_default(), + // Two different evidentiary states reach this level, and they + // owe the user different sentences. `Absent` is a confirmed + // negative — the probe found something and ruled it out, so + // "install a driver" is the whole story. `Inconclusive` was + // looked at and showed nothing either way; claiming a + // confirmed absence there would tell a host that already has a + // working driver to install one, so it names the gap and the + // way to close it. + match vulkan { + VulkanStatus::Inconclusive => + "no Vulkan GPU driver was found either - `vulkaninfo` is not installed, \ + so install it to check, or install a driver (Mesa RADV on Linux, the \ + AMD display driver on Windows) to run models on this GPU", + _ => + "no Vulkan GPU driver was found - install one (Mesa RADV on Linux, the \ + AMD display driver on Windows) to run models on this GPU", + } ), // The covering family is the fact; it is already printed on its own // line above, so this one says what it means rather than repeating diff --git a/crates/rocm-core/src/host_gpu/vulkan.rs b/crates/rocm-core/src/host_gpu/vulkan.rs index fc211a9283..93d7c62787 100644 --- a/crates/rocm-core/src/host_gpu/vulkan.rs +++ b/crates/rocm-core/src/host_gpu/vulkan.rs @@ -58,6 +58,16 @@ pub enum VulkanStatus { /// The probe looked and found no GPU Vulkan driver. A software rasteriser /// alone lands here, not in [`Self::Available`]. Absent, + /// The probe looked and came away with nothing either way: no ICD manifest, + /// no enumerated device, and no `vulkaninfo` to ask. + /// + /// Distinct from [`Self::Unknown`], which is a platform that cannot be + /// probed at all. The difference decides what the host is told: an + /// unprobed platform keeps the benefit of the doubt, while this one has + /// been looked at and nothing was found, so it must not be promised a + /// Vulkan path — but neither may it be told a driver is definitely absent, + /// since the sources that came back empty are not exhaustive. + Inconclusive, /// The probe could not look (an unsupported platform). /// /// The default: a value nobody has filled in is one nobody has probed, and @@ -416,7 +426,7 @@ pub fn classify_vulkan_support( // `vulkaninfo` and gets nothing from it, and that is a real answer. if icds.is_empty() && devices.is_empty() && !enumerated { return VulkanSupport { - status: VulkanStatus::Unknown, + status: VulkanStatus::Inconclusive, detail: Some( "no Vulkan driver was found, but `vulkaninfo` is not installed either, so \ nothing here confirms one is absent" @@ -617,8 +627,9 @@ GPU0: let unasked = classify_vulkan_support(Vec::new(), Vec::new(), true, false); assert_eq!( unasked.status, - VulkanStatus::Unknown, - "with no ICD and no way to enumerate, absence is unproven" + VulkanStatus::Inconclusive, + "with no ICD and no way to enumerate, absence is unproven — but the \ + host WAS looked at, which `Unknown` (an unprobed platform) does not say" ); assert!( unasked diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index e64ae34929..976bc2786d 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -789,3 +789,15 @@ Feature: GPU detection and system inspection When the user inspects that machine Then the inspection does not claim models can run through Vulkan And the inspection says the only renderer it found runs on the CPU + + # The shape examine-51 does not reach. examine-51 plants a `vulkaninfo` that + # fails, which is a confirmed absence; most hosts without a driver have no + # `vulkaninfo` at all, and that host must be told the check could not be made + # rather than either promised Vulkan or declared to have nothing. + @id:examine-cannot-confirm-vulkan-without-vulkaninfo @requires-os:linux + Scenario: examine-53 - A machine that cannot be checked for Vulkan is not told either way + Given a machine whose GPU has no official support and no way to check Vulkan + When the user inspects that machine + Then the inspection reports the GPU as unsupported by the official stack + And the inspection does not claim models can run through Vulkan + And the inspection says it could not confirm whether a Vulkan driver is there diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index a5bc2b38c3..ad030c302e 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -3486,6 +3486,37 @@ async fn unsupported_gpu_without_vulkan(world: &mut E2eWorld) { plant_unsupported_gpu(world, None, None); } +/// The *common* no-driver shape, and the one examine-51 does not cover: a host +/// with no ICD and no `vulkaninfo` to ask. `plant_unsupported_gpu` always writes +/// a `vulkaninfo`, so this removes it again — the probe must then report that it +/// could not confirm, rather than either promising Vulkan or declaring it absent. +#[given("a machine whose GPU has no official support and no way to check Vulkan")] +async fn unsupported_gpu_without_vulkaninfo(world: &mut E2eWorld) { + plant_unsupported_gpu(world, None, None); + let bin = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("fake-bin"); + std::fs::remove_file(bin.join("vulkaninfo")).expect("failed to remove the planted vulkaninfo"); +} + +#[then("the inspection says it could not confirm whether a Vulkan driver is there")] +async fn assert_vulkan_unconfirmed(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + let line = rocm_support_line(output).expect("no rocm_support line"); + assert!( + line.contains("`vulkaninfo` is not installed"), + "the support line must name the gap that stopped it confirming, not assert \ + a driver is absent: {line}\n{output}" + ); + assert!( + !line.contains("Vulkan backend") && !line.contains("runs on"), + "and it must not promise a Vulkan path it never established: {line}\n{output}" + ); +} + #[given("a machine whose GPU has no official support and only a software renderer")] async fn unsupported_gpu_with_software_renderer(world: &mut E2eWorld) { plant_unsupported_gpu( From b4479e5493a4b2859de48ef1fbc5d3b338005087 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 17:06:12 +0200 Subject: [PATCH 23/59] fix(examine): finish removing the split-out work from this PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three leftovers from the split, all of which made this diff claim work it no longer ships. `stdio_client.rs` still carried an older, UNCONDITIONAL version of the endpoint key-file clearing — the one that broke `rocmd`'s contract before #1034 made it opt-in. Reset to main's. The guard lives in #1034 alone now, and `rocmd`'s own `the_endpoint_key_file_reaches_the_child_only_when_it_holds_a_usable_key` is green again here. `docs/testing.md` still described four support levels and pointed at examine-50/51/52. This change ships three levels and examine-49; the fourth level and those scenarios are #1036's, and the text now says so. The `NoGpuBackend` arm in `assert_support_level_advice_fits` was dead here — nothing can produce that level without the probe — so it goes back to #1036 with the level it tests. Signed-off-by: Roman Sirokov --- .../rocm-engine-protocol/src/stdio_client.rs | 49 ------------------- docs/testing.md | 24 ++++----- tests/e2e-cucumber/tests/e2e/examine_steps.rs | 9 ---- 3 files changed, 12 insertions(+), 70 deletions(-) diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 67511ad01d..5a8989e5a7 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -155,15 +155,6 @@ where } if let Some(key_file) = options.endpoint_key_file { command.env(ENDPOINT_API_KEY_FILE_ENV, key_file); - } else { - // Cleared, not merely left unset. The caller said this child gets no - // endpoint key, and an engine plugin is a third-party binary: a value - // this process happens to have inherited must not reach it just - // because nothing overwrote it. `rocm`'s plugin calls pass `None` - // here, so this is what keeps a 0600 key-file path off their - // environment. See - // `an_external_plugin_child_is_not_handed_the_endpoint_key_file`. - command.env_remove(ENDPOINT_API_KEY_FILE_ENV); } if options.request_progress { command.env(ENGINE_PROGRESS_STDERR_ENV, "1"); @@ -684,46 +675,6 @@ mod tests { assert!(added_env(&command).is_empty(), "{:?}", added_env(&command)); } - /// A child told it gets no endpoint key must have the variable CLEARED, - /// not merely left unset. - /// - /// `rocm` passes `endpoint_key_file: None` for every external engine - /// plugin — a third-party binary — and the path it would otherwise carry - /// points at a mode-0600 file holding a live endpoint credential. Merely - /// not setting it would still hand the value through whenever this process - /// inherited one. - /// - /// Asserts the distinction directly: `Command` records `env_remove` as an - /// explicit `(key, None)` entry, so a plain "is it absent" check would pass - /// even with the clearing deleted. - #[test] - fn an_external_plugin_child_is_not_handed_the_endpoint_key_file() { - let key_var = std::ffi::OsStr::new(ENDPOINT_API_KEY_FILE_ENV); - let command = engine_stdio_command( - Path::new("/an/engine"), - ["stdio"], - &EngineStdioOptions { - app_paths: None, - env_root: None, - endpoint_key_file: None, - request_progress: false, - forward_stderr: false, - note_failing_exit_after_answer: false, - }, - ); - - let entry = command - .get_envs() - .find(|(key, _)| *key == key_var) - .map(|(_, value)| value); - assert_eq!( - entry, - Some(None), - "the key-file variable must be explicitly CLEARED when no key file \ - is passed, not merely left unset" - ); - } - #[test] fn each_option_sets_exactly_its_own_variable() { let paths = AppPaths { diff --git a/docs/testing.md b/docs/testing.md index 642d5e5c6c..92f0ca370d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -3892,22 +3892,22 @@ row is the backend and ordinal with no name — correct and unspecific, rather than specific and wrong. No CI lane has two AMD GPUs, so that branch is covered by unit tests in [host_gpu.rs](../crates/rocm-core/src/host_gpu.rs) only. -**`rocm examine` prints a `rocm_support` line on every host**, at one of four -levels (`gpu_support_level`): ROCm/HIP-supported, Vulkan-only, no GPU backend, -or unknown. Unknown covers both "nothing detected" and "a target this build's -catalog does not recognise" — conflating the latter with "confirmed to have no -ROCm build" would turn a detection gap into a claim about the hardware, which -is why it is distinct from the `NoGpuBackend` level that *is* a confirmed -answer. `examine-49` holds the line against the `detected_gfx_target` and -`compatible_therock_family` the same report prints, so every lane checks the -level its own hardware implies; `examine-50`, `examine-51` and `examine-52` -cover the Vulkan tiers (a driver present, none present, and a software -renderer that must not count as a GPU path) and are `@requires-os:linux`: +**`rocm examine` prints a `rocm_support` line on every host**, at one of three +levels (`gpu_support_level`): ROCm/HIP-supported, Vulkan-only, or unknown. +Unknown covers both "nothing detected" and "a target this build's catalog does +not recognise" — conflating the latter with "confirmed to have no ROCm build" +would turn a detection gap into a claim about the hardware. `examine-49` holds +the line against the `detected_gfx_target` and `compatible_therock_family` the +same report prints, so every lane checks the level its own hardware implies: ```bash -cargo xtask e2e -- -n '(examine-49|examine-5[012])' +cargo xtask e2e -- -n '(examine-49)' ``` +A fourth level, `NoGpuBackend`, and the scenarios covering the Vulkan tiers +live in the Vulkan-probe change (#1036), which is what makes that level +knowable. + ## Windows Tool Notes The TheRock SDK wheel install path should not require users to install global diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 8fbe54b3cf..06a170a267 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -3312,15 +3312,6 @@ async fn assert_support_level_advice_fits(world: &mut E2eWorld) { (_, Some(_), Some(_)) => ("rocm serve ", &["rocm install sdk"]), // ROCm-capable with no managed SDK yet: that install is the next step. (_, Some(_), None) => ("rocm install sdk", &[]), - // A GPU no package family covers, and the Vulkan probe found no GPU - // driver either: there is no backend to serve on, so the line owes a - // way to GET one rather than a serve command that would be refused. - // Branched on the rendered tier the way the renderer itself is — a - // flat expectation here failed on exactly this hardware, since this - // scenario is untagged and runs on every lane. - (Some(_), None, _) if line.contains("no Vulkan GPU driver was found") => { - ("install one", &["rocm serve", "rocm install sdk"]) - } // A GPU no package family covers — confirmed Vulkan-only (gfx90c) or // merely unrecognized. Either way the SDK has nothing for it and // Lemonade's Vulkan backend is what can serve on it. From a4dd8a638837186e50c9b41bfd84e625a808440c Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 17:17:14 +0200 Subject: [PATCH 24/59] refactor(examine): resolve vulkaninfo through the one PATH lookup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `probe_exec::which` already does this, and `docs/architecture.md` names it as the only copy of that lookup. My hand-rolled walk was a second one — and the Unknown/Absent split depends on it agreeing with the resolution `capture_optional_command` uses, which is the same `tool_path_candidates`. `which` also drops empty PATH entries, which mine did not. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/host_gpu/vulkan.rs | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/crates/rocm-core/src/host_gpu/vulkan.rs b/crates/rocm-core/src/host_gpu/vulkan.rs index 93d7c62787..2dbfb544de 100644 --- a/crates/rocm-core/src/host_gpu/vulkan.rs +++ b/crates/rocm-core/src/host_gpu/vulkan.rs @@ -342,14 +342,7 @@ fn detect_vulkan_devices() -> Vec { /// A host that genuinely has no Vulkan driver is the second case: `vulkaninfo` /// is installed and fails. Only the first is a gap in the probe. fn vulkan_enumeration_available() -> bool { - let Some(path) = std::env::var_os("PATH") else { - return false; - }; - std::env::split_paths(&path).any(|dir| { - crate::probe_exec::tool_path_candidates("vulkaninfo") - .iter() - .any(|name| dir.join(name).is_file()) - }) + crate::probe_exec::which("vulkaninfo") } /// Whether this platform's Vulkan surfaces can be read at all. From bf2a323ebef84170b8bdde3ddd62598cda2e1729 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 17:35:40 +0200 Subject: [PATCH 25/59] docs(examine): say the support line is printed on every host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `rocm examine`'s output changed on every host — the `rocm_support` line used to appear only for a recognised GPU with no ROCm build, and now appears always, at one of three levels. None of the three surfaces a user reads said so: the `examine` `--help` text, README's "GPUs with no ROCm build", and the examine steps in docs/manual-testing.md all still described the gfx90c case alone. Each now says the same thing: three levels, one line on every host, and that `unknown` covers both an undetected GPU and a target this build's catalog does not recognise — detection gaps rather than a verdict about the hardware. The manual-testing step also now tells the tester the line must be present on a supported card and on a machine with no GPU, which is what makes its absence a regression they can see. Also corrects a comment citing serve-53, which is main's Anthropic tool-use scenario; this change's is serve-55. Signed-off-by: Roman Sirokov --- README.md | 6 ++++++ apps/rocm/src/cli.rs | 13 ++++++++----- apps/rocm/src/therock/family.rs | 2 +- docs/manual-testing.md | 5 ++++- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index bb6ec82e45..29a73f1bd2 100644 --- a/README.md +++ b/README.md @@ -509,6 +509,12 @@ family is substituted for it. `rocm examine` reports the same conclusion on a These GPUs still work with the Vulkan backend, which needs no ROCm SDK. +`rocm examine` prints that `rocm_support` line on **every** host, not only this +case, at one of three levels: ROCm/HIP-supported, Vulkan-only (the case above), +or unknown. Unknown covers both a GPU nothing detected and a target this build's +catalog does not recognise — detection gaps rather than a verdict about the +hardware, so neither is reported as "no ROCm build". + #### Compiler toolchain (--devel) Pass `--devel` to also install the compiler and headers needed to build GPU diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index 459d6d47d0..507146bfca 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -63,11 +63,14 @@ pub(crate) struct Cli { pub(crate) enum Command { /// Check this computer's GPU, ROCm install, engines, and setup folders. /// - /// A GPU that is recognized but has no ROCm build at all - `gfx90c`, the - /// Vega iGPU in Renoir, Lucienne, Cezanne and Barcelo APUs - is reported on - /// a `rocm_support` line saying so, so that an empty - /// `compatible_therock_family` does not read as a failed probe. Such a GPU - /// still works through the Vulkan backend. + /// A `rocm_support` line is printed on **every** host, at one of three + /// levels: ROCm/HIP-supported, Vulkan-only, or unknown. Vulkan-only covers + /// a GPU that is recognized but has no ROCm build at all — `gfx90c`, the + /// Vega iGPU in Renoir, Lucienne, Cezanne and Barcelo APUs — which still + /// works through Lemonade's Vulkan backend. Unknown covers both a GPU + /// nothing detected and a target this build's catalog does not recognise: + /// those are detection gaps, not a verdict on the hardware, which is why + /// an empty `compatible_therock_family` does not read as a failed probe. Examine { /// Emit the machine-readable Examination JSON (for diagnosis tooling). #[arg(long)] diff --git a/apps/rocm/src/therock/family.rs b/apps/rocm/src/therock/family.rs index c9362ac406..31b836fcbc 100644 --- a/apps/rocm/src/therock/family.rs +++ b/apps/rocm/src/therock/family.rs @@ -197,7 +197,7 @@ pub(super) fn resolve_family( /// reachable only on a host with at least one such card, and the pre-flight /// therefore sees ordinal 0. `rocm_core`'s /// `a_gfx90c_host_the_kernel_gives_no_compute_node_still_has_a_usable_gpu` -/// drives both halves off one planted `/sys`, and serve-53 runs the real binary +/// drives both halves off one planted `/sys`, and serve-55 runs the real binary /// against that same host shape on a GPU-less lane. fn unsupported_host_gpu_message(raw_arch: &str) -> String { format!( diff --git a/docs/manual-testing.md b/docs/manual-testing.md index c0d4c14c1e..1aac94ca85 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -290,7 +290,10 @@ checks, which cannot be made anywhere else: `compatible_therock_family`, plus a `rocm_support` line saying the GPU is detected but not supported by ROCm and that the Vulkan backend can still be used. A `` target, or a missing `rocm_support` line, means detection - regressed. + regressed. That line is printed on **every** host, at one of three levels — + ROCm/HIP-supported, Vulkan-only (this case), or unknown — so on a supported + card or a machine with no GPU it must still be there, saying the level that + host's hardware implies rather than being absent. - A plain `rocm install sdk`, with no `--family`, gives the same refusal with the wording for a *detected* GPU rather than a requested one. - `rocm diagnose` reports `info-20-gpu-no-rocm-support`, naming `gfx90c` and From 60aa2883babe1f0a1338f498fae2ade00d7639f1 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 17:45:55 +0200 Subject: [PATCH 26/59] fix(examine): do not report an unchecked host as having no GPU backend MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mapping `Inconclusive` to `NoGpuBackend` was wrong, and wrong in the direction that matters: that level renders as a confirmed fact and is serialized for JSON consumers, and this state has not earned it. On Windows the Khronos `Drivers` key is legacy and `vulkaninfo` is rarely on PATH, so a working Adrenalin laptop — the machine shape ROCMAI-550 was reported from — plausibly lands here and would be told it has no GPU backend. That is a worse answer than the static claim this change set out to replace. It does not belong on `VulkanOnly` either, which promises a backend that may not be there. So it gets its own level, `VulkanUnconfirmed`: say what was looked for, that nothing was found, and how to settle it — install `vulkaninfo` to check, or a driver to fix it. Code, the architecture entry and the PR description now agree on this, which they did not before. Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 7 +++++ crates/rocm-core/src/gfx_target.rs | 37 +++++++++++++++++++++---- crates/rocm-core/src/host_gpu/render.rs | 11 ++++++++ docs/architecture.md | 2 +- 4 files changed, 50 insertions(+), 7 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 285a5493aa..7bc534ded1 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -1087,6 +1087,13 @@ const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str // `Unknown`, because what it means is the opposite: `Unknown` promises // that a command can still answer, and here the question is already // answered and the answer is no. + // Looked at and inconclusive. Says the same "no ROCm/HIP" fact as the + // arm below, but stops short of its claim about Vulkan, because + // nothing here established it either way. + rocm_core::GpuSupportLevel::VulkanUnconfirmed => { + "default embedded Lemonade server; this GPU has no ROCm/HIP support, and whether a \ + Vulkan driver is present could not be determined - run `rocm examine`" + } rocm_core::GpuSupportLevel::NoGpuBackend => { "default embedded Lemonade server; this GPU has no ROCm/HIP support and no Vulkan driver was found, so no GPU backend can run here" } diff --git a/crates/rocm-core/src/gfx_target.rs b/crates/rocm-core/src/gfx_target.rs index 157049d16e..cb6ab7bcba 100644 --- a/crates/rocm-core/src/gfx_target.rs +++ b/crates/rocm-core/src/gfx_target.rs @@ -577,6 +577,22 @@ pub enum GpuSupportLevel { /// a host with no Vulkan driver installed, or with only a software /// rasteriser, was sent to a backend that cannot start. VulkanOnly, + /// Recognized, confirmed to have no ROCm/HIP build, and the Vulkan probe + /// could not reach an answer — no ICD manifest, no enumerated device, and + /// no `vulkaninfo` to ask. + /// + /// Deliberately NOT folded into [`Self::NoGpuBackend`]. That level renders + /// as a confirmed fact and is serialized for JSON consumers, and this state + /// has not earned it: on Windows the Khronos `Drivers` key is legacy and + /// `vulkaninfo` is rarely on PATH, so a working Adrenalin laptop — the + /// machine shape ROCMAI-550 was reported from — can land here. Telling it + /// there is no GPU backend would be a worse answer than the static claim + /// this change replaced. + /// + /// Nor is it [`Self::VulkanOnly`], which promises a backend that may not be + /// there. It is its own answer: say what was looked for, that nothing was + /// found, and how to settle it. + VulkanUnconfirmed, /// Recognized, confirmed to have no ROCm/HIP build, **and** no Vulkan GPU /// driver was found: this host has no GPU path at all. /// @@ -624,10 +640,16 @@ pub fn gpu_support_level( // no-driver shape, since most such hosts have no `vulkaninfo` // either. The wording it gets is hedged rather than a confirmed // absence — see `render.rs`. - if vulkan.is_usable() || vulkan == crate::host_gpu::VulkanStatus::Unknown { - GpuSupportLevel::VulkanOnly - } else { - GpuSupportLevel::NoGpuBackend + match vulkan { + // A platform the probe cannot look at keeps the benefit of the + // doubt, as it did before any probe existed. + v if v.is_usable() || v == crate::host_gpu::VulkanStatus::Unknown => { + GpuSupportLevel::VulkanOnly + } + // Looked at, and nothing either way. Not a confirmed absence — + // see `VulkanUnconfirmed`. + crate::host_gpu::VulkanStatus::Inconclusive => GpuSupportLevel::VulkanUnconfirmed, + _ => GpuSupportLevel::NoGpuBackend, } } _ => GpuSupportLevel::Unknown, @@ -1151,8 +1173,11 @@ mod tests { None, crate::host_gpu::VulkanStatus::Inconclusive ), - GpuSupportLevel::NoGpuBackend, - "a host that was probed and showed nothing must not be told it runs on Vulkan" + GpuSupportLevel::VulkanUnconfirmed, + "a host probed with no answer must be told neither that Vulkan runs nor \ + that it is absent — a Windows box with a working Adrenalin driver lands \ + here, since the legacy Khronos key and a missing `vulkaninfo` leave no \ + evidence to find" ); assert_eq!( gpu_support_level(Some("gfx90c"), None, crate::host_gpu::VulkanStatus::Unknown), diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index d655b4b1e8..c7c204495c 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -112,6 +112,17 @@ impl ExamineSummary { // none. Unlike `Unknown` below, there is nothing left to go and find // out on this host, so the advice is to install a driver rather than // to serve a model and read back what happened. + // Looked at, nothing found, and nothing that could have found it. + // Says what was searched and how to settle it, rather than either + // promising a backend or declaring one absent — neither of which + // the evidence supports. See `GpuSupportLevel::VulkanUnconfirmed`. + GpuSupportLevel::VulkanUnconfirmed => format!( + " rocm_support: GPU detected ({}) but not supported by ROCm; no Vulkan driver \ + was found either, and `vulkaninfo` is not installed to confirm that - install \ + `vulkaninfo` to check, or a Vulkan driver (Mesa RADV on Linux, the AMD display \ + driver on Windows) to run models on this GPU\n", + detected.unwrap_or_default() + ), GpuSupportLevel::NoGpuBackend => format!( " rocm_support: GPU detected ({}) but not supported by ROCm, and {}\n", detected.unwrap_or_default(), diff --git a/docs/architecture.md b/docs/architecture.md index 89af344d5b..b4c772a869 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -319,7 +319,7 @@ The library every first-party app and engine crate depends on. It holds host exa | [`host_gpu/linux.rs`](../crates/rocm-core/src/host_gpu/linux.rs) | Linux sysfs, KFD and DRM probes: the KFD topology, the DRM ip-discovery gfx target, the amdgpu device and driver version, the primary GPU name. | domain | | [`host_gpu/os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) | Kernel version, distro name, installed RAM, and the CPU model and vendor for the host summary. One reader returns the CPU model and vendor together for both the summary and `Examination::probe`: Linux from `/proc/cpuinfo`, Windows from the CPU row of the system inventory. `detect_kernel_version` is now the only `uname -r` reader; [`examine/host_env.rs`](../crates/rocm-core/src/examine/host_env.rs)'s `probe_os` calls it for the examination's `kernel_release`. | domain | | [`host_gpu/render.rs`](../crates/rocm-core/src/host_gpu/render.rs) | The `rocm examine` text rendering of an `ExamineSummary`. | domain | -| [`host_gpu/vulkan.rs`](../crates/rocm-core/src/host_gpu/vulkan.rs) | Whether a model can run through Vulkan on this host: ICD manifests, the Windows Khronos registry and `vulkaninfo --summary`, classified into `VulkanStatus`. `Absent` is reserved for a probe that found something and ruled it out (a software-only ICD, or an enumeration of only software devices); with no ICD, no device and no `vulkaninfo` to ask, the answer is `Unknown`, because neither source is exhaustive and claiming absence would tell a host with a working driver to install one. A software rasteriser never counts as a GPU path. | +| [`host_gpu/vulkan.rs`](../crates/rocm-core/src/host_gpu/vulkan.rs) | Whether a model can run through Vulkan on this host: ICD manifests, the Windows Khronos registry and `vulkaninfo --summary`, classified into `VulkanStatus`. `Absent` is reserved for a probe that found something and ruled it out (a software-only ICD, or an enumeration of only software devices); with no ICD, no device and no `vulkaninfo` to ask, the answer is `Inconclusive`, which reaches the user as the `VulkanUnconfirmed` support level rather than a confirmed absence, because neither source is exhaustive and claiming absence would tell a host with a working driver to install one. A software rasteriser never counts as a GPU path. | | [`host_gpu/windows_inventory.rs`](../crates/rocm-core/src/host_gpu/windows_inventory.rs) | The Windows display-adapter inventory: the PowerShell queries, their parsers, and assembling one inventory from several sources, including the system query whose CPU row the CPU reader in [`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) takes. | domain | | [`host_gpu/wsl.rs`](../crates/rocm-core/src/host_gpu/wsl.rs) | WSL detection and the Windows host driver as seen from the guest: the WSL2 signals, the librocdxg chain and the host-driver probes. | domain | | [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs) | The generic append-only JSONL store engine: sibling `.lock` file, single-line appends, torn-tail repair, size rotation and tail-first reads. | domain | From 2ee299a49823879374716f739761dcc121b460fb Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 18:07:25 +0200 Subject: [PATCH 27/59] fix(cli): keep the new examine help text ASCII-only The `examine` long help rewritten in bf2a323eb used em-dashes, which `every_help_text_in_the_command_tree_is_ascii` rejects: non-ASCII punctuation renders as mojibake on a Windows console using the OEM code page (ROCMAI-543). Parenthesize the aside instead, and spell `recognize` the same way as the `recognized` three lines above it. Signed-off-by: Roman Sirokov --- apps/rocm/src/cli.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index 507146bfca..9707913c5e 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -65,10 +65,10 @@ pub(crate) enum Command { /// /// A `rocm_support` line is printed on **every** host, at one of three /// levels: ROCm/HIP-supported, Vulkan-only, or unknown. Vulkan-only covers - /// a GPU that is recognized but has no ROCm build at all — `gfx90c`, the - /// Vega iGPU in Renoir, Lucienne, Cezanne and Barcelo APUs — which still + /// a GPU that is recognized but has no ROCm build at all (`gfx90c`, the + /// Vega iGPU in Renoir, Lucienne, Cezanne and Barcelo APUs), which still /// works through Lemonade's Vulkan backend. Unknown covers both a GPU - /// nothing detected and a target this build's catalog does not recognise: + /// nothing detected and a target this build's catalog does not recognize: /// those are detection gaps, not a verdict on the hardware, which is why /// an empty `compatible_therock_family` does not read as a failed probe. Examine { From 795ca92e9d2adbafc846bc1ab1ec01c41b9e807d Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 18:23:17 +0200 Subject: [PATCH 28/59] fix(examine): say what the Vulkan probe found, and sync the docs to it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups on the Vulkan probe. Dead and false wording: - `render.rs` spelled out `Inconclusive` and `Absent` arms inside the `VulkanOnly` branch, which `gpu_support_level` can never route there. They stay spelled out so a new `VulkanStatus` is still a compile error, but share hedged wording so a future routing change cannot silently resurrect the promise this level exists to stop making. The same branch's `NoGpuBackend` arm had an `Inconclusive` case that is now unreachable too, since that host goes to `VulkanUnconfirmed`. - `classify_vulkan_support` reported "no Vulkan driver found" on a host where `vulkaninfo` enumerated a non-AMD GPU. A device that enumerated is proof a driver answered; the ICD search is a fixed directory list that `vulkaninfo` can see past, so an empty ICD list does not license denying the driver. It now says what was established instead. - `a_non_amd_gpu_does_not_make_vulkan_an_amd_path` asserted `DriverPresentDevicesUnverified`, which `is_usable()` and routes to `VulkanOnly` — the opposite of the discrimination its name claimed, so a known limitation read as an intended behavior. Renamed to say what it pins, and it now asserts the routing too, so tightening this has to be deliberate. examine-53 was not hermetic. `plant_unsupported_gpu` leaves the runner's `PATH` on the end precisely so a real `vulkaninfo` cannot answer for a fixture, and deleting the planted stub walked straight into that: on a host with `vulkaninfo` installed the scenario described the runner. `PATH` is now narrowed to the fake directory. Verified both ways on a Linux box with a `vulkaninfo` planted on `PATH` — the scenario fails without this change and passes with it. Docs were describing the old static claim: - README.md still said these GPUs "still work with the Vulkan backend" unconditionally, the exact claim this change replaces with a probed one, and listed three support levels where there are now five. - docs/testing.md listed three levels and deferred `NoGpuBackend` to this PR; it now documents all five and what each of examine-50..53 plants. - docs/manual-testing.md gives the Renoir walkthrough in all three Vulkan directions, since no CI lane has that silicon. - docs/architecture.md's `vulkan.rs` row was missing the layer column every other row has. `gpu_support_level`'s own rustdoc still said `vulkan` decides only between `VulkanOnly` and `NoGpuBackend`, and a leftover paragraph from an earlier test had been welded onto the doc comment of another. Signed-off-by: Roman Sirokov --- README.md | 37 ++++++++++-- crates/rocm-core/src/gfx_target.rs | 9 +-- crates/rocm-core/src/host_gpu/render.rs | 50 +++++++--------- crates/rocm-core/src/host_gpu/vulkan.rs | 58 +++++++++++++++++-- docs/architecture.md | 2 +- docs/manual-testing.md | 36 ++++++++++-- docs/testing.md | 38 ++++++++---- tests/e2e-cucumber/tests/e2e/examine_steps.rs | 17 +++++- 8 files changed, 184 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index 29a73f1bd2..b4dc44d644 100644 --- a/README.md +++ b/README.md @@ -507,13 +507,40 @@ a target yourself with `--family gfx90c`: the request is refused, and no other family is substituted for it. `rocm examine` reports the same conclusion on a `rocm_support` line (see [Examine](#examine)). -These GPUs still work with the Vulkan backend, which needs no ROCm SDK. +These GPUs can still work with the Vulkan backend, which needs no ROCm SDK — +but whether they *do* depends on a Vulkan driver being installed, so `rocm +examine` looks for one rather than assuming it. It reads the Vulkan ICD +manifests (the Khronos `Drivers` registry key on Windows) and, where available, +enumerates devices with `vulkaninfo`. A software rasteriser such as lavapipe or +swiftshader does not count as a GPU driver. `rocm examine` prints that `rocm_support` line on **every** host, not only this -case, at one of three levels: ROCm/HIP-supported, Vulkan-only (the case above), -or unknown. Unknown covers both a GPU nothing detected and a target this build's -catalog does not recognise — detection gaps rather than a verdict about the -hardware, so neither is reported as "no ROCm build". +case, at one of five levels: + +| Level | Meaning | +| --- | --- | +| ROCm/HIP-supported | A TheRock package family covers this GPU. | +| Vulkan-only | No ROCm build, **and** a Vulkan GPU driver was found here. | +| Vulkan-unconfirmed | No ROCm build, and the probe could not reach an answer — no ICD, no enumerated device, and no `vulkaninfo` to ask. | +| no GPU backend | No ROCm build, **and** the probe looked and found no Vulkan GPU driver. | +| unknown | No GPU detected, or a target this build's catalog does not recognize. | + +The last row is a detection gap rather than a verdict about the hardware, which +is why it is not reported as "no ROCm build". Vulkan-unconfirmed is kept +separate from "no GPU backend" for the same reason in the other direction: on +Windows the Khronos `Drivers` key is legacy and `vulkaninfo` is rarely on PATH, +so a working machine can land there, and it has not earned a confirmed "no". + +`rocm examine --json` reports the evidence behind this in a `vulkan` object +(status, ICD manifests found, devices enumerated). The field is optional, so an +older `rocm remote doctor` summary that predates it reads as unknown rather +than as a false "no Vulkan". + +The probe runs on `rocm examine` only. `rocm engines` and the `rocm diagnose` +finding `info-20-gpu-no-rocm-support` still speak from the gfx table alone, so +they can name Vulkan on a host where `rocm examine` would say otherwise; +spawning `vulkaninfo` (and PowerShell on Windows) was judged too expensive for +a plain listing command. #### Compiler toolchain (--devel) diff --git a/crates/rocm-core/src/gfx_target.rs b/crates/rocm-core/src/gfx_target.rs index cb6ab7bcba..2216244f81 100644 --- a/crates/rocm-core/src/gfx_target.rs +++ b/crates/rocm-core/src/gfx_target.rs @@ -614,8 +614,9 @@ pub enum GpuSupportLevel { /// the detected gfx target and whichever TheRock family (if any) covers it — /// plus what the Vulkan probe found. /// -/// `vulkan` only ever decides between [`GpuSupportLevel::VulkanOnly`] and -/// [`GpuSupportLevel::NoGpuBackend`], and is consulted only once the catalog has +/// `vulkan` only ever decides between [`GpuSupportLevel::VulkanOnly`], +/// [`GpuSupportLevel::VulkanUnconfirmed`] and [`GpuSupportLevel::NoGpuBackend`], +/// and is consulted only once the catalog has /// confirmed there is no ROCm build. A host with a covering family is `RocmHip` /// whatever Vulkan reports, and an unrecognized target stays `Unknown`: finding /// a Vulkan driver says nothing about whether ROCm would also work there, so it @@ -1154,10 +1155,6 @@ mod tests { ); } - /// The defect this guards against: `VulkanOnly` used to be inferred from the - /// gfx table alone, so a Renoir laptop with no Vulkan driver installed — or - /// with only a CPU rasteriser — was told to go and serve a model on a - /// backend that cannot start. /// The common no-driver host: no ICD, no device, and no `vulkaninfo` to /// ask. It must not be promised a Vulkan path — that is the symptom this /// change exists to remove, and it is the usual shape, since a host with no diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index c7c204495c..6ecd9d3493 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -100,18 +100,18 @@ impl ExamineSummary { "Vulkan was not probed on this platform, so the Vulkan backend is \ unconfirmed" .to_owned(), - VulkanStatus::Inconclusive => - "no Vulkan driver was found, and `vulkaninfo` is not installed to \ - confirm it" - .to_owned(), - VulkanStatus::Absent => "no Vulkan GPU driver was found".to_owned(), + // Unreachable: `gpu_support_level` routes `Inconclusive` to + // `VulkanUnconfirmed` and `Absent` to `NoGpuBackend`, so + // neither can arrive under `VulkanOnly`. Kept spelled out + // rather than folded into a `_` so that adding a + // `VulkanStatus` stays a compile error here, and worded + // hedged rather than promising so that a future routing + // change cannot silently resurrect the claim this level + // exists to stop making. + VulkanStatus::Inconclusive | VulkanStatus::Absent => + "the Vulkan backend is unconfirmed".to_owned(), } ), - // Both halves are established facts: the catalog confirmed there is - // no ROCm build, and the probe looked for a Vulkan driver and found - // none. Unlike `Unknown` below, there is nothing left to go and find - // out on this host, so the advice is to install a driver rather than - // to serve a model and read back what happened. // Looked at, nothing found, and nothing that could have found it. // Says what was searched and how to settle it, rather than either // promising a backend or declaring one absent — neither of which @@ -123,26 +123,18 @@ impl ExamineSummary { driver on Windows) to run models on this GPU\n", detected.unwrap_or_default() ), + // Both halves are established facts: the catalog confirmed there is + // no ROCm build, and the probe looked for a Vulkan driver and found + // none. Unlike `VulkanUnconfirmed` above, there is nothing left to + // go and find out on this host, so "install a driver" is the whole + // story. Only `Absent` reaches this level — `gpu_support_level` + // routes the probed-but-inconclusive host to `VulkanUnconfirmed` + // precisely so that this sentence is never said to one. GpuSupportLevel::NoGpuBackend => format!( - " rocm_support: GPU detected ({}) but not supported by ROCm, and {}\n", - detected.unwrap_or_default(), - // Two different evidentiary states reach this level, and they - // owe the user different sentences. `Absent` is a confirmed - // negative — the probe found something and ruled it out, so - // "install a driver" is the whole story. `Inconclusive` was - // looked at and showed nothing either way; claiming a - // confirmed absence there would tell a host that already has a - // working driver to install one, so it names the gap and the - // way to close it. - match vulkan { - VulkanStatus::Inconclusive => - "no Vulkan GPU driver was found either - `vulkaninfo` is not installed, \ - so install it to check, or install a driver (Mesa RADV on Linux, the \ - AMD display driver on Windows) to run models on this GPU", - _ => - "no Vulkan GPU driver was found - install one (Mesa RADV on Linux, the \ - AMD display driver on Windows) to run models on this GPU", - } + " rocm_support: GPU detected ({}) but not supported by ROCm, and no Vulkan GPU \ + driver was found - install one (Mesa RADV on Linux, the AMD display driver on \ + Windows) to run models on this GPU\n", + detected.unwrap_or_default() ), // The covering family is the fact; it is already printed on its own // line above, so this one says what it means rather than repeating diff --git a/crates/rocm-core/src/host_gpu/vulkan.rs b/crates/rocm-core/src/host_gpu/vulkan.rs index 2dbfb544de..5c0ec3e063 100644 --- a/crates/rocm-core/src/host_gpu/vulkan.rs +++ b/crates/rocm-core/src/host_gpu/vulkan.rs @@ -429,7 +429,14 @@ pub fn classify_vulkan_support( devices, }; } - let detail = if icds.is_empty() { + // A device that enumerated is proof a driver answered, so "no Vulkan + // driver found" would be false here even though no ICD manifest was + // read: the ICD search is not exhaustive, and `vulkaninfo` can see a + // driver it misses. Say what was actually established — hardware + // answered, none of it AMD — rather than denying the driver. + let detail = if !devices.is_empty() { + "Vulkan devices were enumerated, but none of them is an AMD GPU".to_owned() + } else if icds.is_empty() { "no Vulkan driver found".to_owned() } else { "only a software (CPU) Vulkan driver is installed; that is not a GPU path".to_owned() @@ -702,8 +709,16 @@ GPU1: ); } + /// Pins a **known limitation**, not a desired behavior: an enumeration that + /// returned only non-AMD hardware is not treated as evidence against an AMD + /// Vulkan path, so such a host is still routed to `VulkanOnly` and told to + /// serve a model. An earlier name for this test claimed the opposite ("does + /// not make Vulkan an AMD path") while asserting exactly this, which hid the + /// limitation behind a reassuring name. Asserting the routing too, rather + /// than only the status, is what keeps that from happening again: if this is + /// ever tightened, this test fails and has to be rewritten deliberately. #[test] - fn a_non_amd_gpu_does_not_make_vulkan_an_amd_path() { + fn a_non_amd_only_enumeration_is_still_routed_to_vulkan_today() { let nvidia = "\ GPU0: \tvendorID = 0x10de @@ -718,9 +733,44 @@ GPU0: true, ); - // The driver is installed and is not software, so the honest answer is - // "unverified for AMD", not "available". + // The driver is installed and is not software, so the status stops short + // of "available" — but it is still `is_usable()`. assert_eq!(support.status, VulkanStatus::DriverPresentDevicesUnverified); + assert_eq!( + crate::gfx_target::gpu_support_level(Some("gfx90c"), None, support.status), + crate::gfx_target::GpuSupportLevel::VulkanOnly, + "today a non-AMD-only enumeration still promises this host a Vulkan path; \ + this is the documented limitation, and changing it must break this test" + ); + } + + /// A device that enumerated is proof a driver answered, so the report must + /// not deny one just because no ICD manifest was read: the ICD search is a + /// fixed directory list that `vulkaninfo` can see past. The earlier wording + /// said "no Vulkan driver found" on exactly this host, which is false. + #[test] + fn an_enumerated_non_amd_device_is_not_reported_as_a_missing_driver() { + let nvidia = "\ +GPU0: +\tvendorID = 0x10de +\tdeviceType = PHYSICAL_DEVICE_TYPE_DISCRETE_GPU +\tdeviceName = NVIDIA GeForce GTX 1050 Ti +\tdriverName = NVIDIA +"; + // No ICD manifest at all, so the pre-fix code fell to "no Vulkan driver + // found" despite the enumeration above. + let support = + classify_vulkan_support(Vec::new(), parse_vulkaninfo_summary(nvidia), true, true); + + let detail = support.detail.as_deref().unwrap_or_default(); + assert!( + !detail.contains("no Vulkan driver found"), + "a driver answered with a device, so it cannot be reported missing: {support:?}" + ); + assert!( + detail.contains("none of them is an AMD GPU"), + "the report should say what was established: {support:?}" + ); } #[test] diff --git a/docs/architecture.md b/docs/architecture.md index b4c772a869..9508cea9ba 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -319,7 +319,7 @@ The library every first-party app and engine crate depends on. It holds host exa | [`host_gpu/linux.rs`](../crates/rocm-core/src/host_gpu/linux.rs) | Linux sysfs, KFD and DRM probes: the KFD topology, the DRM ip-discovery gfx target, the amdgpu device and driver version, the primary GPU name. | domain | | [`host_gpu/os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) | Kernel version, distro name, installed RAM, and the CPU model and vendor for the host summary. One reader returns the CPU model and vendor together for both the summary and `Examination::probe`: Linux from `/proc/cpuinfo`, Windows from the CPU row of the system inventory. `detect_kernel_version` is now the only `uname -r` reader; [`examine/host_env.rs`](../crates/rocm-core/src/examine/host_env.rs)'s `probe_os` calls it for the examination's `kernel_release`. | domain | | [`host_gpu/render.rs`](../crates/rocm-core/src/host_gpu/render.rs) | The `rocm examine` text rendering of an `ExamineSummary`. | domain | -| [`host_gpu/vulkan.rs`](../crates/rocm-core/src/host_gpu/vulkan.rs) | Whether a model can run through Vulkan on this host: ICD manifests, the Windows Khronos registry and `vulkaninfo --summary`, classified into `VulkanStatus`. `Absent` is reserved for a probe that found something and ruled it out (a software-only ICD, or an enumeration of only software devices); with no ICD, no device and no `vulkaninfo` to ask, the answer is `Inconclusive`, which reaches the user as the `VulkanUnconfirmed` support level rather than a confirmed absence, because neither source is exhaustive and claiming absence would tell a host with a working driver to install one. A software rasteriser never counts as a GPU path. | +| [`host_gpu/vulkan.rs`](../crates/rocm-core/src/host_gpu/vulkan.rs) | Whether a model can run through Vulkan on this host: ICD manifests, the Windows Khronos registry and `vulkaninfo --summary`, classified into `VulkanStatus`. `Absent` is reserved for a probe that found something and ruled it out (a software-only ICD, or an enumeration of only software devices); with no ICD, no device and no `vulkaninfo` to ask, the answer is `Inconclusive`, which reaches the user as the `VulkanUnconfirmed` support level rather than a confirmed absence, because neither source is exhaustive and claiming absence would tell a host with a working driver to install one. A software rasteriser never counts as a GPU path. An enumeration that returned only non-AMD hardware is reported as such rather than as a missing driver, but is still routed to the `VulkanOnly` level — a known limitation. | domain | | [`host_gpu/windows_inventory.rs`](../crates/rocm-core/src/host_gpu/windows_inventory.rs) | The Windows display-adapter inventory: the PowerShell queries, their parsers, and assembling one inventory from several sources, including the system query whose CPU row the CPU reader in [`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) takes. | domain | | [`host_gpu/wsl.rs`](../crates/rocm-core/src/host_gpu/wsl.rs) | WSL detection and the Windows host driver as seen from the guest: the WSL2 signals, the librocdxg chain and the host-driver probes. | domain | | [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs) | The generic append-only JSONL store engine: sibling `.lock` file, single-line appends, torn-tail repair, size rotation and tail-first reads. | domain | diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 1aac94ca85..75ac7ac132 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -288,12 +288,36 @@ checks, which cannot be made anywhere else: - `rocm examine` shows `detected_gfx_target: gfx90c` with an empty `compatible_therock_family`, plus a `rocm_support` line saying the GPU is - detected but not supported by ROCm and that the Vulkan backend can still be - used. A `` target, or a missing `rocm_support` line, means detection - regressed. That line is printed on **every** host, at one of three levels — - ROCm/HIP-supported, Vulkan-only (this case), or unknown — so on a supported - card or a machine with no GPU it must still be there, saying the level that - host's hardware implies rather than being absent. + detected but not supported by ROCm. A `` target, or a missing + `rocm_support` line, means detection regressed. That line is printed on + **every** host, at one of five levels — ROCm/HIP-supported, Vulkan-only, + Vulkan-unconfirmed, no-GPU-backend, or unknown — so on a supported card or a + machine with no GPU it must still be there, saying the level that host's + hardware implies rather than being absent. + + Which of the three middle levels this machine gets is **not** decided by the + gfx table: `rocm examine` probes for a Vulkan driver, so the answer depends on + what is installed. This is the check that cannot be made in CI, because no + lane has this silicon. Walk it in both directions: + + - With the AMD display driver (Windows) or Mesa RADV (Linux) installed, the + line says models can still run through Vulkan and **names the driver**, and + `rocm examine --json` shows `vulkan.status` as `available` with the device + under `vulkan.devices`. Then actually run + `rocm serve --engine lemonade` — the point of the claim is that it + is true, and this is the only place it can be confirmed against real + hardware. + - With the driver removed but `vulkaninfo` still installed, the line says no + Vulkan driver was found and tells you to install one. It must **not** still + name a driver. + - With neither installed, the line says the check could not be made and names + `vulkaninfo` as the way to settle it. It must **not** say the driver is + absent — that is the Windows-laptop case ROCMAI-550 came from, and + declaring absence there is the regression `VulkanUnconfirmed` exists to + prevent. + + A software rasteriser (`lavapipe`, `swiftshader`) must never be reported as a + GPU path in any of the three. - A plain `rocm install sdk`, with no `--family`, gives the same refusal with the wording for a *detected* GPU rather than a requested one. - `rocm diagnose` reports `info-20-gpu-no-rocm-support`, naming `gfx90c` and diff --git a/docs/testing.md b/docs/testing.md index 92f0ca370d..9784c259f4 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -3892,21 +3892,39 @@ row is the backend and ordinal with no name — correct and unspecific, rather than specific and wrong. No CI lane has two AMD GPUs, so that branch is covered by unit tests in [host_gpu.rs](../crates/rocm-core/src/host_gpu.rs) only. -**`rocm examine` prints a `rocm_support` line on every host**, at one of three -levels (`gpu_support_level`): ROCm/HIP-supported, Vulkan-only, or unknown. -Unknown covers both "nothing detected" and "a target this build's catalog does -not recognise" — conflating the latter with "confirmed to have no ROCm build" -would turn a detection gap into a claim about the hardware. `examine-49` holds -the line against the `detected_gfx_target` and `compatible_therock_family` the -same report prints, so every lane checks the level its own hardware implies: +**`rocm examine` prints a `rocm_support` line on every host**, at one of five +levels (`gpu_support_level`): ROCm/HIP-supported, Vulkan-only, +Vulkan-unconfirmed, no-GPU-backend, or unknown. Unknown covers both "nothing +detected" and "a target this build's catalog does not recognize" — conflating +the latter with "confirmed to have no ROCm build" would turn a detection gap +into a claim about the hardware. `examine-49` holds the line against the +`detected_gfx_target` and `compatible_therock_family` the same report prints, +so every lane checks the level its own hardware implies: ```bash cargo xtask e2e -- -n '(examine-49)' ``` -A fourth level, `NoGpuBackend`, and the scenarios covering the Vulkan tiers -live in the Vulkan-probe change (#1036), which is what makes that level -knowable. +The three levels below ROCm/HIP are decided by the Vulkan probe, not by the +gfx table, and each has its own scenario because each is a different +evidentiary state rather than a different wording of one: + +| Scenario | Host it plants | Level asserted | +| --- | --- | --- | +| `examine-50` | An ICD and an enumerated AMD GPU. | Vulkan-only | +| `examine-51` | A `vulkaninfo` that runs and reports nothing. | no GPU backend | +| `examine-52` | Only a software rasteriser (lavapipe). | no GPU backend | +| `examine-53` | No ICD, no device, and no `vulkaninfo` to ask. | Vulkan-unconfirmed | + +```bash +cargo xtask e2e -- -n '(examine-50|examine-51|examine-52|examine-53)' +``` + +`examine-51` and `examine-53` are the pair worth keeping straight: both end +with nothing found, and they differ only in whether a tool was there to look. +That difference is the whole reason `VulkanUnconfirmed` exists, so a change +that collapses them will pass one of these and fail the other rather than +silently reporting a confirmed absence to a host nobody probed. ## Windows Tool Notes diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 9572823b17..a687618f44 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -3478,9 +3478,17 @@ async fn unsupported_gpu_without_vulkan(world: &mut E2eWorld) { } /// The *common* no-driver shape, and the one examine-51 does not cover: a host -/// with no ICD and no `vulkaninfo` to ask. `plant_unsupported_gpu` always writes -/// a `vulkaninfo`, so this removes it again — the probe must then report that it +/// with no ICD and no `vulkaninfo` to ask. The probe must then report that it /// could not confirm, rather than either promising Vulkan or declaring it absent. +/// +/// Deleting the planted stub is not enough on its own, and doing only that would +/// contradict `plant_unsupported_gpu`'s own "planted in both directions" reason: +/// that function leaves the runner's `PATH` on the end, so a CI host with a real +/// `vulkaninfo` installed would answer for the fixture and this scenario would +/// describe the runner rather than the planted machine. So `PATH` is narrowed to +/// the fake directory as well. `lspci` stays in it, and the fake executables are +/// `#!/bin/sh` scripts reached by absolute path, so nothing else the probe needs +/// is lost — only the ability to find a `vulkaninfo` anywhere. #[given("a machine whose GPU has no official support and no way to check Vulkan")] async fn unsupported_gpu_without_vulkaninfo(world: &mut E2eWorld) { plant_unsupported_gpu(world, None, None); @@ -3491,6 +3499,11 @@ async fn unsupported_gpu_without_vulkaninfo(world: &mut E2eWorld) { .path() .join("fake-bin"); std::fs::remove_file(bin.join("vulkaninfo")).expect("failed to remove the planted vulkaninfo"); + // Pushed after `plant_unsupported_gpu`'s own entry: `command_env` is applied + // in order, so this is the `PATH` the child sees. + let path = + std::env::join_paths(std::iter::once(bin)).expect("fake-bin cannot hold a separator"); + world.command_env.push(("PATH", path)); } #[then("the inspection says it could not confirm whether a Vulkan driver is there")] From 348080301b07d0d5c5387f483bb2676ce9b62106 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 18:57:14 +0200 Subject: [PATCH 29/59] fix(examine): stop describing hosts the probe did not see that way MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Second review round. Four of the findings survived the previous pass. A driver that enumerated devices was still reported as having enumerated none. The previous pass fixed that sentence on the `Absent` path but left the `DriverPresentDevicesUnverified` one, which says "but no device was enumerated" — false for the host it is most likely to describe, an NVIDIA box whose ICD is installed and whose hardware simply is not AMD. `driver_label`'s ICD fallback was not filtered through `icd_is_gpu_driver` the way `classify_vulkan_support`'s own naming is. An `Available` host whose AMD device omitted the optional `driverName`, with lavapipe also installed, could be told the software rasteriser is the driver behind the GPU path it was just promised — in the one place the user reads a driver name, and against this module's own invariant. `lemonade_engine_note`'s `VulkanUnconfirmed` arm told the reader to run `rocm examine`. That note is rendered *inside* `rocm examine`'s own output, via `examine_human_report` -> `append_examine_engine_inventory`, so the advice was circular. It now names what would actually settle the question. The comment above it claimed only `rocm examine` reaches this level; `build_codex_bridge_snapshot` reaches it too, since it builds the level from `examine.vulkan_status()` rather than the cheap probe. `examine --help` still described three support levels. This change makes five. The `Inconclusive` render path had no test, despite being the motivating Windows/Adrenalin case: `gfx_target.rs` pinned the classifier's half and nothing pinned what the user reads. The new test builds its fixture directly rather than through `vulkan_support`, which plants an AMD device — the one thing this state cannot have. Mutation-checked by giving the arm the confirmed-absence wording, which turns it red. Also broadens `docs/architecture.md`'s `Absent` description, which listed only the software-ICD and software-device cases and omitted a `vulkaninfo` that ran and found nothing, which is what examine-51 plants. Signed-off-by: Roman Sirokov --- apps/rocm/src/cli.rs | 20 +++++++----- apps/rocm/src/main.rs | 21 ++++++++---- crates/rocm-core/src/host_gpu/render.rs | 43 +++++++++++++++++++++++++ crates/rocm-core/src/host_gpu/vulkan.rs | 23 +++++++++++-- docs/architecture.md | 2 +- 5 files changed, 90 insertions(+), 19 deletions(-) diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index 9707913c5e..3fec99c642 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -63,14 +63,18 @@ pub(crate) struct Cli { pub(crate) enum Command { /// Check this computer's GPU, ROCm install, engines, and setup folders. /// - /// A `rocm_support` line is printed on **every** host, at one of three - /// levels: ROCm/HIP-supported, Vulkan-only, or unknown. Vulkan-only covers - /// a GPU that is recognized but has no ROCm build at all (`gfx90c`, the - /// Vega iGPU in Renoir, Lucienne, Cezanne and Barcelo APUs), which still - /// works through Lemonade's Vulkan backend. Unknown covers both a GPU - /// nothing detected and a target this build's catalog does not recognize: - /// those are detection gaps, not a verdict on the hardware, which is why - /// an empty `compatible_therock_family` does not read as a failed probe. + /// A `rocm_support` line is printed on **every** host, at one of five + /// levels: ROCm/HIP-supported, Vulkan-only, Vulkan-unconfirmed, no GPU + /// backend, or unknown. The three middle levels are decided by probing for + /// a Vulkan driver rather than by the gfx table: a GPU that is recognized + /// but has no ROCm build at all (`gfx90c`, the Vega iGPU in Renoir, + /// Lucienne, Cezanne and Barcelo APUs) runs through Lemonade's Vulkan + /// backend only if a driver is actually installed. Vulkan-unconfirmed means + /// the probe could not reach an answer, which is not the same as finding + /// none. Unknown covers both a GPU nothing detected and a target this + /// build's catalog does not recognize: those are detection gaps, not a + /// verdict on the hardware, which is why an empty + /// `compatible_therock_family` does not read as a failed probe. Examine { /// Emit the machine-readable Examination JSON (for diagnosis tooling). #[arg(long)] diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 7bc534ded1..1a50180c9c 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -1081,18 +1081,25 @@ const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str rocm_core::GpuSupportLevel::RocmHip => { "default embedded Lemonade server with ROCm llama.cpp backend" } - // Only `rocm examine` reaches this level — the cheap probe behind - // `host_gpu_support_level` passes `VulkanStatus::Unknown` and so lands - // on `VulkanOnly` — but the arm is spelled out rather than folded into - // `Unknown`, because what it means is the opposite: `Unknown` promises - // that a command can still answer, and here the question is already - // answered and the answer is no. // Looked at and inconclusive. Says the same "no ROCm/HIP" fact as the // arm below, but stops short of its claim about Vulkan, because // nothing here established it either way. + // + // Reached by every caller that passes a real `VulkanStatus`, which is + // `rocm examine` *and* `build_codex_bridge_snapshot` (the + // `InternalBridgeSnapshot` command and the MCP `bridge_snapshot`/ + // `engines` tools), which builds the level from `examine.vulkan_status()` + // rather than the cheap `host_gpu_support_level`. So the advice cannot + // be "run `rocm examine`": this very note is rendered *inside* that + // command's own output, through `examine_human_report` -> + // `append_examine_engine_inventory`, and told its reader to run the + // command they were already reading. It names the thing that would + // actually settle the question instead. rocm_core::GpuSupportLevel::VulkanUnconfirmed => { "default embedded Lemonade server; this GPU has no ROCm/HIP support, and whether a \ - Vulkan driver is present could not be determined - run `rocm examine`" + Vulkan driver is present could not be determined - install `vulkaninfo` to check, \ + or a Vulkan driver (Mesa RADV on Linux, the AMD display driver on Windows) to run \ + models on this GPU" } rocm_core::GpuSupportLevel::NoGpuBackend => { "default embedded Lemonade server; this GPU has no ROCm/HIP support and no Vulkan driver was found, so no GPU backend can run here" diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index 6ecd9d3493..99a41f3d48 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -563,6 +563,49 @@ mod tests { ); } + /// The motivating case, and the one nothing rendered: a host that **was** + /// probed and showed nothing either way — no ICD, no enumerated device, and + /// no `vulkaninfo` to ask. On Windows the Khronos `Drivers` key is legacy + /// and `vulkaninfo` is rarely on PATH, so a working Adrenalin laptop (the + /// machine ROCMAI-550 was reported from) lands here. + /// + /// `gfx_target.rs` pins that the classifier turns this into + /// `VulkanUnconfirmed`; this pins what the user actually reads, which is the + /// half that can go wrong on its own. Both directions are asserted: it must + /// not promise a backend, and it must not declare one absent either. + #[test] + fn examine_render_hedges_a_host_that_was_probed_and_showed_nothing() { + // Built directly rather than through `vulkan_support`, which plants an + // AMD device: an enumerated device is the one thing this state cannot + // have, and a fixture that contradicts its own status proves nothing. + let summary = gfx90c_summary_with_vulkan(Some(VulkanSupport { + status: VulkanStatus::Inconclusive, + detail: Some( + "no Vulkan driver was found, but `vulkaninfo` is not installed either, so \ + nothing here confirms one is absent" + .to_owned(), + ), + ..VulkanSupport::default() + })); + + let rendered = summary.render_text(); + + assert!( + rendered.contains("`vulkaninfo` is not installed"), + "it must name the gap that stopped it confirming: {rendered}" + ); + assert!( + !rendered.contains("the Vulkan backend can be used"), + "nothing here established a Vulkan path: {rendered}" + ); + assert!( + !rendered.contains("and no Vulkan GPU driver was found - install one"), + "and nothing established its absence either, so the confirmed-absence \ + sentence must not be used: {rendered}" + ); + assert!(rendered.contains("vulkan: inconclusive"), "{rendered}"); + } + /// An unmapped target must not be *claimed* unsupported, even though every /// host now gets a `rocm_support` line. /// diff --git a/crates/rocm-core/src/host_gpu/vulkan.rs b/crates/rocm-core/src/host_gpu/vulkan.rs index 5c0ec3e063..42273760b0 100644 --- a/crates/rocm-core/src/host_gpu/vulkan.rs +++ b/crates/rocm-core/src/host_gpu/vulkan.rs @@ -158,8 +158,17 @@ impl VulkanSupport { .find(|device| device.is_amd() && device.is_hardware()) .and_then(|device| device.driver_name.clone()) .or_else(|| { + // Filtered the same way `classify_vulkan_support` filters its + // own driver naming. Without this, an `Available` host whose AMD + // device omitted the optional `driverName`, on a machine that + // also has lavapipe installed, would be told the software + // rasteriser is the driver behind the GPU path it was just + // promised — contradicting this module's "a software rasteriser + // never counts as a GPU path" invariant in the one place the + // user actually reads a driver name. self.icds .iter() + .filter(|icd| icd_is_gpu_driver(icd)) .find_map(|icd| icd_driver_label(icd)) .map(str::to_owned) }) @@ -456,11 +465,19 @@ pub fn classify_vulkan_support( || "a Vulkan driver is installed".to_owned(), |label| format!("{label} is installed"), ); + // Two different hosts reach this status and they are not owed the same + // sentence. An empty `devices` really is "nothing answered". A non-empty one + // means `vulkaninfo` answered and the hardware it listed simply was not AMD + // — saying no device was enumerated there is false, and it is the shape an + // NVIDIA box with an AMD iGPU disabled lands in. + let detail = if devices.is_empty() { + format!("{named}, but no device was enumerated (install `vulkaninfo` to confirm)") + } else { + format!("{named}, but none of the devices it enumerated is an AMD GPU") + }; VulkanSupport { status: VulkanStatus::DriverPresentDevicesUnverified, - detail: Some(format!( - "{named}, but no device was enumerated (install `vulkaninfo` to confirm)" - )), + detail: Some(detail), icds, devices, } diff --git a/docs/architecture.md b/docs/architecture.md index 9508cea9ba..1bcf189499 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -319,7 +319,7 @@ The library every first-party app and engine crate depends on. It holds host exa | [`host_gpu/linux.rs`](../crates/rocm-core/src/host_gpu/linux.rs) | Linux sysfs, KFD and DRM probes: the KFD topology, the DRM ip-discovery gfx target, the amdgpu device and driver version, the primary GPU name. | domain | | [`host_gpu/os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) | Kernel version, distro name, installed RAM, and the CPU model and vendor for the host summary. One reader returns the CPU model and vendor together for both the summary and `Examination::probe`: Linux from `/proc/cpuinfo`, Windows from the CPU row of the system inventory. `detect_kernel_version` is now the only `uname -r` reader; [`examine/host_env.rs`](../crates/rocm-core/src/examine/host_env.rs)'s `probe_os` calls it for the examination's `kernel_release`. | domain | | [`host_gpu/render.rs`](../crates/rocm-core/src/host_gpu/render.rs) | The `rocm examine` text rendering of an `ExamineSummary`. | domain | -| [`host_gpu/vulkan.rs`](../crates/rocm-core/src/host_gpu/vulkan.rs) | Whether a model can run through Vulkan on this host: ICD manifests, the Windows Khronos registry and `vulkaninfo --summary`, classified into `VulkanStatus`. `Absent` is reserved for a probe that found something and ruled it out (a software-only ICD, or an enumeration of only software devices); with no ICD, no device and no `vulkaninfo` to ask, the answer is `Inconclusive`, which reaches the user as the `VulkanUnconfirmed` support level rather than a confirmed absence, because neither source is exhaustive and claiming absence would tell a host with a working driver to install one. A software rasteriser never counts as a GPU path. An enumeration that returned only non-AMD hardware is reported as such rather than as a missing driver, but is still routed to the `VulkanOnly` level — a known limitation. | domain | +| [`host_gpu/vulkan.rs`](../crates/rocm-core/src/host_gpu/vulkan.rs) | Whether a model can run through Vulkan on this host: ICD manifests, the Windows Khronos registry and `vulkaninfo --summary`, classified into `VulkanStatus`. `Absent` is reserved for a probe that was able to look and ruled a GPU path out: a software-only ICD, an enumeration of only software devices, or a `vulkaninfo` that ran and reported nothing (what `examine-51` plants); with no ICD, no device and no `vulkaninfo` to ask, the answer is `Inconclusive`, which reaches the user as the `VulkanUnconfirmed` support level rather than a confirmed absence, because neither source is exhaustive and claiming absence would tell a host with a working driver to install one. A software rasteriser never counts as a GPU path. An enumeration that returned only non-AMD hardware is reported as such rather than as a missing driver, but is still routed to the `VulkanOnly` level — a known limitation. | domain | | [`host_gpu/windows_inventory.rs`](../crates/rocm-core/src/host_gpu/windows_inventory.rs) | The Windows display-adapter inventory: the PowerShell queries, their parsers, and assembling one inventory from several sources, including the system query whose CPU row the CPU reader in [`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) takes. | domain | | [`host_gpu/wsl.rs`](../crates/rocm-core/src/host_gpu/wsl.rs) | WSL detection and the Windows host driver as seen from the guest: the WSL2 signals, the librocdxg chain and the host-driver probes. | domain | | [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs) | The generic append-only JSONL store engine: sibling `.lock` file, single-line appends, torn-tail repair, size rotation and tail-first reads. | domain | From d9b16c9740a6873433109e44a2a9e943434f6463 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 19:48:40 +0200 Subject: [PATCH 30/59] fix(examine): a vulkaninfo that failed has not looked at the host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The probe treated "`vulkaninfo` is on PATH" as "enumeration happened". It is not the same thing, and the gap pointed the wrong way: on a host whose ICD sits outside the fixed search directories, a `vulkaninfo` that exits non-zero, times out, or is too old to know `--summary` produced a *confirmed* `Absent` and told the user "no Vulkan GPU driver was found - install one" — the unearned claim this whole change exists to stop making. `capture_optional_command` already has the right signal and it was being thrown away: it yields `Some` only for an exit-0 run, so the capture itself distinguishes "answered" from "did not answer". `detect_vulkan_devices` now returns that alongside the devices, through a pure `vulkan_devices_from_capture` seam so the decision is testable without a Vulkan install, and `vulkan_enumeration_available`'s PATH lookup is gone. examine-51 was pinning the conflation rather than catching it: it planted a `vulkaninfo` that exits 1 and asserted a confirmed absence. The genuine "looked and found nothing" host is one that exits 0 with an empty device list, which is what it plants now — and which makes docs/testing.md's description of that fixture true rather than aspirational. Verified both ways on Linux: with the old `exit 1` stub, examine-51 now FAILS, because that host is no longer reported as a confirmed absence. The unit test covers the seam itself and goes red when the capture result is swapped back for a PATH lookup. Also qualifies two places that overclaimed what `VulkanOnly` means. The README row and `GpuSupportLevel::VulkanOnly`'s rustdoc both said a driver "was found", but `VulkanStatus::Unknown` — an unprobeable platform, or a summary from an older `rocm remote doctor` — maps there too, deliberately, to keep the benefit of the doubt. The level means "found, or nobody could look"; `render.rs` is what tells the two apart for the reader. Signed-off-by: Roman Sirokov --- README.md | 2 +- crates/rocm-core/src/gfx_target.rs | 8 ++ crates/rocm-core/src/host_gpu/render.rs | 13 +-- crates/rocm-core/src/host_gpu/vulkan.rs | 91 ++++++++++++++----- tests/e2e-cucumber/tests/e2e/examine_steps.rs | 25 +++-- 5 files changed, 101 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index b4dc44d644..81b69100e4 100644 --- a/README.md +++ b/README.md @@ -520,7 +520,7 @@ case, at one of five levels: | Level | Meaning | | --- | --- | | ROCm/HIP-supported | A TheRock package family covers this GPU. | -| Vulkan-only | No ROCm build, **and** a Vulkan GPU driver was found here. | +| Vulkan-only | No ROCm build, **and** either a Vulkan GPU driver was found here, or the platform cannot be probed for one at all (anything other than Linux and Windows, or a summary from an older `rocm remote doctor` that predates the probe). The `rocm_support` line says which of the two it is, and only names a driver when one was actually found. | | Vulkan-unconfirmed | No ROCm build, and the probe could not reach an answer — no ICD, no enumerated device, and no `vulkaninfo` to ask. | | no GPU backend | No ROCm build, **and** the probe looked and found no Vulkan GPU driver. | | unknown | No GPU detected, or a target this build's catalog does not recognize. | diff --git a/crates/rocm-core/src/gfx_target.rs b/crates/rocm-core/src/gfx_target.rs index 2216244f81..862980342a 100644 --- a/crates/rocm-core/src/gfx_target.rs +++ b/crates/rocm-core/src/gfx_target.rs @@ -570,6 +570,14 @@ pub enum GpuSupportLevel { /// was **found on this host**, so Vulkan-based engines (the default Lemonade /// backend) run on it. /// + /// One exception, and it is deliberate: [`VulkanStatus::Unknown`] — a + /// platform the probe cannot read at all, or a summary from before the probe + /// existed — also lands here, keeping the benefit of the doubt it had before + /// any probe existed rather than withholding the only path such a host has. + /// So this level means "a driver was found, **or** nobody could look"; it is + /// `render.rs` that tells the two apart in what the user reads, and it only + /// names a driver when one was actually found. + /// /// The second half of that sentence is evidence, not inference. This variant /// used to be returned from the gfx table alone — "no ROCm build" was taken /// to mean "Vulkan then", on a machine nobody had looked at. Every surface diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index 99a41f3d48..626a13edf9 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -118,9 +118,10 @@ impl ExamineSummary { // the evidence supports. See `GpuSupportLevel::VulkanUnconfirmed`. GpuSupportLevel::VulkanUnconfirmed => format!( " rocm_support: GPU detected ({}) but not supported by ROCm; no Vulkan driver \ - was found either, and `vulkaninfo` is not installed to confirm that - install \ - `vulkaninfo` to check, or a Vulkan driver (Mesa RADV on Linux, the AMD display \ - driver on Windows) to run models on this GPU\n", + was found, and `vulkaninfo` did not answer to confirm that - install \ + `vulkaninfo` (or find out why it fails) to check, or install a Vulkan driver \ + (Mesa RADV on Linux, the AMD display driver on Windows) to run models on this \ + GPU\n", detected.unwrap_or_default() ), // Both halves are established facts: the catalog confirmed there is @@ -581,8 +582,8 @@ mod tests { let summary = gfx90c_summary_with_vulkan(Some(VulkanSupport { status: VulkanStatus::Inconclusive, detail: Some( - "no Vulkan driver was found, but `vulkaninfo` is not installed either, so \ - nothing here confirms one is absent" + "no Vulkan driver was found, and `vulkaninfo` did not answer (it is not \ + installed, or it failed to run), so nothing here confirms one is absent" .to_owned(), ), ..VulkanSupport::default() @@ -591,7 +592,7 @@ mod tests { let rendered = summary.render_text(); assert!( - rendered.contains("`vulkaninfo` is not installed"), + rendered.contains("`vulkaninfo` did not answer"), "it must name the gap that stopped it confirming: {rendered}" ); assert!( diff --git a/crates/rocm-core/src/host_gpu/vulkan.rs b/crates/rocm-core/src/host_gpu/vulkan.rs index 42273760b0..4b8bc52f44 100644 --- a/crates/rocm-core/src/host_gpu/vulkan.rs +++ b/crates/rocm-core/src/host_gpu/vulkan.rs @@ -337,21 +337,30 @@ const fn detect_icds() -> Vec { /// The devices `vulkaninfo --summary` reports, or an empty list when the tool /// is not installed. Absence of the tool is not absence of a driver, which is /// why this cannot decide [`VulkanStatus`] on its own. -fn detect_vulkan_devices() -> Vec { - capture_optional_command("vulkaninfo", &["--summary"]) - .map(|output| parse_vulkaninfo_summary(&output)) - .unwrap_or_default() +fn detect_vulkan_devices() -> (Vec, bool) { + vulkan_devices_from_capture(capture_optional_command("vulkaninfo", &["--summary"])) } -/// Whether `vulkaninfo` is on this host at all, regardless of what it said. +/// The pure half of [`detect_vulkan_devices`]: the devices the capture named, +/// and whether `vulkaninfo` actually **answered**. /// -/// The distinction the classification turns on is "could we ask" versus "we -/// asked and nothing was there" — and an empty device list cannot tell those -/// apart, because a missing binary and one that exits non-zero both yield it. -/// A host that genuinely has no Vulkan driver is the second case: `vulkaninfo` -/// is installed and fails. Only the first is a gap in the probe. -fn vulkan_enumeration_available() -> bool { - crate::probe_exec::which("vulkaninfo") +/// The classification turns on "could we ask" versus "we asked and nothing was +/// there", and the second half of that is what this bool carries. It is +/// deliberately *not* "the binary exists": `capture_optional_command` yields +/// `Some` only for an exit-0 run, so a `vulkaninfo` that exits non-zero, times +/// out, or is too old to know `--summary` is `None` here — none of which is a +/// host that was looked at and found empty. +/// +/// Using mere presence on `PATH` for this was wrong in the direction that +/// matters: on a host whose ICD sits outside the fixed search directories, a +/// failing `vulkaninfo` produced a *confirmed* `Absent` and told the user "no +/// Vulkan GPU driver was found - install one", which is the unearned claim +/// this probe exists to stop making. +fn vulkan_devices_from_capture(captured: Option) -> (Vec, bool) { + captured.map_or_else( + || (Vec::new(), false), + |output| (parse_vulkaninfo_summary(&output), true), + ) } /// Whether this platform's Vulkan surfaces can be read at all. @@ -362,12 +371,8 @@ const fn vulkan_is_probeable() -> bool { /// What this host offers for Vulkan. #[must_use] pub fn detect_vulkan_support() -> VulkanSupport { - classify_vulkan_support( - detect_icds(), - detect_vulkan_devices(), - vulkan_is_probeable(), - vulkan_enumeration_available(), - ) + let (devices, enumerated) = detect_vulkan_devices(); + classify_vulkan_support(detect_icds(), devices, vulkan_is_probeable(), enumerated) } /// The pure decision behind [`detect_vulkan_support`], split out so every @@ -423,15 +428,19 @@ pub fn classify_vulkan_support( // `Absent` renders as "no Vulkan GPU driver was found - install one", // which would then be told to a host that has one. // - // `enumerated` is deliberately "the tool is installed", not "the tool - // returned devices": a host genuinely without a driver HAS - // `vulkaninfo` and gets nothing from it, and that is a real answer. + // `enumerated` is "the tool answered", not "the tool returned devices" + // and not "the tool is installed": a host genuinely without a driver + // runs `vulkaninfo` successfully and gets an empty device list, and + // that is a real answer. A `vulkaninfo` that exits non-zero, times out, + // or is too old for `--summary` is not — it answered nothing, so it + // cannot be the evidence that turns an empty result into a confirmed + // absence. if icds.is_empty() && devices.is_empty() && !enumerated { return VulkanSupport { status: VulkanStatus::Inconclusive, detail: Some( - "no Vulkan driver was found, but `vulkaninfo` is not installed either, so \ - nothing here confirms one is absent" + "no Vulkan driver was found, and `vulkaninfo` did not answer (it is not \ + installed, or it failed to run), so nothing here confirms one is absent" .to_owned(), ), icds, @@ -761,6 +770,42 @@ GPU0: ); } + /// The distinction the whole classification rests on, at the seam where it + /// is actually decided. + /// + /// `capture_optional_command` collapses "not installed", "exited non-zero", + /// "timed out" and "too old for `--summary`" into one `None`, and **none** + /// of those is a host that was looked at. Reading presence on `PATH` as + /// "it answered" meant a failing `vulkaninfo`, on a host whose ICD sits + /// outside the fixed search directories, produced a confirmed `Absent` and + /// told the user to install a driver they already had. + #[test] + fn only_a_vulkaninfo_that_answered_counts_as_having_looked() { + let (devices, enumerated) = vulkan_devices_from_capture(None); + assert!( + !enumerated && devices.is_empty(), + "a run that produced no output did not look at this host" + ); + assert_eq!( + classify_vulkan_support(Vec::new(), devices, true, enumerated).status, + VulkanStatus::Inconclusive, + "so it must not be reported as a confirmed absence" + ); + + // The genuine "looked and found none": exit 0, empty device list. + let (devices, enumerated) = + vulkan_devices_from_capture(Some("Devices:\n=======\n".to_owned())); + assert!( + enumerated && devices.is_empty(), + "an empty answer is still an answer" + ); + assert_eq!( + classify_vulkan_support(Vec::new(), devices, true, enumerated).status, + VulkanStatus::Absent, + "and that one has earned the confirmed absence" + ); + } + /// A device that enumerated is proof a driver answered, so the report must /// not deny one just because no ICD manifest was read: the ICD search is a /// fixed directory list that `vulkaninfo` can see past. The earlier wording diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index a687618f44..b7da1ad2b9 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -3401,12 +3401,19 @@ GPU0: /// `vulkaninfo_summary` what `vulkaninfo --summary` prints; `None` for either /// means that half of the evidence is absent. /// -/// The `vulkaninfo` stub is planted in **both** directions. When a scenario -/// wants the tool absent it is planted as a failing command rather than left -/// out, because `PATH` still reaches the rest of the machine: a runner with a -/// real `vulkaninfo` installed would otherwise answer for it, and a scenario -/// about a host with no Vulkan driver has to be hermetic about the host and not -/// only about the planted root. +/// The `vulkaninfo` stub is planted in **both** directions, because `PATH` +/// still reaches the rest of the machine: a runner with a real `vulkaninfo` +/// installed would otherwise answer for the fixture, and a scenario about a +/// host with no Vulkan driver has to be hermetic about the host and not only +/// about the planted root. +/// +/// `None` plants a stub that **succeeds and reports no devices**, not one that +/// fails. That distinction is the whole point of the `enumerated` input to +/// `classify_vulkan_support`: a tool that exits non-zero answered nothing, and +/// must leave the host `Inconclusive`, while one that runs and lists nothing is +/// the genuine "looked and found none" that earns a confirmed `Absent`. The +/// earlier `exit 1` stub pinned exactly the conflation this probe exists to +/// avoid, so examine-51 passed for the wrong reason. fn plant_unsupported_gpu( world: &mut E2eWorld, icd_manifest: Option<&str>, @@ -3448,7 +3455,9 @@ fn plant_unsupported_gpu( .expect("failed to write the fake lspci"); let vulkaninfo = match vulkaninfo_summary { Some(summary) => format!("#!/bin/sh\ncat <<'ROCM_E2E_EOF'\n{summary}ROCM_E2E_EOF\n"), - None => "#!/bin/sh\nexit 1\n".to_owned(), + // Exit 0 with an empty `Devices:` section: the tool answered, and the + // answer is "none". See this function's doc comment. + None => "#!/bin/sh\nprintf 'Devices:\\n=======\\n'\nexit 0\n".to_owned(), }; rocm_core::fake_executable::write_fake_executable(&bin.join("vulkaninfo"), &vulkaninfo) .expect("failed to write the fake vulkaninfo"); @@ -3511,7 +3520,7 @@ async fn assert_vulkan_unconfirmed(world: &mut E2eWorld) { let output = world.cli_output.as_ref().expect("no command was run"); let line = rocm_support_line(output).expect("no rocm_support line"); assert!( - line.contains("`vulkaninfo` is not installed"), + line.contains("`vulkaninfo` did not answer"), "the support line must name the gap that stopped it confirming, not assert \ a driver is absent: {line}\n{output}" ); From e443ed62087541772113feb5d0790e9ac0c85345 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 20:53:30 +0200 Subject: [PATCH 31/59] fix(examine): stop naming a backend for a GPU the catalog does not know MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups on the support-level messaging. The `Unknown` arm recommended `rocm serve --engine lemonade` for a detected-but-unrecognised target. An unrecognised target is a gap in this build's catalog, and a newer ROCm-capable GPU falls into it just as readily as a Vulkan-only one, so pinning the Vulkan backend is the same unestablished claim this level exists to avoid, only pointing the other way. The test next to it already gave the right reasoning — "`rocm serve` runs and its `device:` row settles the backend" — which is true only of the unpinned command, so the advice and its own justification disagreed. Plain `rocm serve ` now, and the test pins that. `assert_device_matches` panicked through `.expect` when no serve summary had been recorded. Its real invariant — the backend named by `device:` must be one the engine could have selected — has already been asserted by then; the cross-surface comparison is an extra that needs a Given which captured `rocm serve`'s output. serve-05 and serve-07 both have one, but a future scenario reusing this Then for the backend check alone would have failed for a reason about the fixture rather than about the product. Two costs are now recorded rather than left implicit, both raised in review: - `SERVICE_DEVICE_PROBE_BUDGET`'s ~18s worst case is written up as a decision, naming who pays it (`rocm services list` and the chat `services` tool, holding the process-wide engine env lock), the two cheaper options considered and why each was rejected, and the fact that it is a reversible constant rather than a structural choice. - `pinned_gpu_name` deduplicates its Windows display-inventory read within a call but does not cache across calls, and `write_running_state` runs more than once per launch, so a Windows launch pays several PowerShell/ CIM probes. It is on the launch path only, which is why the cost is accepted; a cache would need hotplug invalidation to stay correct. Also drops a stray blank line in `docs/testing.md`. Signed-off-by: Roman Sirokov --- apps/rocm/src/engine_ipc.rs | 20 +++++++++++++++++++ crates/rocm-core/src/host_gpu.rs | 10 ++++++++++ crates/rocm-core/src/host_gpu/render.rs | 20 +++++++++++++++---- docs/testing.md | 1 - tests/e2e-cucumber/tests/e2e/serving_steps.rs | 13 ++++++++---- 5 files changed, 55 insertions(+), 9 deletions(-) diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index ff9793975e..3fcf016643 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -469,6 +469,26 @@ where /// applies equally to the chat `services` tool, which renders through the same /// [`crate::services_text::render_services_text`]. /// +/// # The decision, recorded (review on #678) +/// +/// This is accepted rather than merely observed. `rocm services list` and the +/// chat `services` tool can both block ~18s on a host whose engines are slow to +/// answer, and for the duration the built-in probe holds the process-wide +/// engine env lock [`with_scoped_builtin_engine_env`] takes, so a concurrent +/// caller waits behind it. +/// +/// Two cheaper options were considered and not taken. Moving the probe off the +/// render path means the row stops reflecting the host at the moment it is +/// printed, which is the whole reason ROCMAI-550 added it — the `device:` row +/// exists to answer "what is this actually running on *now*". Probing once and +/// printing nothing for the rest makes the output depend on service ordering, +/// which reads as a bug to anyone who lists twice and sees different rows. +/// +/// Both remain reasonable calls and this one is reversible: it is a constant and +/// a start gate, not a structural choice. If a maintainer prefers a 10s hard +/// cap, the change is a per-probe deadline, which needs a cancellable probe +/// (see below) rather than a smaller number here. +/// /// Why not a per-probe deadline that would hold the total at 10s: the probe is /// `engine_request`, which for a built-in engine is a synchronous in-process /// call and for an external plugin is a `wait()` on a spawned child. Neither diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index 87b6ad185f..e9c79cbb10 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -501,6 +501,16 @@ pub fn pinned_gpu_name(gpu_indices: &[u32]) -> Option { // launch path. The read is eager rather than lazy only on the platform that // uses it: off Windows nothing is probed here at all, and on Windows the // count already needed it before any of the decisions below could run. + // + // The cost, stated rather than implied (raised in review on #678): this + // deduplicates *within* a call and is **not** cached across calls. Each + // engine's `write_running_state` calls it, and that runs more than once per + // launch (`starting`, then `running`), so a Windows launch pays that many + // PowerShell/CIM probes. It is on the launch path only — no request path + // reaches it — which is why the cost is accepted rather than cached: a + // cache would have to be invalidated on GPU hotplug to stay correct, and no + // caller is hot enough to earn that. Unverified on Windows; nothing + // available to this change can build the MSVC target. let windows_inventory = runtime_is_windows() .then(detect_windows_examine_inventory) .flatten(); diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index e6f0f479f2..89b0e85595 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -101,12 +101,20 @@ impl ExamineSummary { // nor its absence" in both arms was length without information. // The distinction it was carrying lives in this comment and in // `GpuSupportLevel`'s own docs, which is where it is load-bearing. + // The command is deliberately NOT `--engine lemonade`. An + // unrecognised target is a gap in this build's catalog, and a newer + // ROCm-capable GPU falls into it just as readily as a Vulkan-only + // one — naming the Vulkan backend would make exactly the kind of + // unestablished claim this level exists to avoid, only in the other + // direction. Plain `rocm serve` lets the engine choose, and the + // `device:` row then reports what it actually used, which is the + // answer this host needs and nothing here can predict. GpuSupportLevel::Unknown => match detected { Some(gfx) => format!( " rocm_support: unknown - {gfx} is not in this build's \ - catalog; serve with `rocm serve --engine lemonade`, \ - then read the `device:` row of `rocm services list` for the \ - backend it used\n" + catalog; serve with `rocm serve `, then read the \ + `device:` row of `rocm services list` for the backend it \ + used\n" ), None => " rocm_support: unknown - no AMD GPU was detected; run \ `rocm diagnose` to find out why none is visible\n" @@ -556,7 +564,11 @@ mod tests { // distinction against the behaviour rather than the wording. for (detected, expected_command) in [ (None, "rocm diagnose"), - (Some("gfx803"), "rocm serve --engine lemonade"), + // Plain `rocm serve`, not `--engine lemonade`: see the arm's own + // comment. The reasoning this test already gave — "`rocm serve` + // runs and its `device:` row settles the backend" — is only true of + // the unpinned command. + (Some("gfx803"), "rocm serve "), ] { let rendered = examine_summary_with_gpu(detected, None).render_text(); let line = rendered diff --git a/docs/testing.md b/docs/testing.md index 92f0ca370d..60cb0030ae 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -3857,7 +3857,6 @@ the scenario carries `@requires-os:linux`. On the GPU lanes it is `@requires-no-gpu` that excludes it. The missing hook is the deciding gate only where a Linux, GPU-less lane runs a prebuilt binary. - ## The Reported Backend Device And Support Level ROCMAI-550 made two surfaces report what this host can actually do instead of a diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 9f8feabe73..03624b221f 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -3221,10 +3221,15 @@ async fn assert_services_list_reports_device(world: &mut E2eWorld) { // that scenario in model_serving.feature). What remains asserted is the // half that is a real invariant: the two surfaces read one engine over one // RPC, so where both speak they must say the same thing. - let summary = world - .serve_summary - .as_deref() - .expect("the Given step must have recorded the serve summary"); + // And only when a summary was recorded at all. The backend assertion above + // is this step's real invariant and has already run; the cross-surface + // comparison is an extra that needs a Given which captured `rocm serve`'s + // own output. serve-05 and serve-07 both have one today, but panicking here + // would fail any future scenario reusing this Then for the backend check + // alone — a failure about the fixture rather than about the product. + let Some(summary) = world.serve_summary.as_deref() else { + return; + }; if let Some(summary_device) = device_row_value(summary) { assert_eq!( summary_device, device, From ea647e0a7250bdc59eed44deac7818826bd432ec Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 12 Nov 2026 12:10:12 +0200 Subject: [PATCH 32/59] fix(examine): stop a non-AMD enumeration reading as a missing driver MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups. The sharpest finding was a question I could answer by measuring rather than reasoning, and the answer was not the one the code assumed. A host that enumerated only non-AMD hardware was classified `Absent`, which renders "no Vulkan GPU driver was found - install one". Last round fixed that sentence's *detail* text and left the *status* alone, so the two then contradicted each other: the `vulkan:` line said devices had been enumerated while the support line beside it said to install a driver. A driver that answered is not an absent one, so it now reports `DriverPresentDevicesUnverified` — the same evidentiary state as an installed driver that enumerated nothing for us. The test asserted only `detail`, which is exactly why the two could drift; it now pins the status and the level it routes to. `gpu_support_level`'s `_ => NoGpuBackend` wildcard would have routed any future `VulkanStatus` to a level that renders as a confirmed fact. Every variant is spelled out now, with no guard: `v if v.is_usable()` reads better but does not count toward exhaustiveness, which is what forced the wildcard in the first place. The `VulkanUnconfirmed` advice said "install `vulkaninfo`". Measured on a real driverless Linux host: `vulkaninfo --summary` is installed, prints "Found no drivers!" and exits 1 — which this probe classifies as inconclusive. So the advice was telling a user to install the tool they were already running, and could never clear the state. It now names installing a *driver* as the step that can, and invites running `vulkaninfo` by hand to see what it says. That same measurement makes examine-51's fixture a known gap rather than a representative case, and the feature comment now says so instead of claiming the opposite: a real driverless host lands at examine-53's level, not examine-51's. Telling "no drivers" apart from "the tool is broken" needs the loader's own message, which the shared capture discards. examine-50's `contains("radv")` was satisfied by the `vulkan:` evidence line, so it never showed the support sentence named the driver at all. It asserts on the `rocm_support` line now. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/gfx_target.rs | 21 ++++++++-- crates/rocm-core/src/host_gpu/render.rs | 18 ++++++--- crates/rocm-core/src/host_gpu/vulkan.rs | 38 +++++++++++++++++-- tests/e2e-cucumber/features/examine.feature | 14 +++++-- tests/e2e-cucumber/tests/e2e/examine_steps.rs | 13 +++++-- 5 files changed, 84 insertions(+), 20 deletions(-) diff --git a/crates/rocm-core/src/gfx_target.rs b/crates/rocm-core/src/gfx_target.rs index 862980342a..f062a71efd 100644 --- a/crates/rocm-core/src/gfx_target.rs +++ b/crates/rocm-core/src/gfx_target.rs @@ -649,16 +649,29 @@ pub fn gpu_support_level( // no-driver shape, since most such hosts have no `vulkaninfo` // either. The wording it gets is hedged rather than a confirmed // absence — see `render.rs`. + // Every variant is spelled out, with no `_` and no guard. A guard + // (`v if v.is_usable()`) reads well but does not count toward + // exhaustiveness, so the arms below had to end in a wildcard — and + // a wildcard here silently routes any future `VulkanStatus` to + // `NoGpuBackend`, which renders as a confirmed fact. Listing them + // makes a new variant a compile error instead, the same discipline + // `render.rs` uses. The first arm is `VulkanStatus::is_usable` plus + // `Unknown`; keep them in step. match vulkan { - // A platform the probe cannot look at keeps the benefit of the - // doubt, as it did before any probe existed. - v if v.is_usable() || v == crate::host_gpu::VulkanStatus::Unknown => { + // A device answered, or a driver is installed and nothing + // contradicted it. + crate::host_gpu::VulkanStatus::Available + | crate::host_gpu::VulkanStatus::DriverPresentDevicesUnverified => { GpuSupportLevel::VulkanOnly } + // A platform the probe cannot look at keeps the benefit of the + // doubt, as it did before any probe existed. + crate::host_gpu::VulkanStatus::Unknown => GpuSupportLevel::VulkanOnly, // Looked at, and nothing either way. Not a confirmed absence — // see `VulkanUnconfirmed`. crate::host_gpu::VulkanStatus::Inconclusive => GpuSupportLevel::VulkanUnconfirmed, - _ => GpuSupportLevel::NoGpuBackend, + // The only status that has earned a confirmed absence. + crate::host_gpu::VulkanStatus::Absent => GpuSupportLevel::NoGpuBackend, } } _ => GpuSupportLevel::Unknown, diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index 84320fddd4..172023d649 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -117,11 +117,17 @@ impl ExamineSummary { // promising a backend or declaring one absent — neither of which // the evidence supports. See `GpuSupportLevel::VulkanUnconfirmed`. GpuSupportLevel::VulkanUnconfirmed => format!( + // Deliberately does NOT say "install `vulkaninfo`". Measured on + // a real driverless Linux host: `vulkaninfo --summary` is + // installed, prints "Found no drivers!" and exits 1 — which + // lands here. Telling that user to install the tool they are + // already running is advice that can never clear the state. + // Install a *driver* is the step that can. " rocm_support: GPU detected ({}) but not supported by ROCm; no Vulkan driver \ - was found, and `vulkaninfo` did not answer to confirm that - install \ - `vulkaninfo` (or find out why it fails) to check, or install a Vulkan driver \ - (Mesa RADV on Linux, the AMD display driver on Windows) to run models on this \ - GPU\n", + was found, and `vulkaninfo` could not confirm that either (not installed, or \ + it failed to run) - install a Vulkan driver (Mesa RADV on Linux, the AMD \ + display driver on Windows) to run models on this GPU, and run `vulkaninfo` \ + yourself to see what it says\n", detected.unwrap_or_default() ), // Both halves are established facts: the catalog confirmed there is @@ -590,7 +596,7 @@ mod tests { let summary = gfx90c_summary_with_vulkan(Some(VulkanSupport { status: VulkanStatus::Inconclusive, detail: Some( - "no Vulkan driver was found, and `vulkaninfo` did not answer (it is not \ + "no Vulkan driver was found, and `vulkaninfo` could not confirm that either (not \ installed, or it failed to run), so nothing here confirms one is absent" .to_owned(), ), @@ -600,7 +606,7 @@ mod tests { let rendered = summary.render_text(); assert!( - rendered.contains("`vulkaninfo` did not answer"), + rendered.contains("`vulkaninfo` could not confirm"), "it must name the gap that stopped it confirming: {rendered}" ); assert!( diff --git a/crates/rocm-core/src/host_gpu/vulkan.rs b/crates/rocm-core/src/host_gpu/vulkan.rs index 4b8bc52f44..5e36ef5eba 100644 --- a/crates/rocm-core/src/host_gpu/vulkan.rs +++ b/crates/rocm-core/src/host_gpu/vulkan.rs @@ -439,7 +439,7 @@ pub fn classify_vulkan_support( return VulkanSupport { status: VulkanStatus::Inconclusive, detail: Some( - "no Vulkan driver was found, and `vulkaninfo` did not answer (it is not \ + "no Vulkan driver was found, and `vulkaninfo` could not confirm that either (not \ installed, or it failed to run), so nothing here confirms one is absent" .to_owned(), ), @@ -452,9 +452,25 @@ pub fn classify_vulkan_support( // read: the ICD search is not exhaustive, and `vulkaninfo` can see a // driver it misses. Say what was actually established — hardware // answered, none of it AMD — rather than denying the driver. - let detail = if !devices.is_empty() { - "Vulkan devices were enumerated, but none of them is an AMD GPU".to_owned() - } else if icds.is_empty() { + // A device that enumerated is proof a driver answered, so this host is + // not one where "no Vulkan GPU driver was found - install one" is true. + // Fixing only the detail text last round left the *status* saying the + // opposite of the detail beside it: `Absent` renders that install + // advice, while the `vulkan:` line said devices had been enumerated. + // It reports what was actually established instead — a driver answered, + // and none of what it listed is AMD, which is the same evidentiary + // state as an installed driver that enumerated nothing for us. + if !devices.is_empty() { + return VulkanSupport { + status: VulkanStatus::DriverPresentDevicesUnverified, + detail: Some( + "Vulkan devices were enumerated, but none of them is an AMD GPU".to_owned(), + ), + icds, + devices, + }; + } + let detail = if icds.is_empty() { "no Vulkan driver found".to_owned() } else { "only a software (CPU) Vulkan driver is installed; that is not a GPU path".to_owned() @@ -833,6 +849,20 @@ GPU0: detail.contains("none of them is an AMD GPU"), "the report should say what was established: {support:?}" ); + + // The status and the rendered line, not just the detail. Asserting only + // the detail is what let the two disagree: the text said devices were + // enumerated while `Absent` rendered "install one" right beside it. + assert_eq!( + support.status, + VulkanStatus::DriverPresentDevicesUnverified, + "a driver that answered is not an absent one: {support:?}" + ); + assert_eq!( + crate::gfx_target::gpu_support_level(Some("gfx90c"), None, support.status), + crate::gfx_target::GpuSupportLevel::VulkanOnly, + "and it must not route to the confirmed-absence level" + ); } #[test] diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index 976bc2786d..049d2e91dc 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -791,9 +791,17 @@ Feature: GPU detection and system inspection And the inspection says the only renderer it found runs on the CPU # The shape examine-51 does not reach. examine-51 plants a `vulkaninfo` that - # fails, which is a confirmed absence; most hosts without a driver have no - # `vulkaninfo` at all, and that host must be told the check could not be made - # rather than either promised Vulkan or declared to have nothing. + # runs and reports no devices, which is the only shape that earns a confirmed + # absence; this host has no `vulkaninfo` to ask at all, and must be told the + # check could not be made rather than either promised Vulkan or declared to + # have nothing. + # + # Known gap, measured rather than assumed: a real `vulkaninfo` on a Linux host + # with no reachable driver does NOT produce examine-51's shape. It prints + # "Found no drivers!" and exits 1, which this probe classifies as + # inconclusive — so such a host lands at *this* scenario's level rather than + # examine-51's. Telling "no drivers" apart from "the tool is broken" needs the + # loader's own message, which the shared capture discards. @id:examine-cannot-confirm-vulkan-without-vulkaninfo @requires-os:linux Scenario: examine-53 - A machine that cannot be checked for Vulkan is not told either way Given a machine whose GPU has no official support and no way to check Vulkan diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index b7da1ad2b9..ef00e9a722 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -3520,7 +3520,7 @@ async fn assert_vulkan_unconfirmed(world: &mut E2eWorld) { let output = world.cli_output.as_ref().expect("no command was run"); let line = rocm_support_line(output).expect("no rocm_support line"); assert!( - line.contains("`vulkaninfo` did not answer"), + line.contains("`vulkaninfo` could not confirm"), "the support line must name the gap that stopped it confirming, not assert \ a driver is absent: {line}\n{output}" ); @@ -3566,9 +3566,16 @@ async fn assert_vulkan_driver_named(world: &mut E2eWorld) { let output = world.cli_output.as_ref().expect("no command was run"); // The driver the planted device reported, not a guess from the ICD file // name: naming it is what lets a reader check the claim. + // + // Asserted on the `rocm_support` line specifically, not the whole output. + // The `vulkan:` evidence line also carries the driver name, so a plain + // `output.contains("radv")` passed even when the support sentence — the + // half that actually makes the claim — named nothing at all. + let line = rocm_support_line(output).expect("no rocm_support line"); assert!( - output.contains("radv"), - "the inspection must name the driver it found:\n{output}" + line.contains("radv"), + "the support sentence itself must name the driver, not just the \ + evidence line: {line}\n{output}" ); } From 05df6f946b4bb554da8d15519970f7d1382d633c Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 12 Nov 2026 12:23:19 +0200 Subject: [PATCH 33/59] test(examine): split the examine-49 arm the Unknown-tier advice outgrew MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changing the Unknown tier's advice to plain `rocm serve ` left the examine-49 step still requiring `--engine lemonade` for *every* detected GPU with no covering family. A lane with an unrecognized GPU would therefore have failed against correct output; no current lane has one, which is the only reason it stayed green. The arm now splits the two tiers it was conflating. An unrecognized target expects plain `rocm serve ` and forbids `--engine lemonade` — naming the Vulkan backend for a card the catalog has no opinion about is the claim the advice change removed. A target confirmed to have no ROCm build still expects it. They are told apart by the line's own level wording, which is what the user reads. Signed-off-by: Roman Sirokov --- tests/e2e-cucumber/tests/e2e/examine_steps.rs | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 06a170a267..41285d1810 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -3312,9 +3312,25 @@ async fn assert_support_level_advice_fits(world: &mut E2eWorld) { (_, Some(_), Some(_)) => ("rocm serve ", &["rocm install sdk"]), // ROCm-capable with no managed SDK yet: that install is the next step. (_, Some(_), None) => ("rocm install sdk", &[]), - // A GPU no package family covers — confirmed Vulkan-only (gfx90c) or - // merely unrecognized. Either way the SDK has nothing for it and - // Lemonade's Vulkan backend is what can serve on it. + // A GPU no package family covers. Two different tiers land here and + // they do NOT owe the same command, which this arm used to miss: + // + // * merely unrecognized — the catalog has no opinion, and it may be a + // newer ROCm-capable card, so pinning the Vulkan backend would claim + // something nothing established. Plain `rocm serve `, and + // `--engine lemonade` is forbidden. + // * confirmed to have no ROCm build (gfx90c) — Vulkan is the backend + // that can serve it, so the advice names it. + // + // Told apart by the line's own level wording, which is what the user + // reads. Before this split the arm required `--engine lemonade` for + // both, so a lane with an unrecognized GPU would have failed against + // *correct* output; no current lane has one, which is the only reason + // it stayed green. + (Some(_), None, _) if line.contains("is not in this build's catalog") => ( + "rocm serve ", + &["rocm install sdk", "--engine lemonade"], + ), (Some(_), None, _) => ( "rocm serve --engine lemonade", &["rocm install sdk"], From 6dfdb720e189b6d7bf224cdb6a61e32c7b4f973c Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 12 Nov 2026 13:13:00 +0200 Subject: [PATCH 34/59] docs(testing): describe the levels this change has, not an unmerged one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The backend-device section announced a fourth level and pointed at the Vulkan-probe change for it. This branch has three levels and no probe, so that text reads as false the moment this merges on its own — which is the only way it can merge, since the probe is a separate PR. Rewritten to describe what is here: Vulkan-only inferred from the gfx table, three levels, and the probe named as separate work rather than as a reference the reader can follow. Signed-off-by: Roman Sirokov --- docs/testing.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index be34549a2f..b7baab4c50 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -3914,9 +3914,11 @@ same report prints, so every lane checks the level its own hardware implies: cargo xtask e2e -- -n '(examine-49)' ``` -A fourth level, `NoGpuBackend`, and the scenarios covering the Vulkan tiers -live in the Vulkan-probe change (#1036), which is what makes that level -knowable. +Vulkan-only here is inferred from the gfx table, not probed: this change has +three levels and this section describes three. Probing the host for a Vulkan +driver, and the further levels that evidence makes knowable, is separate work. +Written this way deliberately — a forward reference to an unmerged change reads +as false the moment this merges on its own. ## Windows Tool Notes From 96de042ae94da95bceb3f4f0e808fee4e3d16df8 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 13 Nov 2026 08:38:32 +0200 Subject: [PATCH 35/59] fix(examine): keep the Vulkan probe out of the cheap gather scope `gather_scoped` probed Vulkan unconditionally, so the lean scope -- documented as the one that skips the PowerShell query, and used by the runtime-install finalizer to read the GPU target and nothing else -- spawned `vulkaninfo`, or PowerShell on Windows, for a field its only caller never reads. Gate the probe on the same scope flag, renamed `host_probes` because it now buys both the Windows driver facts and the Vulkan read rather than being a detail of either. The lean scope leaves `vulkan` as `None`, which the field already defines as "not probed" and `vulkan_status` already maps to `Unknown`, never to an absence of Vulkan, so no caller sees a new answer. `the_summary_without_driver_facts_does_not_probe_vulkan` pins it: restoring the unconditional probe fails it rather than quietly putting a process back on the install path. Also corrects two documentation claims the review flagged. The README said the probe runs on `rocm examine` only; it runs on every full gather, so it now names them. docs/host-probing.md did not mention the probe at all, so it gains a section with the sources, the 1.5 s timeout each platform is held to, which scopes pay it, and the Windows cold-start timeout that reads as Vulkan-unconfirmed. Signed-off-by: Roman Sirokov --- README.md | 20 ++++++++--- crates/rocm-core/src/host_gpu.rs | 60 +++++++++++++++++++++++++------- docs/host-probing.md | 17 +++++++++ 3 files changed, 80 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 81b69100e4..a2c1570f16 100644 --- a/README.md +++ b/README.md @@ -536,11 +536,21 @@ so a working machine can land there, and it has not earned a confirmed "no". older `rocm remote doctor` summary that predates it reads as unknown rather than as a false "no Vulkan". -The probe runs on `rocm examine` only. `rocm engines` and the `rocm diagnose` -finding `info-20-gpu-no-rocm-support` still speak from the gfx table alone, so -they can name Vulkan on a host where `rocm examine` would say otherwise; -spawning `vulkaninfo` (and PowerShell on Windows) was judged too expensive for -a plain listing command. +The probe runs wherever the full host summary is gathered, not on `rocm examine` +alone: the text and JSON of `rocm examine`, the MCP and bridge snapshots, the +free-form answer, `rocm install driver` and its `--reconcile` form, and `rocmd`'s +host snapshot, MCP and sandbox paths. Each of those pays one `vulkaninfo` (or, on +Windows, one registry read) per gather. + +It does not run on the lean scope, `ExamineSummary::gather_without_driver_facts`, +which the runtime-install finalizer uses to read the GPU target: that leaves +`vulkan` as `None`, meaning "not probed", which reads as unknown rather than as a +false "no Vulkan". + +`rocm engines` and the `rocm diagnose` finding `info-20-gpu-no-rocm-support` still +speak from the gfx table alone, so they can name Vulkan on a host where `rocm +examine` would say otherwise; spawning `vulkaninfo` (and PowerShell on Windows) +was judged too expensive for a plain listing command. #### Compiler toolchain (--devel) diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index a6fbdd01fd..562831b051 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -253,17 +253,29 @@ impl ExamineSummary { Self::gather_scoped(true, Some(&examination.kernel_release)) } - /// The host summary without the Windows driver version and date, for a - /// caller that reads neither. On an AMD Windows host that skips the - /// PowerShell video-controller query `gather` adds for them; the GPU - /// target, CPU, RAM and every other field are the same. + /// The host summary without the probes that spawn a process purely to + /// describe the host, for a caller that reads none of their fields: the + /// Windows driver version and date, and the Vulkan probe. + /// + /// On an AMD Windows host that skips the PowerShell video-controller query + /// `gather` adds for the driver facts, and on every platform it skips the + /// `vulkaninfo` (or Windows registry) read behind [`vulkan`](Self::vulkan), + /// which is left `None` — "not probed", which + /// [`vulkan_status`](Self::vulkan_status) maps to [`VulkanStatus::Unknown`] + /// and never to "no Vulkan". The GPU target, CPU, RAM and every other field + /// are the same. pub fn gather_without_driver_facts() -> Result { Self::gather_scoped(false, None) } - fn gather_scoped(driver_facts: bool, examined_kernel: Option<&str>) -> Result { + /// `host_probes` buys the fields that cost a spawned process to describe + /// the host — the Windows driver facts and the Vulkan probe. It is not a + /// detail of either one: a caller that wants neither must pay for neither, + /// which is what [`gather_without_driver_facts`](Self::gather_without_driver_facts) + /// is for. + fn gather_scoped(host_probes: bool, examined_kernel: Option<&str>) -> Result { let paths = AppPaths::discover()?; - let windows_inventory = if driver_facts { + let windows_inventory = if host_probes { detect_windows_examine_inventory_with_driver_facts() } else { detect_windows_examine_inventory() @@ -299,7 +311,7 @@ impl ExamineSummary { detected_gfx_target, compatible_therock_family, detected_therock_family, - vulkan: Some(detect_vulkan_support()), + vulkan: host_probes.then(detect_vulkan_support), driver: detect_driver_summary_with_windows_inventory( windows_inventory.as_ref(), wsl.as_ref(), @@ -831,11 +843,12 @@ mod tests { } } - /// `gather_without_driver_facts` is `gather` minus the driver version and - /// date: the GPU target a caller reads is the same, and it never carries a - /// driver record the full summary lacks. That it starts no PowerShell query - /// for them is the `the_startup_paths_do_not_start_the_driver_fact_query` - /// test, which counts the sources the same scope runs. + /// `gather_without_driver_facts` is `gather` minus the fields that cost a + /// spawned process: the GPU target a caller reads is the same, and it never + /// carries a driver record the full summary lacks. That it starts no + /// PowerShell query for them is the + /// `the_startup_paths_do_not_start_the_driver_fact_query` test, which counts + /// the sources the same scope runs. #[test] fn the_summary_without_driver_facts_keeps_the_gpu_target() { let full = ExamineSummary::gather().expect("full summary"); @@ -854,6 +867,29 @@ mod tests { ); } + /// The lean scope must not run the Vulkan probe. This is the assertion, not + /// a timing measurement: the probe spawns `vulkaninfo` on Linux and starts + /// PowerShell on Windows, and the whole point of this scope is that a caller + /// reading only the GPU target does not pay for a process it will not read. + /// + /// Restoring `vulkan: Some(detect_vulkan_support())` in `gather_scoped` + /// fails this. `None` is not a loss of information: it is the "not probed" + /// value the field documents, which `vulkan_status` reports as `Unknown` + /// rather than as an absence of Vulkan. + #[test] + fn the_summary_without_driver_facts_does_not_probe_vulkan() { + let lean = ExamineSummary::gather_without_driver_facts().expect("lean summary"); + assert_eq!( + lean.vulkan, None, + "the lean scope probed Vulkan; it must stay out of the cheap paths" + ); + assert_eq!( + lean.vulkan_status(), + VulkanStatus::Unknown, + "an unprobed host must read as Unknown, never as having no Vulkan" + ); + } + /// The runtime check stays at the call site; what it feeds is decided here, /// so it is covered on every platform. Replacing the driver info with `None` /// in this function fails this test. diff --git a/docs/host-probing.md b/docs/host-probing.md index 097a970258..b7b13e70ba 100644 --- a/docs/host-probing.md +++ b/docs/host-probing.md @@ -188,6 +188,23 @@ Proposal: the summary stack's inventory survives and `probe_gpus_windows` builds Hosts: this pair is native Windows only; on WSL `gpus[]` is already empty (see the host table), so rebuilding it from the inventory changes nothing there. The characterisation test covers an inventory that answers with one AMD adapter, with an AMD and an NVIDIA adapter, with an NVIDIA adapter only, with no adapter, that fails, and that times out, because `None` in the summary and an empty `gpus[]` with a `probe_failures` entry in the examination are different answers to "no adapter" and "could not ask", and the convergence must keep that distinction. +## Vulkan support + +Verdict: not a duplicate — only the summary stack reads it — but it is a spawned process on a path that several commands reach, so it belongs in this note. + +`detect_vulkan_support` in [vulkan.rs](../crates/rocm-core/src/host_gpu/vulkan.rs) answers two questions: which ICD manifests the loader would find, and which devices actually enumerate. + +| | Linux | Windows | +|---|---|---| +| ICDs | the manifest directories, read from the filesystem | `powershell` over the Khronos `Drivers` registry keys | +| Devices | `vulkaninfo --summary` | the same registry read; `vulkaninfo` is rarely on PATH | +| Timeout | 1.5 s (`OPTIONAL_COMMAND_TIMEOUT`), per candidate spelling | the same 1.5 s, which a cold PowerShell start can exceed | +| Failure | `None` from `capture_optional_command`, which does not distinguish "not installed" from "timed out" | the same | + +Who pays it. The probe runs once per full gather, so every caller of `ExamineSummary::gather` pays one: the text and JSON of `rocm examine`, the MCP and bridge snapshots, the free-form answer, `rocm install driver` and its `--reconcile` form, and `rocmd`'s host snapshot, MCP and sandbox paths. It is deliberately *not* run by the lean scope, `gather_without_driver_facts`, which the runtime-install finalizer uses to read the GPU target and nothing else; that scope leaves `vulkan` as `None` and `vulkan_status` reports `Unknown`. `the_summary_without_driver_facts_does_not_probe_vulkan` pins that, so restoring an unconditional probe fails a test rather than quietly adding a process to the install path. + +What a user can see: on a Windows host whose PowerShell start is slower than 1.5 s, the probe times out and the host reads Vulkan-unconfirmed rather than Vulkan-capable — the same answer it gives a host with no ICD at all. That collapse is in `capture_optional_command`, not in the parser, and it is the gap called out in the review of #1036: the Windows branch has no unit coverage of its failure and timeout cases. + ## Marketing name to gfx target Verdict: confirmed duplicate, but not equivalent. The two cover different names. From e2447af6c4f2efdfd34e20a6c45b500b22599567 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 13 Nov 2026 11:17:11 +0200 Subject: [PATCH 36/59] fix(examine): restore the advice check's arms for the two probed Vulkan tiers `assert_support_level_advice_fits` backs examine-49, which is untagged and so runs on every lane's real hardware. Its `(Some(_), None, _)` catch-all demands the line contain `rocm serve --engine lemonade`, which neither the `NoGpuBackend` nor the `VulkanUnconfirmed` text ever says, so any host with a detected, ROCm-unsupported GPU in either tier fails it. No lane carries that hardware shape today, which is the only reason this is latent rather than red. An arm covering this was added on the stacked base, lost when the Vulkan work was split out, re-added here, and then dropped a second time by the merge that pulled the base back in. Restoring it as two arms, keyed on each tier's own wording, and saying in the comment that it has been lost to a merge before. Neither tier owes `--engine lemonade`: on both, nothing established a backend that can serve the GPU, so naming an engine promises a run the evidence does not support. Both forbid it along with `rocm install sdk`. Also fixes two contradictions the same review found: `lemonade_engine_note`'s `VulkanUnconfirmed` arm told the user to install `vulkaninfo`, a few lines below the `rocm_support:` line that deliberately does not -- because on a real driverless Linux host the tool is already installed, prints "Found no drivers!" and exits 1, which is what lands there. Both strings render in the same `rocm examine` report. Nothing asserts this text, so the wording change was caught by review rather than by a test; that gap is already on the review as an open item. docs/manual-testing.md told a tester that removing the driver while keeping `vulkaninfo` yields confirmed-absence wording. The feature file records, as a measured fact, that such a host lands at `VulkanUnconfirmed` instead. The walkthrough now says so, and says that seeing "could not confirm" there is correct rather than a regression. Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 13 +++++++--- docs/manual-testing.md | 25 +++++++++++++------ tests/e2e-cucumber/tests/e2e/examine_steps.rs | 22 ++++++++++++++++ 3 files changed, 49 insertions(+), 11 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 1a50180c9c..6c12d53589 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -1095,11 +1095,18 @@ const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str // `append_examine_engine_inventory`, and told its reader to run the // command they were already reading. It names the thing that would // actually settle the question instead. + // + // That thing is NOT "install `vulkaninfo`", for the reason the matching + // arm in `host_gpu::render` spells out: on a real driverless Linux host + // the tool is already installed, prints "Found no drivers!" and exits 1, + // which is exactly what lands here. Both strings appear in the same + // `rocm examine` report, so advising that install would contradict the + // `rocm_support:` line a few lines above it. rocm_core::GpuSupportLevel::VulkanUnconfirmed => { "default embedded Lemonade server; this GPU has no ROCm/HIP support, and whether a \ - Vulkan driver is present could not be determined - install `vulkaninfo` to check, \ - or a Vulkan driver (Mesa RADV on Linux, the AMD display driver on Windows) to run \ - models on this GPU" + Vulkan driver is present could not be determined - install a Vulkan driver (Mesa \ + RADV on Linux, the AMD display driver on Windows) to run models on this GPU, and \ + run `vulkaninfo` yourself to see what it says" } rocm_core::GpuSupportLevel::NoGpuBackend => { "default embedded Lemonade server; this GPU has no ROCm/HIP support and no Vulkan driver was found, so no GPU backend can run here" diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 75ac7ac132..375fca56de 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -307,14 +307,23 @@ checks, which cannot be made anywhere else: `rocm serve --engine lemonade` — the point of the claim is that it is true, and this is the only place it can be confirmed against real hardware. - - With the driver removed but `vulkaninfo` still installed, the line says no - Vulkan driver was found and tells you to install one. It must **not** still - name a driver. - - With neither installed, the line says the check could not be made and names - `vulkaninfo` as the way to settle it. It must **not** say the driver is - absent — that is the Windows-laptop case ROCMAI-550 came from, and - declaring absence there is the regression `VulkanUnconfirmed` exists to - prevent. + - With the driver removed but `vulkaninfo` still installed, expect the + *could-not-confirm* wording on Linux rather than a confirmed absence, and it + must **not** still name a driver. This is the known gap recorded on + examine-53, measured rather than assumed: a real `vulkaninfo` with no + reachable driver prints "Found no drivers!" and exits **1**, and the probe + counts only an exit-0 run as having answered, so the host is classed + inconclusive. Telling "no drivers" apart from "the tool is broken" needs the + loader's own message, which the shared capture discards. Seeing "could not + confirm" here is correct, not a regression. + - With neither installed, the line says the check could not be made — the same + level as the bullet above — and tells you to install a Vulkan driver, while + pointing at running `vulkaninfo` yourself to see what it reports. It must + **not** say the driver is absent, and it must **not** tell you to *install* + `vulkaninfo`: on the host above, that tool is already there, so that advice + could never clear the state. The absence claim is the Windows-laptop case + ROCMAI-550 came from, and declaring absence there is the regression + `VulkanUnconfirmed` exists to prevent. A software rasteriser (`lavapipe`, `swiftshader`) must never be reported as a GPU path in any of the three. diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 1697fd318b..8723a8443a 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -3350,6 +3350,28 @@ async fn assert_support_level_advice_fits(world: &mut E2eWorld) { "rocm serve ", &["rocm install sdk", "--engine lemonade"], ), + // The two probed Vulkan tiers land here too, and neither owes `--engine + // lemonade`: on both, nothing has established a backend that can serve + // this GPU, so naming an engine would promise a run the evidence does + // not support. A driver is what clears either condition. + // + // Split out because the catch-all below demands `--engine lemonade`, + // which neither tier's text contains. No lane reaches that assertion + // today only because none carries a detected, ROCm-unsupported GPU in + // these tiers; dropping these arms again makes the first such runner + // red rather than failing here. This arm has been lost to a merge once + // already. + // + // Told apart by each tier's own wording, as the catalog arm above is: + // `VulkanUnconfirmed` says the probe could not settle it, `NoGpuBackend` + // says it looked and found nothing. + (Some(_), None, _) if line.contains("could not confirm") => ( + "install a Vulkan driver", + &["rocm install sdk", "--engine lemonade"], + ), + (Some(_), None, _) if line.contains("no Vulkan GPU driver was found") => { + ("install one", &["rocm install sdk", "--engine lemonade"]) + } (Some(_), None, _) => ( "rocm serve --engine lemonade", &["rocm install sdk"], From 816ac1d5897de78e9d584bcf41ad4d544ea9fda7 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 13 Nov 2026 15:02:33 +0200 Subject: [PATCH 37/59] fix(serve): skip the device probe for a service that was already running MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The summary's device probe was gated on `report.status == "ready"` alone, under a comment claiming it used "the same gate as the smoke test". It did not: the smoke test beside it, and the Anthropic-endpoint wait below it, both also require `!report.already_running`. A reused service is `ready` by definition, so every repeat `rocm serve` of a live model opened the probe and paid its full retry budget plus one in-flight probe — about 14s with Lemonade's 8s inference probe — sequentially, ahead of the Anthropic-endpoint wait, for a row the command cannot learn anything new for. This is the traced cause of the five Anthropic scenarios failing on the Strix Halo Windows lane, all of which reuse a service from their precondition. The gate moves into `serve_device_probe_opens` with the probe injected as a closure, so a unit test can assert the probe is not merely discarded on the reuse path but never called — the wall time is the regression, not the row. Signed-off-by: Roman Sirokov --- apps/rocm/src/serve_cmd.rs | 96 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 5 deletions(-) diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 7e95e78e18..86da2195e7 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -456,6 +456,32 @@ fn reuse_pregate_opens( .flatten() } +/// Whether the deployment summary may spend a device probe, given what the +/// launch report says happened. +/// +/// `!already_running` is the load-bearing half, not a cosmetic echo of the +/// smoke test's gate. A service that was already running is `"ready"` by +/// definition, so keying on the status alone opens this on *every* repeat +/// `rocm serve` of a live model — and the probe is not free: on a miss it costs +/// `engine_ipc::SERVE_DEVICE_PROBE_BUDGET` plus one in-flight probe, about 14s +/// with Lemonade's 8s inference probe, paid sequentially before the summary's +/// remaining waits. That is a long stall bought for a row this command cannot +/// learn anything new for — the engine answered the same thing when the service +/// was launched, and a *running* service's device is what `rocm services list` +/// reads, with its own budget for exactly that. +/// +/// `probe` is a closure rather than a value for that reason: on the reuse path +/// the cost must not be paid at all, which is what the unit test pins. +fn serve_device_probe_opens( + already_running: bool, + status: &str, + probe: impl FnOnce() -> Option, +) -> Option { + (!already_running && status == "ready") + .then(probe) + .flatten() +} + /// The no-usable-GPU bail's host precondition, read from the visibility-resolved /// set of usable AMD GPU ordinals. /// @@ -1251,8 +1277,11 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // Read what the engine actually selected rather than guessing from // the pre-launch GPU plan: `gpu_indices` above can be empty for a // Vulkan-only GPU that ROCm/amd-smi never enumerates, even though - // the server is running fine on it. Only meaningful once the - // server is ready to answer, same gate as the smoke test. + // the server is running fine on it. + // + // Only for a launch this command performed — see + // [`serve_device_probe_opens`] for why skipping the reuse path is + // the point rather than tidiness. // // Asked more than once, briefly: `engines/lemonade` answers // `"unknown"` until its own inference probe has come back, which is @@ -1260,9 +1289,9 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // bounded (`engine_ipc::SERVE_DEVICE_PROBE_BUDGET`) and `None` is a // supported outcome — the summary then has no `device:` row, and // `rocm services list` shows it once the engine can say. - let device = (report.status == "ready") - .then(|| fetch_service_device_briefly(&paths, &selected_engine, &report.service_id)) - .flatten(); + let device = serve_device_probe_opens(report.already_running, &report.status, || { + fetch_service_device_briefly(&paths, &selected_engine, &report.service_id) + }); let mut notes = collect_serve_notes( cpu_only, @@ -1493,6 +1522,63 @@ mod tests { ); } + /// Regression coverage for the device probe's gate (ROCMAI-550 review). + /// + /// The gate shipped as `report.status == "ready"` alone, under a comment + /// claiming it was "the same gate as the smoke test" — which it was not: the + /// smoke test and the Anthropic-endpoint wait beside it both also require + /// `!already_running`. A reused service is `"ready"` by definition, so the + /// omission made every repeat `rocm serve` of a live model pay the probe's + /// full retry budget (plus one in-flight probe, ~14s with Lemonade) before + /// printing a summary of a launch that did not happen. + /// + /// The probe is asserted *not consulted*, not merely that the row is absent: + /// the cost is the bug, and a gate that probed and then discarded the answer + /// would satisfy a value-only assertion. + #[test] + fn the_device_probe_is_not_spent_on_a_service_that_was_already_running() { + let mut probed = false; + assert!( + serve_device_probe_opens(true, "ready", || { + probed = true; + Some("gpu") + }) + .is_none(), + "a reused service must not open the device probe: it is `ready` by \ + definition, so the budget would be spent on every repeat `serve`" + ); + assert!( + !probed, + "the probe must not even be called on the reuse path — the wall time \ + is the regression, not the `device:` row" + ); + + // A launch that is not ready has nothing to ask the engine about yet. + let mut probed_starting = false; + assert!( + serve_device_probe_opens(false, "starting", || { + probed_starting = true; + Some("gpu") + }) + .is_none() + ); + assert!(!probed_starting); + + // The row itself still works for the case it was added for: a fresh + // launch that reached ready, where the engine knows what it selected. + assert_eq!( + serve_device_probe_opens(false, "ready", || Some("gpu")), + Some("gpu"), + "a fresh ready launch must still read the engine's device, or the \ + Vulkan-only `device:` row this PR adds is gone" + ); + // `None` from the engine stays a supported answer: no row, no error. + assert_eq!( + serve_device_probe_opens(false, "ready", || Option::<&str>::None), + None + ); + } + #[test] fn serve_engine_selection_uses_shared_recipe_when_no_override_exists() { let recipe = resolve_builtin_model_recipe("qwen32b").expect("qwen32b recipe"); From aaade2d6548264e61ca5b600a86c21fe7fd10dfa Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 13 Nov 2026 16:22:32 +0200 Subject: [PATCH 38/59] fix(examine): tie the unverified-Vulkan advice to whether vulkaninfo actually answered The final `devices.is_empty()` branch of `classify_vulkan_support` built its detail from the device list alone and never consulted `enumerated` -- the one piece of evidence the rest of the module treats as central. A host with a hardware ICD on disk whose `vulkaninfo --summary` had already run, exited 0 and listed nothing was still told to "install `vulkaninfo` to confirm": advice to install a tool it already has, and which has already given the answer. The common cause there is a permission problem, not a missing tool, so the message now says what was established and points at the `render` group instead. An earlier round on this PR deliberately declined to change this string, reasoning that the driver-present-devices-unverified case is one where the tool may genuinely be absent. That reasoning only holds for `enumerated == false`. Both cases now get their own sentence, and the module's own test asserts the two differ rather than asserting either in isolation. The two `GpuSupportLevel` variants this PR added shipped with no test on either renderer, which is how the engine note was able to contradict the `rocm_support:` line beside it in an earlier round without turning anything red. Both renderers now have a case for each variant, each asserting the *other* variant's wording is absent, so one arm doing duty for both cannot pass. Also drops the MCP `engines` tool from the comment listing the callers that reach these arms: it goes through `host_gpu_support_level`, which always passes `VulkanStatus::Unknown`, so it never can. Signed-off-by: Roman Sirokov Signed-off-by: Roman Sirokov --- apps/rocm/src/examine_report_text.rs | 99 +++++++++++++++++++++++++ apps/rocm/src/main.rs | 20 +++-- crates/rocm-core/src/host_gpu/vulkan.rs | 91 ++++++++++++++++++----- 3 files changed, 182 insertions(+), 28 deletions(-) diff --git a/apps/rocm/src/examine_report_text.rs b/apps/rocm/src/examine_report_text.rs index 20aef803e3..67268d07b4 100644 --- a/apps/rocm/src/examine_report_text.rs +++ b/apps/rocm/src/examine_report_text.rs @@ -1364,6 +1364,105 @@ mod tests { let _ = fs::remove_dir_all(root); } + /// The two support levels this change added carry new user-facing wording + /// on both renderers, and nothing exercised either of them. They are each + /// other's nearest neighbour -- both say "no ROCm/HIP" -- so each case + /// asserts what the *other* one would have said is absent; otherwise a + /// single arm doing duty for both would still pass. + #[test] + fn examine_engine_inventory_separates_an_unconfirmed_vulkan_from_no_backend_at_all() { + let (root, paths) = test_paths("examine-engine-inventory-vulkan-tiers"); + let config = RocmCliConfig::default(); + + let mut unconfirmed = String::new(); + append_examine_engine_inventory( + &mut unconfirmed, + &paths, + &config, + "lemonade", + rocm_core::GpuSupportLevel::VulkanUnconfirmed, + ); + assert!( + unconfirmed.contains("could not be determined"), + "an unprobed Vulkan path must read as unknown, not as a verdict:\n{unconfirmed}" + ); + assert!( + !unconfirmed.contains("no Vulkan driver was found"), + "`VulkanUnconfirmed` must not borrow `NoGpuBackend`'s confirmed absence:\n{unconfirmed}" + ); + assert!( + !unconfirmed.contains("so it runs on the Vulkan llama.cpp backend"), + "nor may it promise the backend `VulkanOnly` promises:\n{unconfirmed}" + ); + + let mut none = String::new(); + append_examine_engine_inventory( + &mut none, + &paths, + &config, + "lemonade", + rocm_core::GpuSupportLevel::NoGpuBackend, + ); + assert!( + none.contains("no Vulkan driver was found") + && none.contains("no GPU backend can run here"), + "a confirmed absence must say so plainly:\n{none}" + ); + assert!( + !none.contains("could not be determined"), + "`NoGpuBackend` is the probed answer, not the missing one:\n{none}" + ); + + assert_ne!( + unconfirmed, none, + "'we could not tell' and 'we looked and there is none' are different facts" + ); + let _ = fs::remove_dir_all(root); + } + + /// `rocm engines` renders the same two notes through a different renderer, + /// so it needs the same pair -- this is the counterpart to + /// `examine_engine_inventory_separates_an_unconfirmed_vulkan_from_no_backend_at_all`. + #[test] + fn engines_inventory_separates_an_unconfirmed_vulkan_from_no_backend_at_all() { + let (root, paths) = test_paths("engines-inventory-vulkan-tiers"); + + let unconfirmed = render_engine_inventory_text_for_host( + Some(&paths), + "lemonade", + rocm_core::GpuSupportLevel::VulkanUnconfirmed, + ); + assert!( + unconfirmed.contains("could not be determined") + && unconfirmed.contains("no ROCm/HIP support"), + "`rocm engines` must report the same unprobed Vulkan state examine does:\n{unconfirmed}" + ); + assert!( + !unconfirmed.contains("Lemonade server with ROCm llama.cpp backend"), + "a host with no ROCm build must not be told it has one:\n{unconfirmed}" + ); + + let none = render_engine_inventory_text_for_host( + Some(&paths), + "lemonade", + rocm_core::GpuSupportLevel::NoGpuBackend, + ); + assert!( + none.contains("no GPU backend can run here"), + "`rocm engines` must not keep promising a backend that was ruled out:\n{none}" + ); + assert!( + !none.contains("could not be determined"), + "the probed answer must not be softened into the unprobed one:\n{none}" + ); + + assert_ne!( + unconfirmed, none, + "the two levels exist precisely because they are not the same claim" + ); + let _ = fs::remove_dir_all(root); + } + /// `rocm engines` makes the same claim `rocm examine` does, through a /// different renderer, so it needs the same pair of tests: /// `examine_engine_inventory_names_vulkan_for_a_gpu_with_no_rocm_support` diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 6c12d53589..e8a0df1572 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -1087,14 +1087,18 @@ const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str // // Reached by every caller that passes a real `VulkanStatus`, which is // `rocm examine` *and* `build_codex_bridge_snapshot` (the - // `InternalBridgeSnapshot` command and the MCP `bridge_snapshot`/ - // `engines` tools), which builds the level from `examine.vulkan_status()` - // rather than the cheap `host_gpu_support_level`. So the advice cannot - // be "run `rocm examine`": this very note is rendered *inside* that - // command's own output, through `examine_human_report` -> - // `append_examine_engine_inventory`, and told its reader to run the - // command they were already reading. It names the thing that would - // actually settle the question instead. + // `InternalBridgeSnapshot` command and the MCP `bridge_snapshot` tool), + // which builds the level from `examine.vulkan_status()` rather than the + // cheap `host_gpu_support_level`. The MCP `engines` tool is deliberately + // NOT in that list: it goes through `host_gpu_support_level`, which + // always passes `VulkanStatus::Unknown`, so it can never land on these + // two arms. + // + // So the advice cannot be "run `rocm examine`": this very note is + // rendered *inside* that command's own output, through + // `examine_human_report` -> `append_examine_engine_inventory`, and told + // its reader to run the command they were already reading. It names the + // thing that would actually settle the question instead. // // That thing is NOT "install `vulkaninfo`", for the reason the matching // arm in `host_gpu::render` spells out: on a real driverless Linux host diff --git a/crates/rocm-core/src/host_gpu/vulkan.rs b/crates/rocm-core/src/host_gpu/vulkan.rs index 5e36ef5eba..742bc64080 100644 --- a/crates/rocm-core/src/host_gpu/vulkan.rs +++ b/crates/rocm-core/src/host_gpu/vulkan.rs @@ -51,9 +51,14 @@ const LINUX_ICD_DIRS: [&str; 4] = [ pub enum VulkanStatus { /// A Vulkan device answered and it is a real GPU. Available, - /// A driver is installed, but no device was enumerated -- `vulkaninfo` is - /// not on PATH, so nothing confirmed a device answers. Likely to work; - /// not established. + /// A driver is installed, but no AMD device was confirmed behind it. + /// Likely to work; not established. + /// + /// Three hosts land here and they are not owed the same sentence, which is + /// why `classify_vulkan_support` builds the detail from `devices` and + /// `enumerated` rather than from the status alone: `vulkaninfo` was not on + /// PATH to ask; it ran and listed nothing; or it ran and listed only + /// non-AMD hardware. DriverPresentDevicesUnverified, /// The probe looked and found no GPU Vulkan driver. A software rasteriser /// alone lands here, not in [`Self::Available`]. @@ -490,15 +495,30 @@ pub fn classify_vulkan_support( || "a Vulkan driver is installed".to_owned(), |label| format!("{label} is installed"), ); - // Two different hosts reach this status and they are not owed the same - // sentence. An empty `devices` really is "nothing answered". A non-empty one - // means `vulkaninfo` answered and the hardware it listed simply was not AMD - // — saying no device was enumerated there is false, and it is the shape an - // NVIDIA box with an AMD iGPU disabled lands in. - let detail = if devices.is_empty() { - format!("{named}, but no device was enumerated (install `vulkaninfo` to confirm)") - } else { + // Three different hosts reach this status and they are not owed the same + // sentence. A non-empty `devices` means `vulkaninfo` answered and the + // hardware it listed simply was not AMD — saying no device was enumerated + // there is false, and it is the shape an NVIDIA box with an AMD iGPU + // disabled lands in. + // + // An empty `devices` splits on `enumerated`, the same evidence the + // `gpu_icds.is_empty()` arm above already branches on. `enumerated == true` + // means `vulkaninfo` ran, exited 0 and listed nothing — the answer is in + // hand, so telling that user to "install `vulkaninfo` to confirm" sends + // them after a tool that is already installed and has already answered + // (the common cause is a permission problem, e.g. not being in the + // `render` group, not a missing tool). Only `enumerated == false` is the + // case where the tool is genuinely missing or failed to run, and only + // there is that advice true. + let detail = if !devices.is_empty() { format!("{named}, but none of the devices it enumerated is an AMD GPU") + } else if enumerated { + format!( + "{named}, but `vulkaninfo` ran and listed no device, so nothing confirms it can \ + reach a GPU here (on Linux, check that your user is in the `render` group)" + ) + } else { + format!("{named}, but no device was enumerated (install `vulkaninfo` to confirm)") }; VulkanSupport { status: VulkanStatus::DriverPresentDevicesUnverified, @@ -632,27 +652,58 @@ GPU0: assert!(icd_is_gpu_driver("radeon_icd.x86_64.json")); } + /// Both halves of this test reach the same *status*; what they must not + /// share is the *sentence*. `enumerated` is the module's central piece of + /// evidence everywhere else, and this arm used to ignore it: a host whose + /// `vulkaninfo` had already run and already listed nothing was still told + /// to "install `vulkaninfo` to confirm". Asserting the two texts differ, + /// rather than asserting each in isolation, is what keeps the advice tied + /// to the evidence. #[test] fn an_installed_driver_with_no_enumerated_device_stays_unverified() { - let support = classify_vulkan_support( + let asked = classify_vulkan_support( vec!["radeon_icd.x86_64.json".to_owned()], Vec::new(), true, true, ); - assert_eq!(support.status, VulkanStatus::DriverPresentDevicesUnverified); + assert_eq!(asked.status, VulkanStatus::DriverPresentDevicesUnverified); assert!( - support.status.is_usable(), + asked.status.is_usable(), "an installed GPU driver is still worth routing to" ); - assert_eq!(support.driver_label().as_deref(), Some("Mesa RADV")); + assert_eq!(asked.driver_label().as_deref(), Some("Mesa RADV")); + let asked_detail = asked.detail.clone().expect("a detail is always set here"); assert!( - support - .detail - .as_deref() - .is_some_and(|detail| detail.contains("no device was enumerated")), - "the gap must be stated, not papered over: {support:?}" + asked_detail.contains("listed no device"), + "the gap must be stated, not papered over: {asked:?}" + ); + assert!( + !asked_detail.contains("install `vulkaninfo`"), + "`vulkaninfo` already ran and already answered; telling this user to install it \ + sends them after a tool they have: {asked:?}" + ); + + // Same ICD, same (empty) device list, but nothing asked: here the tool + // really may be missing, so the install advice is the true one. + let unasked = classify_vulkan_support( + vec!["radeon_icd.x86_64.json".to_owned()], + Vec::new(), + true, + false, + ); + + assert_eq!(unasked.status, VulkanStatus::DriverPresentDevicesUnverified); + let unasked_detail = unasked.detail.clone().expect("a detail is always set here"); + assert!( + unasked_detail.contains("install `vulkaninfo` to confirm"), + "with nothing having looked, the tool is what would settle it: {unasked:?}" + ); + assert_ne!( + asked_detail, unasked_detail, + "`enumerated` is the difference between 'nothing looked' and 'something looked \ + and found nothing'; the two must not be told the same thing" ); } From 604faa30ca5d5ab050aa89b206187ccb79e0bdeb Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 13 Nov 2026 17:23:28 +0200 Subject: [PATCH 39/59] fix(engines): stop `rocm engines` requiring ROCm on a host it says has none The `rocm engines` banner was the constant "ROCm GPU execution is required.", printed a few rows above the per-engine notes. That was consistent while lemonade's note carried the same static ROCm-only sentence; it stopped being consistent as soon as this branch made that note host-aware, so a Vulkan-only host read "ROCm GPU execution is required" directly over "this GPU has no ROCm/HIP support" - the stale claim ROCMAI-550 exists to remove, reinstated on the same screen. What is invariant across every support level is the refusal, not the backend: these engines serve on a GPU and never fall back to the CPU, and Vulkan is one of the GPU paths rather than a fallback from one. So the line now says "GPU execution is required" at every level and adds the ROCm half only where the classifier establishes it. `Unknown` drops it too, which is the point of routing the whole line through `GpuSupportLevel` instead of special-casing `VulkanOnly`: asserting ROCm is required on a host whose support level is unknown is the same unfounded claim, just on the hosts the CLI knows least about. Also assert the Vulkan-only remediation command. `rocm_support_line` recommends `rocm serve --engine lemonade`, and the only test over that arm pinned the pre-existing "the Vulkan backend can still be used" substring, so the sentence could be deleted and the suite stayed green - which AGENTS.md section 3 forbids for advice naming a command. Centralise `first_gpu_index_from_state` into `rocm-engine-protocol` beside `format_reported_device`: lemonade and vllm held byte-identical private copies of the same `gpu_indices` read, and the two engines have to agree on which ordinal they report, not only on how it is spelled. No new crate edge - both already depend on the protocol crate. Fix the `bridge_snapshot` intra-doc link, which named a `builtin_engine_inventory` that does not exist. Signed-off-by: Roman Sirokov --- apps/rocm/src/bridge_snapshot.rs | 2 +- apps/rocm/src/examine_report_text.rs | 99 ++++++++++++++++++++++++- crates/rocm-core/src/host_gpu/render.rs | 18 +++++ crates/rocm-engine-protocol/src/lib.rs | 68 +++++++++++++++++ engines/lemonade/src/device_policy.rs | 18 ++--- engines/vllm/src/state.rs | 17 ++--- 6 files changed, 196 insertions(+), 26 deletions(-) diff --git a/apps/rocm/src/bridge_snapshot.rs b/apps/rocm/src/bridge_snapshot.rs index 741c18ede7..a5c7b648b3 100644 --- a/apps/rocm/src/bridge_snapshot.rs +++ b/apps/rocm/src/bridge_snapshot.rs @@ -73,7 +73,7 @@ pub(crate) fn build_codex_bridge_gpu_snapshot(config: &RocmCliConfig) -> CodexBr } /// The engine catalogue in the codex-bridge snapshot and the bridge's `engines` -/// tool, built from [`builtin_engine_inventory`] with lemonade's note replaced +/// tool, built from [`engine_inventory`] with lemonade's note replaced /// by `lemonade_engine_note` for the given support level. /// /// It takes the level rather than reading the static row because the stale diff --git a/apps/rocm/src/examine_report_text.rs b/apps/rocm/src/examine_report_text.rs index 12d66bee2b..126d433479 100644 --- a/apps/rocm/src/examine_report_text.rs +++ b/apps/rocm/src/examine_report_text.rs @@ -249,7 +249,7 @@ fn render_engine_inventory_text_for_host( output, " Built-in engines are included with rocm-cli. External plugins are optional." ); - let _ = writeln!(output, " ROCm GPU execution is required."); + let _ = writeln!(output, "{}", gpu_execution_line(gpu_support_level)); if let Some(paths) = paths { let _ = writeln!(output, " Plugin folders:"); for (index, path) in engine_plugin_dirs(paths).iter().enumerate() { @@ -276,6 +276,43 @@ fn render_engine_inventory_text_for_host( output } +/// The `rocm engines` banner stating what these engines will and will not run +/// on, for a host at `level`. +/// +/// This line used to be the constant `" ROCm GPU execution is required."`, +/// printed a few rows above the per-engine notes. That was consistent while +/// lemonade's note was the same static ROCm-only sentence, and stopped being +/// consistent the moment [`crate::engine_inventory_note`] made that note +/// host-aware: on a `VulkanOnly` host the banner asserted ROCm was required +/// directly above a row saying this GPU has no ROCm support — reinstating, on +/// the same screen, the stale claim ROCMAI-550 exists to remove. +/// +/// What is actually invariant across every level is the *refusal*, not the +/// backend: rocm-cli serves on a GPU and never silently falls back to CPU, and +/// Vulkan is one of the GPU paths it serves on rather than a fallback from +/// one. So "GPU execution is required" is said at every level and the ROCm +/// half is said only where the classifier establishes it. `Unknown` drops it +/// too, and that is the point of preferring this over a `VulkanOnly`-only +/// special case: "ROCm is required" on a host whose support level is unknown +/// is the same unfounded claim, just on the hosts the CLI knows least about. +/// `Unknown` does not re-name `rocm examine` either — lemonade's own row +/// already does, two lines below. +/// +/// Matched exhaustively, like the notes it has to agree with, so a fourth +/// support level is a compile error here instead of silently inheriting a +/// claim about ROCm. +const fn gpu_execution_line(level: rocm_core::GpuSupportLevel) -> &'static str { + match level { + rocm_core::GpuSupportLevel::RocmHip => " ROCm GPU execution is required.", + rocm_core::GpuSupportLevel::VulkanOnly => { + " GPU execution is required; on this GPU that is the Vulkan backend, not ROCm." + } + rocm_core::GpuSupportLevel::Unknown => { + " GPU execution is required; CPU execution is not supported." + } + } +} + pub(crate) fn append_examine_runtime_state( output: &mut String, paths: &AppPaths, @@ -731,7 +768,13 @@ mod tests { assert!(rendered.contains("Local model engines")); assert!(rendered.contains("Built-in engines are included with rocm-cli")); - assert!(rendered.contains("ROCm GPU execution is required")); + // Not `"ROCm GPU execution is required"`: this renderer probes the real + // host, so the ROCm half of that sentence is now whatever this runner's + // `GpuSupportLevel` is (see `gpu_execution_line`). The part that holds + // on every runner — and the part this test is about — is that engines + // need a GPU at all. Which backend is claimed for which level is pinned + // by `engines_banner_and_engine_note_cannot_contradict_each_other`. + assert!(rendered.contains("GPU execution is required")); assert!(rendered.contains("Plugin folders:")); assert!(rendered.contains(&paths.primary_engine_plugin_dir().display().to_string())); } @@ -1437,4 +1480,56 @@ mod tests { ); let _ = fs::remove_dir_all(root); } + + /// `rocm engines` must not assert and deny ROCm support on one screen. + /// + /// The three tests above each check one half of the page — the per-engine + /// note — and would all pass with the banner above it flatly contradicting + /// them, which is exactly the state this test was added for: the banner was + /// the constant `"ROCm GPU execution is required."` printed a few lines + /// over "this GPU has no ROCm/HIP support". + /// + /// Stated as a relation between the two lines rather than as expected text + /// per level, so it keeps holding if either sentence is reworded: the claim + /// under test is that the banner requires ROCm on exactly the hosts whose + /// row says they have it. That covers every level and not just the + /// `VulkanOnly` one — `Unknown` must not be told ROCm is required either, + /// when its own row says the support level is not yet known. + #[test] + fn engines_banner_and_engine_note_cannot_contradict_each_other() { + let (root, paths) = test_paths("engines-banner-vs-note"); + + for level in [ + rocm_core::GpuSupportLevel::RocmHip, + rocm_core::GpuSupportLevel::VulkanOnly, + rocm_core::GpuSupportLevel::Unknown, + ] { + let output = render_engine_inventory_text_for_host(Some(&paths), "lemonade", level); + + // The invariant that survives every level: these engines serve on a + // GPU, never on the CPU. Dropping the banner entirely would make the + // contradiction assertions below vacuously true. + assert!( + output.contains("GPU execution is required"), + "{level:?}: the GPU requirement must still be stated:\n{output}" + ); + + let banner_requires_rocm = output.contains("ROCm GPU execution is required"); + let note_claims_rocm = + output.contains("default embedded Lemonade server with ROCm llama.cpp backend"); + assert_eq!( + banner_requires_rocm, + matches!(level, rocm_core::GpuSupportLevel::RocmHip), + "{level:?}: the banner may require ROCm only where the classifier \ + establishes it:\n{output}" + ); + assert_eq!( + banner_requires_rocm, note_claims_rocm, + "{level:?}: the banner and lemonade's row must make the same ROCm claim \ + about the same host:\n{output}" + ); + } + + let _ = fs::remove_dir_all(root); + } } diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index 89b0e85595..dc732c4fa2 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -362,6 +362,24 @@ mod tests { rendered.contains("the Vulkan backend can still be used"), "{rendered}" ); + // AGENTS.md §3: advice naming a command is asserted together with the + // behaviour. Saying the Vulkan backend "can still be used" without + // naming how to use it leaves the user where they started, so the + // command is part of the claim, not decoration — without this + // assertion the recommendation could be deleted or garbled and this + // test would still pass. + // + // That the named command *parses* (the subcommand exists and accepts + // `--engine lemonade`) is proved mechanically rather than here: + // `advised_commands::every_advised_command_parses` in `apps/rocm` + // scans production string literals under `crates/`, including this + // one, and routes each through the real clap tree. rocm-core cannot + // see the CLI's argument parser, so this half pins that the sentence + // is printed at all for a Vulkan-only host. + assert!( + rendered.contains("serve on it with `rocm serve --engine lemonade`"), + "a Vulkan-only host must be told the command that serves on it:\n{rendered}" + ); } /// An unmapped target must not be *claimed* unsupported, even though every diff --git a/crates/rocm-engine-protocol/src/lib.rs b/crates/rocm-engine-protocol/src/lib.rs index e9179b9698..35df37bcbe 100644 --- a/crates/rocm-engine-protocol/src/lib.rs +++ b/crates/rocm-engine-protocol/src/lib.rs @@ -631,6 +631,35 @@ pub fn format_reported_device( device } +/// The first pinned GPU ordinal recorded in a service state file, if any. +/// +/// The ordinal half of what [`format_reported_device`] renders, read from the +/// state file every engine writes at launch. It lives beside that function for +/// the same reason that one does: `rocm serve`'s `device:` row and +/// `rocm services list` print whichever engine answered verbatim, so the two +/// engines that pin ordinals (`engines/lemonade`, `engines/vllm`) have to agree +/// on *which* ordinal they report, not only on how it is spelled. They had +/// byte-identical private copies of this, which is one keystroke away from +/// disagreeing silently — the same duplication `format_reported_device` was +/// centralized to end. +/// +/// `None` covers every shape that is not a usable ordinal: no `gpu_indices` +/// key, a non-array value, an empty list (a launch that pinned nothing, which +/// is the default on Windows — see `format_reported_device`), a non-integer +/// entry, and a value too large for `u32`. Those are "no ordinal to report", +/// not errors: state files are written by engine processes of possibly +/// different versions, and a row that says less is better than one that +/// invents an ordinal or refuses to render. +#[must_use] +pub fn first_gpu_index_from_state(state: &Value) -> Option { + state + .get("gpu_indices") + .and_then(Value::as_array) + .and_then(|indices| indices.first()) + .and_then(Value::as_u64) + .and_then(|value| u32::try_from(value).ok()) +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct HealthcheckResponse { /// Engine-reported service state. `ready` is reserved for a service that has @@ -760,6 +789,45 @@ mod tests { ); } + /// Every shape a state file can carry in `gpu_indices` resolves to "an + /// ordinal" or "no ordinal", never to a panic or an invented value. + /// + /// This used to be two byte-identical private helpers in `engines/lemonade` + /// and `engines/vllm`, neither of which tested the shapes below; only the + /// happy path was reachable from either engine's own tests. Centralizing it + /// is what makes the edge cases testable in one place for both callers. + #[test] + fn the_first_pinned_ordinal_is_read_and_every_other_shape_is_absent() { + use serde_json::json; + + assert_eq!( + super::first_gpu_index_from_state(&json!({"gpu_indices": [1, 2]})), + Some(1), + "the first pinned ordinal is the one reported" + ); + // A launch that pinned nothing — the default on Windows, which is the + // host ROCMAI-550 was filed from. + assert_eq!( + super::first_gpu_index_from_state(&json!({"gpu_indices": []})), + None + ); + assert_eq!(super::first_gpu_index_from_state(&json!({})), None); + // A state file written by some other version of an engine must degrade + // to "no ordinal", not to a wrong one. + assert_eq!( + super::first_gpu_index_from_state(&json!({"gpu_indices": "0"})), + None + ); + assert_eq!( + super::first_gpu_index_from_state(&json!({"gpu_indices": [-1]})), + None + ); + assert_eq!( + super::first_gpu_index_from_state(&json!({"gpu_indices": [u64::from(u32::MAX) + 1]})), + None + ); + } + fn resolve_response(served: Option<&str>) -> ResolveModelResponse { ResolveModelResponse { canonical_model_id: "image:tag".to_owned(), diff --git a/engines/lemonade/src/device_policy.rs b/engines/lemonade/src/device_policy.rs index 11a68d70bc..06421794da 100644 --- a/engines/lemonade/src/device_policy.rs +++ b/engines/lemonade/src/device_policy.rs @@ -146,11 +146,15 @@ pub(crate) const fn device_policy_name(policy: &DevicePolicy) -> &'static str { /// /// Formatted by [`rocm_engine_protocol::format_reported_device`] so all three /// engines render one shape; see that function for why the ordinal-less row -/// drops the `gpu` noun rather than printing a bare `vulkan gpu`. +/// drops the `gpu` noun rather than printing a bare `vulkan gpu`. The ordinal +/// is read by [`rocm_engine_protocol::first_gpu_index_from_state`] for the same +/// reason — `engines/vllm` reads the same `gpu_indices` key out of a state file +/// of the same shape, and kept a byte-identical copy of that read until it +/// moved next to the formatter. pub(crate) fn reported_device(state: Option<&Value>, backend: &str) -> String { rocm_engine_protocol::format_reported_device( backend, - state.and_then(first_gpu_index_from_state), + state.and_then(rocm_engine_protocol::first_gpu_index_from_state), state.and_then(gpu_name_from_state).as_deref(), ) } @@ -165,16 +169,6 @@ fn gpu_name_from_state(state: &Value) -> Option { crate::state::value_string(state, "gpu_name") } -/// The first pinned GPU ordinal recorded in service state, if any. -fn first_gpu_index_from_state(state: &Value) -> Option { - state - .get("gpu_indices") - .and_then(Value::as_array) - .and_then(|indices| indices.first()) - .and_then(Value::as_u64) - .and_then(|value| u32::try_from(value).ok()) -} - #[cfg(test)] mod tests { use super::*; diff --git a/engines/vllm/src/state.rs b/engines/vllm/src/state.rs index 95f3d8e064..694988c505 100644 --- a/engines/vllm/src/state.rs +++ b/engines/vllm/src/state.rs @@ -85,24 +85,19 @@ pub(crate) fn healthcheck_service(request: HealthcheckRequest) -> Result String { rocm_engine_protocol::format_reported_device( "rocm", - first_gpu_index_from_state(state), + rocm_engine_protocol::first_gpu_index_from_state(state), value_string(state, "gpu_name").as_deref(), ) } -/// The first pinned GPU ordinal recorded in service state, if any. -fn first_gpu_index_from_state(state: &Value) -> Option { - state - .get("gpu_indices") - .and_then(Value::as_array) - .and_then(|indices| indices.first()) - .and_then(Value::as_u64) - .and_then(|value| u32::try_from(value).ok()) -} - pub(crate) fn endpoint_response(request: EndpointRequest) -> Result { require_nonempty(&request.service_id, "service_id")?; let files = service_files(&request.service_id)?; From 0d5fe99e605feb87b43067b57f64c57485a9e676 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 13 Nov 2026 18:53:32 +0200 Subject: [PATCH 40/59] fix(examine): stop a failed ICD lookup reading as a host with no Vulkan driver The Windows ICD probe shelled out to PowerShell and turned `capture_optional_command`'s `None` -- a missing binary, a non-zero exit and a timeout alike -- into the same empty `Vec` an empty Khronos registry key produces, so a flaky PowerShell on a host with a working AMD driver was told "no Vulkan driver was found". That is the conflation `enumerated` already exists to prevent on the `vulkaninfo` side, and the ICD side had no equivalent signal. `detect_icds` now returns an `icds_listed` flag beside the manifests on every platform (always true on Linux, where reading directories that may not exist is still a lookup; false on Windows whenever PowerShell did not answer; false on the stub platform that cannot look at all), and `classify_vulkan_support` requires *both* sources to have answered before an empty result may be reported as a confirmed absence, naming which source went unread. `driver_label`'s ICD fallback runs only once an AMD device has already been confirmed, so whatever it names is read as the driver behind that device -- but it filtered on "not a known software rasteriser", which admits NVIDIA and Intel. `detect_icds` sorts alphabetically on Linux, so a host with both `nvidia_icd.json` and `radeon_icd.x86_64.json` had a confirmed AMD Vulkan path attributed to NVIDIA's driver. The vendor test now lives in one place, `amd_icd_driver_label`, beside the filename table it reads; the same sibling defect in the detail text that justifies routing a host to Vulkan is fixed with it, while a host where another vendor's driver is genuinely all there is still gets named it. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/host_gpu.rs | 6 +- crates/rocm-core/src/host_gpu/vulkan.rs | 419 +++++++++++++++++++++--- docs/architecture.md | 2 +- docs/host-probing.md | 5 +- 4 files changed, 385 insertions(+), 47 deletions(-) diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index 562831b051..96fdbeb2bf 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -75,9 +75,9 @@ pub use linux::is_amdgpu_device; #[cfg(any(target_os = "linux", test))] pub(crate) use linux::{KfdGpuNode, kfd_gpu_nodes_in}; pub use vulkan::{ - VulkanDevice, VulkanStatus, VulkanSupport, classify_vulkan_support, detect_vulkan_support, - icd_driver_label, icd_is_gpu_driver, parse_vulkaninfo_summary, - parse_windows_vulkan_icd_listing, + VulkanDevice, VulkanStatus, VulkanSupport, amd_icd_driver_label, classify_vulkan_support, + detect_vulkan_support, icd_driver_label, icd_is_gpu_driver, parse_vulkaninfo_summary, + parse_windows_vulkan_icd_listing, windows_icds_from_capture, }; pub(crate) use windows_inventory::detect_windows_amd_driver_info; #[cfg(test)] diff --git a/crates/rocm-core/src/host_gpu/vulkan.rs b/crates/rocm-core/src/host_gpu/vulkan.rs index 742bc64080..842652f82e 100644 --- a/crates/rocm-core/src/host_gpu/vulkan.rs +++ b/crates/rocm-core/src/host_gpu/vulkan.rs @@ -63,8 +63,10 @@ pub enum VulkanStatus { /// The probe looked and found no GPU Vulkan driver. A software rasteriser /// alone lands here, not in [`Self::Available`]. Absent, - /// The probe looked and came away with nothing either way: no ICD manifest, - /// no enumerated device, and no `vulkaninfo` to ask. + /// The probe came away with nothing either way: no ICD manifest, no + /// enumerated device, and at least one of the two sources that never + /// answered — `vulkaninfo` missing or failing, or (on Windows) a registry + /// listing that could not be run. /// /// Distinct from [`Self::Unknown`], which is a platform that cannot be /// probed at all. The difference decides what the host is told: an @@ -163,18 +165,24 @@ impl VulkanSupport { .find(|device| device.is_amd() && device.is_hardware()) .and_then(|device| device.driver_name.clone()) .or_else(|| { - // Filtered the same way `classify_vulkan_support` filters its - // own driver naming. Without this, an `Available` host whose AMD - // device omitted the optional `driverName`, on a machine that - // also has lavapipe installed, would be told the software - // rasteriser is the driver behind the GPU path it was just - // promised — contradicting this module's "a software rasteriser - // never counts as a GPU path" invariant in the one place the - // user actually reads a driver name. + // Restricted to AMD-branded manifests, because this fallback + // only ever runs once an AMD hardware device has *already* been + // confirmed: whatever it names is read as the driver behind + // that device. "Not a known software rasteriser" was too weak a + // filter for that job — `detect_icds` sorts alphabetically on + // Linux, so a dual-vendor host carrying both `nvidia_icd.json` + // and `radeon_icd.x86_64.json` had the NVIDIA manifest win the + // `find_map` and a confirmed AMD Vulkan path was attributed to + // NVIDIA's driver. + // + // Narrowing to AMD subsumes the software-rasteriser exclusion + // this replaces (lavapipe and SwiftShader are not AMD labels), + // so the "a software rasteriser never counts as a GPU path" + // invariant still holds in the one place the user actually + // reads a driver name. self.icds .iter() - .filter(|icd| icd_is_gpu_driver(icd)) - .find_map(|icd| icd_driver_label(icd)) + .find_map(|icd| amd_icd_driver_label(icd)) .map(str::to_owned) }) } @@ -213,6 +221,20 @@ pub fn icd_driver_label(icd: &str) -> Option<&'static str> { None } +/// The driver an ICD manifest's file name implies **when that driver is AMD's**, +/// and `None` for every other vendor (software rasterisers included — none of +/// them is AMD). +/// +/// The vendor test lives here, beside the one table that maps a file name to a +/// label, rather than being spelled out at each call site: a caller that must +/// not name another vendor's driver asks this instead of re-deriving which +/// prefixes are AMD's. It reads the label rather than the file name for the same +/// reason [`icd_is_gpu_driver`] does — one table, one place to extend. +#[must_use] +pub fn amd_icd_driver_label(icd: &str) -> Option<&'static str> { + icd_driver_label(icd).filter(|label| matches!(*label, "Mesa RADV" | "AMDVLK")) +} + /// Whether an ICD manifest names a driver that runs on a GPU. /// /// An unrecognised manifest counts as a GPU driver: a vendor the table does not @@ -296,9 +318,16 @@ pub fn parse_windows_vulkan_icd_listing(output: &str) -> Vec { #[cfg(any(windows, test))] pub(super) const WINDOWS_VULKAN_ICD_SCRIPT: &str = r#"foreach ($key in 'HKLM:\SOFTWARE\Khronos\Vulkan\Drivers','HKLM:\SOFTWARE\WOW6432Node\Khronos\Vulkan\Drivers') { if (Test-Path $key) { foreach ($name in (Get-Item $key).GetValueNames()) { "ICD`t$name" } } }"#; -/// Every Vulkan ICD manifest this host's loader would read. +/// Every Vulkan ICD manifest this host's loader would read, and whether the +/// lookup **ran** — the ICD-side counterpart of `enumerated`. +/// +/// On Linux the lookup is a read of a fixed list of manifest directories, and +/// there is no step in it that can fail in a way distinguishable from an honest +/// "nothing is installed here": a directory that does not exist is exactly what +/// a host with no Vulkan driver looks like. So the flag is unconditionally +/// `true` — the lookup always happens — rather than a placeholder. #[cfg(target_os = "linux")] -fn detect_icds() -> Vec { +fn detect_icds() -> (Vec, bool) { let mut icds = Vec::new(); for dir in LINUX_ICD_DIRS { let Ok(entries) = std::fs::read_dir(host_path(dir)) else { @@ -315,12 +344,19 @@ fn detect_icds() -> Vec { } icds.sort(); icds.dedup(); - icds + (icds, true) } +/// On Windows the lookup is a PowerShell read of the Khronos registry keys, and +/// that genuinely can fail: `capture_optional_command` yields `None` for a +/// missing `powershell`, a non-zero exit **and** a timeout alike. Collapsing +/// that `None` into the empty `Vec` an empty registry key produces made a flaky +/// PowerShell on a host with a working AMD driver read identically to a host +/// with no driver at all, which is the conflation `enumerated` exists to prevent +/// on the `vulkaninfo` side. #[cfg(windows)] -fn detect_icds() -> Vec { - capture_optional_command( +fn detect_icds() -> (Vec, bool) { + windows_icds_from_capture(capture_optional_command( "powershell", &[ "-NoLogo", @@ -329,14 +365,32 @@ fn detect_icds() -> Vec { "-Command", WINDOWS_VULKAN_ICD_SCRIPT, ], - ) - .map(|output| parse_windows_vulkan_icd_listing(&output)) - .unwrap_or_default() + )) } +/// Nothing here can be read on a platform that is neither Linux nor Windows, so +/// the flag says the lookup did not run. The classifier never sees it today — +/// `vulkan_is_probeable()` is false there and returns [`VulkanStatus::Unknown`] +/// first — but it must not claim a lookup happened where none exists. #[cfg(not(any(target_os = "linux", windows)))] -const fn detect_icds() -> Vec { - Vec::new() +const fn detect_icds() -> (Vec, bool) { + (Vec::new(), false) +} + +/// The pure half of the Windows [`detect_icds`]: the manifests the capture +/// named, and whether PowerShell actually **answered**. +/// +/// Compiled on every platform, not just Windows, so the Linux lanes can test the +/// failure path at all — the registry read itself has no gate but CI's +/// `windows-build-and-test` job. `Some` means the script exited 0 and its stdout +/// is in hand, so an empty list from it is a real "the key holds no ICD"; `None` +/// is the un-run lookup that must not be reported as one. +#[must_use] +pub fn windows_icds_from_capture(captured: Option) -> (Vec, bool) { + captured.map_or_else( + || (Vec::new(), false), + |output| (parse_windows_vulkan_icd_listing(&output), true), + ) } /// The devices `vulkaninfo --summary` reports, or an empty list when the tool @@ -376,17 +430,30 @@ const fn vulkan_is_probeable() -> bool { /// What this host offers for Vulkan. #[must_use] pub fn detect_vulkan_support() -> VulkanSupport { + let (icds, icds_listed) = detect_icds(); let (devices, enumerated) = detect_vulkan_devices(); - classify_vulkan_support(detect_icds(), devices, vulkan_is_probeable(), enumerated) + classify_vulkan_support( + icds, + devices, + vulkan_is_probeable(), + icds_listed, + enumerated, + ) } /// The pure decision behind [`detect_vulkan_support`], split out so every /// combination of evidence can be tested without a Vulkan install. +/// +/// `icds_listed` and `enumerated` are the two "did this source answer at all" +/// flags, one per source, and they are what keep "could not look" apart from +/// "looked and found nothing". Both must be true before an empty result may be +/// reported as a confirmed absence. #[must_use] pub fn classify_vulkan_support( icds: Vec, devices: Vec, probeable: bool, + icds_listed: bool, enumerated: bool, ) -> VulkanSupport { if !probeable { @@ -440,14 +507,38 @@ pub fn classify_vulkan_support( // or is too old for `--summary` is not — it answered nothing, so it // cannot be the evidence that turns an empty result into a confirmed // absence. - if icds.is_empty() && devices.is_empty() && !enumerated { + // + // `icds_listed` is the same question asked of the other source, and it + // is why an empty `icds` alone proves nothing: on Windows the listing is + // a PowerShell registry read that can be missing, fail or time out, and + // all three arrive here as the same empty `Vec` a genuinely empty + // registry key produces. Both sources must have answered before an empty + // result may be called an absence. + if icds.is_empty() && devices.is_empty() && !(icds_listed && enumerated) { + // Which source went unread decides the sentence: a host is owed the + // reason nothing could be concluded about it, and "no Vulkan driver + // was found" is a claim only a lookup that ran may make. + let detail = match (icds_listed, enumerated) { + // The lookup ran and found no manifest; only `vulkaninfo` is + // missing. (`(true, true)` cannot reach here — it is the + // confirmed absence handled below.) + (true, _) => { + "no Vulkan driver was found, and `vulkaninfo` could not confirm that either \ + (not installed, or it failed to run), so nothing here confirms one is absent" + } + (false, true) => { + "`vulkaninfo` ran and listed no device, but the installed Vulkan drivers could \ + not be listed on this host, so nothing here confirms a driver is absent" + } + (false, false) => { + "the installed Vulkan drivers could not be listed on this host, and \ + `vulkaninfo` could not confirm anything either (not installed, or it failed \ + to run), so nothing here shows whether a driver is present" + } + }; return VulkanSupport { status: VulkanStatus::Inconclusive, - detail: Some( - "no Vulkan driver was found, and `vulkaninfo` could not confirm that either (not \ - installed, or it failed to run), so nothing here confirms one is absent" - .to_owned(), - ), + detail: Some(detail.to_owned()), icds, devices, }; @@ -475,6 +566,9 @@ pub fn classify_vulkan_support( devices, }; } + // Reaching the first arm means `icds` is empty *and* both sources + // answered (the branch above returned otherwise), so this is the only + // place entitled to deny the driver outright. let detail = if icds.is_empty() { "no Vulkan driver found".to_owned() } else { @@ -488,9 +582,17 @@ pub fn classify_vulkan_support( }; } + // An AMD manifest wins over any other vendor's, for the same reason + // `driver_label` restricts its fallback: this sentence is the justification + // the host is handed for being offered a Vulkan path for its AMD GPU, and on + // a dual-vendor box the alphabetical sort would otherwise open it with + // "NVIDIA is installed". A non-AMD label is still used when it is the only + // one there, because the status beside it claims nothing about AMD — it says + // a driver is installed and no AMD device was confirmed. let named = gpu_icds .iter() - .find_map(|icd| icd_driver_label(icd)) + .find_map(|icd| amd_icd_driver_label(icd)) + .or_else(|| gpu_icds.iter().find_map(|icd| icd_driver_label(icd))) .map_or_else( || "a Vulkan driver is installed".to_owned(), |label| format!("{label} is installed"), @@ -590,7 +692,7 @@ GPU0: fn a_real_renoir_host_reports_vulkan_available_and_names_its_driver() { let devices = parse_vulkaninfo_summary(RENOIR_WINDOWS_SUMMARY); - let support = classify_vulkan_support(Vec::new(), devices, true, true); + let support = classify_vulkan_support(Vec::new(), devices, true, true, true); assert_eq!(support.status, VulkanStatus::Available); assert!(support.status.is_usable()); @@ -620,8 +722,13 @@ GPU0: assert_eq!(devices.len(), 1); assert!(!devices[0].is_hardware()); - let support = - classify_vulkan_support(vec!["lvp_icd.x86_64.json".to_owned()], devices, true, true); + let support = classify_vulkan_support( + vec!["lvp_icd.x86_64.json".to_owned()], + devices, + true, + true, + true, + ); assert_eq!( support.status, @@ -645,6 +752,7 @@ GPU0: Vec::new(), true, true, + true, ); assert_eq!(support.status, VulkanStatus::Absent); @@ -666,6 +774,7 @@ GPU0: Vec::new(), true, true, + true, ); assert_eq!(asked.status, VulkanStatus::DriverPresentDevicesUnverified); @@ -691,6 +800,7 @@ GPU0: vec!["radeon_icd.x86_64.json".to_owned()], Vec::new(), true, + true, false, ); @@ -717,7 +827,7 @@ GPU0: fn no_evidence_either_way_is_not_reported_as_a_confirmed_absence() { // `vulkaninfo` absent: nothing could have found a driver, so nothing // may claim there is none. - let unasked = classify_vulkan_support(Vec::new(), Vec::new(), true, false); + let unasked = classify_vulkan_support(Vec::new(), Vec::new(), true, true, false); assert_eq!( unasked.status, VulkanStatus::Inconclusive, @@ -734,7 +844,7 @@ GPU0: // `vulkaninfo` present and it found nothing: that IS the answer, and // it is the shape a host genuinely without a driver takes. - let nothing = classify_vulkan_support(Vec::new(), Vec::new(), true, true); + let nothing = classify_vulkan_support(Vec::new(), Vec::new(), true, true, true); assert_eq!( nothing.status, VulkanStatus::Absent, @@ -748,12 +858,15 @@ GPU0: Vec::new(), true, true, + true, ); assert_eq!(software_only.status, VulkanStatus::Absent); // macOS and friends: no evidence either way, which must not read as // "no Vulkan" any more than it reads as "Vulkan works". - let unprobeable = classify_vulkan_support(Vec::new(), Vec::new(), false, false); + // An unprobeable platform is the one place `icds_listed` is false + // without a failure: the `cfg` stub there has nothing it could read. + let unprobeable = classify_vulkan_support(Vec::new(), Vec::new(), false, false, false); assert_eq!(unprobeable.status, VulkanStatus::Unknown); assert!(!unprobeable.status.is_usable()); } @@ -770,6 +883,7 @@ GPU0: Vec::new(), true, true, + true, ); assert_eq!(support.status, VulkanStatus::DriverPresentDevicesUnverified); } @@ -788,8 +902,13 @@ GPU1: \tdeviceName = AMD Radeon RX 7900 XTX \tdriverName = radv "; - let support = - classify_vulkan_support(Vec::new(), parse_vulkaninfo_summary(mixed), true, true); + let support = classify_vulkan_support( + Vec::new(), + parse_vulkaninfo_summary(mixed), + true, + true, + true, + ); assert_eq!(support.status, VulkanStatus::Available); assert_eq!(support.driver_label().as_deref(), Some("radv")); @@ -824,6 +943,7 @@ GPU0: parse_vulkaninfo_summary(nvidia), true, true, + true, ); // The driver is installed and is not software, so the status stops short @@ -854,7 +974,7 @@ GPU0: "a run that produced no output did not look at this host" ); assert_eq!( - classify_vulkan_support(Vec::new(), devices, true, enumerated).status, + classify_vulkan_support(Vec::new(), devices, true, true, enumerated).status, VulkanStatus::Inconclusive, "so it must not be reported as a confirmed absence" ); @@ -867,7 +987,7 @@ GPU0: "an empty answer is still an answer" ); assert_eq!( - classify_vulkan_support(Vec::new(), devices, true, enumerated).status, + classify_vulkan_support(Vec::new(), devices, true, true, enumerated).status, VulkanStatus::Absent, "and that one has earned the confirmed absence" ); @@ -888,8 +1008,13 @@ GPU0: "; // No ICD manifest at all, so the pre-fix code fell to "no Vulkan driver // found" despite the enumeration above. - let support = - classify_vulkan_support(Vec::new(), parse_vulkaninfo_summary(nvidia), true, true); + let support = classify_vulkan_support( + Vec::new(), + parse_vulkaninfo_summary(nvidia), + true, + true, + true, + ); let detail = support.detail.as_deref().unwrap_or_default(); assert!( @@ -970,4 +1095,214 @@ GPU0: assert!(parse_vulkaninfo_summary("Vulkan Instance Version: 1.3.301\n").is_empty()); assert!(parse_vulkaninfo_summary("").is_empty()); } + + /// The ICD side of "could not look" versus "looked and found nothing" — the + /// same distinction `enumerated` draws for `vulkaninfo`, which the ICD + /// source had no signal for. + /// + /// On Windows the listing is a PowerShell registry read, and + /// `capture_optional_command` reports a missing binary, a non-zero exit and + /// a timeout as one `None`, which `unwrap_or_default()` then turned into the + /// very empty `Vec` an empty Khronos key produces. Asserting that the two + /// details *differ*, rather than asserting each alone, is what keeps the + /// claim tied to the evidence: collapsing them back together has to fail + /// here. + #[test] + fn a_failed_icd_lookup_is_not_reported_as_a_host_without_a_driver() { + // Nothing read the registry, and `vulkaninfo` did not answer either. + let unread = classify_vulkan_support(Vec::new(), Vec::new(), true, false, false); + // Same host, except the lookup ran and genuinely found no manifest. + let read = classify_vulkan_support(Vec::new(), Vec::new(), true, true, false); + + assert_eq!(unread.status, VulkanStatus::Inconclusive); + assert_eq!(read.status, VulkanStatus::Inconclusive); + let unread_detail = unread.detail.clone().expect("a detail is always set here"); + let read_detail = read.detail.clone().expect("a detail is always set here"); + assert!( + !unread_detail.contains("no Vulkan driver was found"), + "the driver list was never read, so nothing here may report it empty: {unread:?}" + ); + assert!( + unread_detail.contains("could not be listed"), + "the host is owed the reason nothing could be concluded about it: {unread:?}" + ); + assert!( + read_detail.contains("no Vulkan driver was found"), + "a lookup that ran and found nothing is entitled to say so: {read:?}" + ); + assert_ne!( + unread_detail, read_detail, + "`icds_listed` is the difference between a lookup that never ran and one that ran \ + and came back empty; a host whose PowerShell failed must not be told what a host \ + with a genuinely empty registry key is told" + ); + + // `Absent` is the confirmed claim, and it takes *both* sources having + // answered. A `vulkaninfo` that ran and listed nothing is not enough on + // its own while the driver list went unread. + let only_vulkaninfo_answered = + classify_vulkan_support(Vec::new(), Vec::new(), true, false, true); + assert_eq!( + only_vulkaninfo_answered.status, + VulkanStatus::Inconclusive, + "one source of two is not an exhaustive look: {only_vulkaninfo_answered:?}" + ); + assert!( + only_vulkaninfo_answered + .detail + .as_deref() + .is_some_and(|detail| detail.contains("could not be listed")), + "{only_vulkaninfo_answered:?}" + ); + assert_eq!( + classify_vulkan_support(Vec::new(), Vec::new(), true, true, true).status, + VulkanStatus::Absent, + "both sources answered and both came back empty: that one has earned the claim" + ); + } + + /// The Windows seam itself, at the only level a Linux lane can reach it: the + /// pure half of `detect_icds`. The registry read behind it is exercised + /// nowhere but CI's `windows-build-and-test`, so the failure and timeout + /// paths — all of which arrive here as `None` — are covered through this. + #[test] + fn a_powershell_that_never_answered_is_not_an_empty_registry_key() { + // Missing `powershell`, a non-zero exit, or a timeout: all one `None`. + let (icds, listed) = windows_icds_from_capture(None); + assert!( + icds.is_empty() && !listed, + "a run that produced no output did not read the registry" + ); + assert_eq!( + classify_vulkan_support(icds, Vec::new(), true, listed, false).status, + VulkanStatus::Inconclusive, + "so it must not be reported as a confirmed absence" + ); + + // Exit 0 with no `ICD` row: the key really is empty, and that answer is + // what a host with no Vulkan driver installed looks like. + let (icds, listed) = windows_icds_from_capture(Some(String::new())); + assert!( + icds.is_empty() && listed, + "an empty answer is still an answer" + ); + assert_eq!( + classify_vulkan_support(icds, Vec::new(), true, listed, true).status, + VulkanStatus::Absent, + "and that one has earned the confirmed absence" + ); + + let (icds, listed) = windows_icds_from_capture(Some( + "ICD\tC:\\Windows\\System32\\amdvlk64.json\n".to_owned(), + )); + assert!(listed); + assert_eq!(icds.len(), 1, "{icds:?}"); + assert_eq!(amd_icd_driver_label(&icds[0]), Some("AMDVLK")); + } + + /// A confirmed AMD Vulkan path must never be attributed to another vendor's + /// driver. `detect_icds` sorts alphabetically on Linux, so on a host with + /// both manifests installed `nvidia_icd.json` came first and won the old + /// "any manifest that is not a software rasteriser" fallback — naming NVIDIA + /// as the driver behind an AMD device the probe had already confirmed. + #[test] + fn a_dual_vendor_host_names_the_amd_driver_behind_a_confirmed_amd_device() { + // The AMD entry omits the optional `driverName`, which is the only + // reason the ICD fallback runs at all. + let dual_vendor = "\ +GPU0: +\tvendorID = 0x10de +\tdeviceType = PHYSICAL_DEVICE_TYPE_DISCRETE_GPU +\tdeviceName = NVIDIA GeForce GTX 1050 Ti +\tdriverName = NVIDIA +GPU1: +\tvendorID = 0x1002 +\tdeviceType = PHYSICAL_DEVICE_TYPE_DISCRETE_GPU +\tdeviceName = AMD Radeon RX 7900 XTX +"; + let devices = parse_vulkaninfo_summary(dual_vendor); + assert_eq!(devices.len(), 2, "{devices:?}"); + assert!( + devices[1].is_amd() && devices[1].driver_name.is_none(), + "{devices:?}" + ); + + let support = classify_vulkan_support( + // Alphabetical, as `detect_icds` returns them on Linux. + vec![ + "nvidia_icd.json".to_owned(), + "radeon_icd.x86_64.json".to_owned(), + ], + devices, + true, + true, + true, + ); + + assert_eq!(support.status, VulkanStatus::Available); + assert_eq!( + support.driver_label().as_deref(), + Some("Mesa RADV"), + "the confirmed device is AMD's, so only an AMD manifest may name its driver: \ + {support:?}" + ); + + // The sibling instance: the sentence that justifies offering a host a + // Vulkan path when no AMD device was confirmed at all. + let unconfirmed = classify_vulkan_support( + vec![ + "nvidia_icd.json".to_owned(), + "radeon_icd.x86_64.json".to_owned(), + ], + Vec::new(), + true, + true, + false, + ); + let unconfirmed_detail = unconfirmed.detail.as_deref().unwrap_or_default(); + assert!( + unconfirmed_detail.starts_with("Mesa RADV is installed"), + "the AMD manifest must open the sentence an AMD GPU is routed on: {unconfirmed:?}" + ); + assert!(!unconfirmed_detail.contains("NVIDIA"), "{unconfirmed:?}"); + + // A host where NVIDIA genuinely is all there is still gets named it: + // that status claims nothing about AMD. But `driver_label` — which only + // ever speaks for a confirmed AMD device — must stay silent. + let nvidia_only = classify_vulkan_support( + vec!["nvidia_icd.json".to_owned()], + Vec::new(), + true, + true, + false, + ); + assert!( + nvidia_only + .detail + .as_deref() + .is_some_and(|detail| detail.starts_with("NVIDIA is installed")), + "naming the only driver there is remains correct: {nvidia_only:?}" + ); + assert_eq!( + nvidia_only.driver_label(), + None, + "no AMD device, no AMD manifest: there is no AMD driver to name: {nvidia_only:?}" + ); + + assert_eq!( + amd_icd_driver_label("radeon_icd.x86_64.json"), + Some("Mesa RADV") + ); + assert_eq!(amd_icd_driver_label("amdvlk64.json"), Some("AMDVLK")); + assert_eq!(amd_icd_driver_label("amd_icd64.json"), Some("AMDVLK")); + assert_eq!(amd_icd_driver_label("nvidia_icd.json"), None); + assert_eq!(amd_icd_driver_label("intel_icd.x86_64.json"), None); + assert_eq!( + amd_icd_driver_label("lvp_icd.x86_64.json"), + None, + "a software rasteriser is never an AMD GPU driver, so narrowing to AMD keeps the \ + exclusion the old filter provided" + ); + assert_eq!(amd_icd_driver_label("some_new_vendor_icd.json"), None); + } } diff --git a/docs/architecture.md b/docs/architecture.md index 903466ae44..731e5758de 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -322,7 +322,7 @@ The library every first-party app and engine crate depends on. It holds host exa | [`host_gpu/linux.rs`](../crates/rocm-core/src/host_gpu/linux.rs) | Linux sysfs, KFD and DRM probes: the KFD topology, the DRM ip-discovery gfx target, the amdgpu device and driver version, the primary GPU name. | domain | | [`host_gpu/os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) | Kernel version, distro name, installed RAM, and the CPU model and vendor for the host summary. One reader returns the CPU model and vendor together for both the summary and `Examination::probe`: Linux from `/proc/cpuinfo`, Windows from the CPU row of the system inventory. `detect_kernel_version` is now the only `uname -r` reader; [`examine/host_env.rs`](../crates/rocm-core/src/examine/host_env.rs)'s `probe_os` calls it for the examination's `kernel_release`. | domain | | [`host_gpu/render.rs`](../crates/rocm-core/src/host_gpu/render.rs) | The `rocm examine` text rendering of an `ExamineSummary`. | domain | -| [`host_gpu/vulkan.rs`](../crates/rocm-core/src/host_gpu/vulkan.rs) | Whether a model can run through Vulkan on this host: ICD manifests, the Windows Khronos registry and `vulkaninfo --summary`, classified into `VulkanStatus`. `Absent` is reserved for a probe that was able to look and ruled a GPU path out: a software-only ICD, an enumeration of only software devices, or a `vulkaninfo` that ran and reported nothing (what `examine-51` plants); with no ICD, no device and no `vulkaninfo` to ask, the answer is `Inconclusive`, which reaches the user as the `VulkanUnconfirmed` support level rather than a confirmed absence, because neither source is exhaustive and claiming absence would tell a host with a working driver to install one. A software rasteriser never counts as a GPU path. An enumeration that returned only non-AMD hardware is reported as such rather than as a missing driver, but is still routed to the `VulkanOnly` level — a known limitation. | domain | +| [`host_gpu/vulkan.rs`](../crates/rocm-core/src/host_gpu/vulkan.rs) | Whether a model can run through Vulkan on this host: ICD manifests, the Windows Khronos registry and `vulkaninfo --summary`, classified into `VulkanStatus`. `Absent` is reserved for a probe that was able to look and ruled a GPU path out: a software-only ICD, an enumeration of only software devices, or a `vulkaninfo` that ran and reported nothing (what `examine-51` plants). It takes *both* sources having answered — `icds_listed` for the driver listing, `enumerated` for `vulkaninfo` — so when either could not be read (a Windows `powershell` that is missing, fails or times out; a `vulkaninfo` that is absent or exits non-zero) and nothing was found, the answer is `Inconclusive`, which reaches the user as the `VulkanUnconfirmed` support level rather than a confirmed absence, with a detail naming which source went unread. Neither source is exhaustive, and claiming absence would tell a host with a working driver to install one. A software rasteriser never counts as a GPU path. An enumeration that returned only non-AMD hardware is reported as such rather than as a missing driver, but is still routed to the `VulkanOnly` level — a known limitation. | domain | | [`host_gpu/windows_inventory.rs`](../crates/rocm-core/src/host_gpu/windows_inventory.rs) | The Windows display-adapter inventory: the PowerShell queries, their parsers, and assembling one inventory from several sources, including the system query whose CPU row the CPU reader in [`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) takes. | domain | | [`host_gpu/wsl.rs`](../crates/rocm-core/src/host_gpu/wsl.rs) | WSL detection and the Windows host driver as seen from the guest: the WSL2 signals, the librocdxg chain and the host-driver probes. | domain | | [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs) | The generic append-only JSONL store engine: sibling `.lock` file, single-line appends, torn-tail repair, size rotation and tail-first reads. | domain | diff --git a/docs/host-probing.md b/docs/host-probing.md index b7b13e70ba..9b1ef9f929 100644 --- a/docs/host-probing.md +++ b/docs/host-probing.md @@ -200,10 +200,13 @@ Verdict: not a duplicate — only the summary stack reads it — but it is a spa | Devices | `vulkaninfo --summary` | the same registry read; `vulkaninfo` is rarely on PATH | | Timeout | 1.5 s (`OPTIONAL_COMMAND_TIMEOUT`), per candidate spelling | the same 1.5 s, which a cold PowerShell start can exceed | | Failure | `None` from `capture_optional_command`, which does not distinguish "not installed" from "timed out" | the same | +| Lookup ran? | always true: reading directories that may not exist is still a lookup | `false` whenever `powershell` is missing, exits non-zero or times out | + +The last row is `icds_listed`, the ICD-side counterpart of the `enumerated` flag that `vulkan_devices_from_capture` returns for `vulkaninfo`. Both say whether that source *answered*, and `classify_vulkan_support` requires both before it will report an empty result as a confirmed absence. On a platform that is neither Linux nor Windows the flag is `false` — there is nothing to read there — though the classifier never consults it, since `vulkan_is_probeable()` returns `Unknown` first. Who pays it. The probe runs once per full gather, so every caller of `ExamineSummary::gather` pays one: the text and JSON of `rocm examine`, the MCP and bridge snapshots, the free-form answer, `rocm install driver` and its `--reconcile` form, and `rocmd`'s host snapshot, MCP and sandbox paths. It is deliberately *not* run by the lean scope, `gather_without_driver_facts`, which the runtime-install finalizer uses to read the GPU target and nothing else; that scope leaves `vulkan` as `None` and `vulkan_status` reports `Unknown`. `the_summary_without_driver_facts_does_not_probe_vulkan` pins that, so restoring an unconditional probe fails a test rather than quietly adding a process to the install path. -What a user can see: on a Windows host whose PowerShell start is slower than 1.5 s, the probe times out and the host reads Vulkan-unconfirmed rather than Vulkan-capable — the same answer it gives a host with no ICD at all. That collapse is in `capture_optional_command`, not in the parser, and it is the gap called out in the review of #1036: the Windows branch has no unit coverage of its failure and timeout cases. +What a user can see: on a Windows host whose PowerShell start is slower than 1.5 s, the probe times out and the host reads Vulkan-unconfirmed rather than Vulkan-capable. It is no longer told the *same thing* as a host with a genuinely empty registry key, which is what `icds_listed` buys: the timed-out host is told the driver list could not be read, never that no driver was found, and it can no longer reach `Absent` at all. The collapse that remains is inside `capture_optional_command`, which still cannot say *which* failure it was ("not installed" and "timed out" are one `None`), so the sentence names the possibilities rather than the cause. `windows_icds_from_capture` is the pure seam where that `None` is turned into "did not answer", and it is unit-tested on every platform; the registry read itself still has no gate but CI's `windows-build-and-test`. ## Marketing name to gfx target From 1bc2bada9f6b8a72e824dd5873caa4544f388b42 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 13 Nov 2026 18:53:54 +0200 Subject: [PATCH 41/59] fix(cli): stop markdown markup reaching `--help`, and make the Vulkan-only claim one string Non-blocking items from the review on #678. `rocm examine --help` printed literal `**every**`: clap renders doc comments verbatim, and the file's ASCII guard cannot see markup made of ASCII. The emphasis is dropped, and a sibling test walks the same command tree rejecting `**`/`__` so it cannot come back. Hidden `__engine-*` subcommand names are read off the tree and removed before the search rather than the `__` half being given up. No other help text in the tree was affected. "Unknown covers both a GPU nothing detected" was not English, in `--help` and in README.md both; reworded in sync. `install sdk`'s two refusals re-derived the Vulkan-only condition from `gfx_target_has_no_rocm_build` and re-typed its claim, which was exactly equivalent today and a second place to change tomorrow. They now gate on `rocm_core::gpu_support_level`'s `VulkanOnly` tier and interpolate one `VULKAN_ONLY_SUPPORT_CLAIM`, shared with `rocm examine`'s `rocm_support` line. All three strings are byte-identical to before. `SERVICE_DEVICE_PROBE_BUDGET` claimed the "chat `services` tool" pays the budget. The natural-language command does; the structured MCP tool of the same name renders through `render_services_tool_result_text`, has no `device:` row and probes nothing. The comment now says which is which, rather than the tool being given a row it was never in scope for. Signed-off-by: Roman Sirokov --- README.md | 2 +- apps/rocm/src/cli.rs | 74 +++++++++++++++-- apps/rocm/src/engine_ipc.rs | 24 ++++-- apps/rocm/src/services_text.rs | 10 +-- apps/rocm/src/therock/family.rs | 101 +++++++++++++++++++++--- crates/rocm-core/src/gfx_target.rs | 43 ++++++++++ crates/rocm-core/src/host_gpu/render.rs | 7 +- crates/rocm-core/src/lib.rs | 4 +- 8 files changed, 232 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 2f8b63494f..a71fc591c0 100644 --- a/README.md +++ b/README.md @@ -545,7 +545,7 @@ These GPUs still work with the Vulkan backend, which needs no ROCm SDK. `rocm examine` prints that `rocm_support` line on **every** host, not only this case, at one of three levels: ROCm/HIP-supported, Vulkan-only (the case above), -or unknown. Unknown covers both a GPU nothing detected and a target this build's +or unknown. Unknown covers both no GPU detected and a target this build's catalog does not recognise — detection gaps rather than a verdict about the hardware, so neither is reported as "no ROCm build". diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index da05e42872..1f8fe71d0e 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -63,14 +63,14 @@ pub(crate) struct Cli { pub(crate) enum Command { /// Check this computer's GPU, ROCm install, engines, and setup folders. /// - /// A `rocm_support` line is printed on **every** host, at one of three + /// A `rocm_support` line is printed on every host, at one of three /// levels: ROCm/HIP-supported, Vulkan-only, or unknown. Vulkan-only covers /// a GPU that is recognized but has no ROCm build at all (`gfx90c`, the /// Vega iGPU in Renoir, Lucienne, Cezanne and Barcelo APUs), which still - /// works through Lemonade's Vulkan backend. Unknown covers both a GPU - /// nothing detected and a target this build's catalog does not recognize: - /// those are detection gaps, not a verdict on the hardware, which is why - /// an empty `compatible_therock_family` does not read as a failed probe. + /// works through Lemonade's Vulkan backend. Unknown covers both no GPU + /// detected and a target this build's catalog does not recognize: those + /// are detection gaps, not a verdict on the hardware, which is why an + /// empty `compatible_therock_family` does not read as a failed probe. Examine { /// Emit the machine-readable Examination JSON (for diagnosis tooling). #[arg(long)] @@ -2680,4 +2680,68 @@ mod tests { assert_ascii_help(&mut Cli::command(), "rocm"); } + + /// Sibling of `every_help_text_in_the_command_tree_is_ascii`, for the same + /// reason one level down: clap renders a doc comment verbatim, with no + /// markdown processing anywhere in its pipeline, so `**every**` written in + /// a doc comment reaches the user's terminal as literal asterisks. Review + /// on #678 caught exactly that on `Examine`, which the ASCII test could + /// not see because `*` and `_` are ASCII. + /// + /// Only `**` and `__` are rejected, and only those: they have no honest + /// use in plain help text. A single `*` does (`gfx110*`, a glob in an + /// example), and backticks are the repo's deliberate house style for + /// quoting commands in help, so neither is banned here. + /// + /// The hidden `__engine-*` subcommands are a real `__` in rendered help — + /// their own usage lines print their names — so each command's internal + /// names are removed before the search rather than the check being + /// dropped. The list is read off the command tree, not hard-coded, so a + /// new hidden command does not make this fail spuriously. + #[test] + fn no_help_text_in_the_command_tree_contains_markdown_emphasis() { + fn assert_no_markdown_emphasis(cmd: &mut clap::Command, path: &str) { + let mut internal: Vec = cmd + .get_subcommands() + .map(|sub| sub.get_name().to_owned()) + .filter(|name| name.starts_with("__")) + .collect(); + if cmd.get_name().starts_with("__") { + internal.push(cmd.get_name().to_owned()); + } + + // Both renderings, for the same reason as the ASCII test: `-h` + // truncates at the first paragraph, so emphasis in a later one is + // only visible in the long form. + for rendered in [cmd.render_help(), cmd.render_long_help()] { + let mut rendered = rendered.to_string(); + for name in &internal { + rendered = rendered.replace(name.as_str(), ""); + } + for marker in ["**", "__"] { + assert!( + !rendered.contains(marker), + "`{path} --help` must not contain markdown emphasis (`{marker}`); \ + clap renders doc comments verbatim, so the user sees the markup \ + itself: {rendered}" + ); + } + } + // Collect first: `find_subcommand_mut` needs a mutable borrow that + // cannot overlap the iterator borrowing `cmd`. + let names: Vec = cmd + .get_subcommands() + .map(|sub| sub.get_name().to_owned()) + .collect(); + for name in names { + let child_path = format!("{path} {name}"); + let child = cmd + .find_subcommand_mut(&name) + .expect("subcommand name was just enumerated from this command"); + assert_no_markdown_emphasis(child, &child_path); + } + } + + assert_no_markdown_emphasis(&mut Cli::command(), "rocm"); + } } diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index e1a611bae6..4954c4e04d 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -469,16 +469,28 @@ where /// is a *start* gate: it is tested before each probe, never inside one, so a /// probe begun at 9.99s still runs to its own timeout. The arithmetic is 10s /// of elapsed budget plus Lemonade's 8s `INFERENCE_PROBE_TIMEOUT`. That bound -/// applies equally to the chat `services` tool, which renders through the same +/// applies equally to `rocm chat` when its natural-language interpreter +/// resolves a request to the `services` command: that path runs the real +/// command in process (`chat::mcp_bridge`'s `run_rocm_read_only_in_process`) +/// and so renders through the same /// [`crate::services_text::render_services_text`]. /// +/// It does *not* apply to the structured MCP tool that is also named +/// `services`. That one renders through `chat::mcp_bridge`'s +/// `render_services_tool_result_text`, a different and older function with no +/// `device:` row, so a bridge client calling the tool makes no probe and waits +/// for nothing — unlike the sibling `engines` MCP tool, which ROCMAI-550 did +/// make host-aware. The shared name is the trap, so this says which surface it +/// means (review on #678); giving the tool the row is a change in its own +/// right, not something this constant may be read as already covering. +/// /// # The decision, recorded (review on #678) /// -/// This is accepted rather than merely observed. `rocm services list` and the -/// chat `services` tool can both block ~18s on a host whose engines are slow to -/// answer, and for the duration the built-in probe holds the process-wide -/// engine env lock [`with_scoped_builtin_engine_env`] takes, so a concurrent -/// caller waits behind it. +/// This is accepted rather than merely observed. `rocm services list` and +/// `rocm chat`'s `services` command can both block ~18s on a host whose +/// engines are slow to answer, and for the duration the built-in probe holds +/// the process-wide engine env lock [`with_scoped_builtin_engine_env`] takes, +/// so a concurrent caller waits behind it. /// /// Two cheaper options were considered and not taken. Moving the probe off the /// render path means the row stops reflecting the host at the moment it is diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index cdac966bb4..497e376e31 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -402,15 +402,15 @@ fn format_anthropic_endpoint_line(url: &str, responding: bool) -> String { } } -/// `rocm services list`, and the chat `services` tool, which renders the same -/// text. +/// `rocm services list`, and `rocm chat`'s `services` command, which runs this +/// in process and renders the same text. Not the MCP tool of the same name, +/// which has its own renderer and no `device:` row. /// /// **No longer a pure local read.** Since ROCMAI-550 it makes one live /// `Healthcheck` RPC per *ready* service to fill the `device:` row, bounded in /// aggregate by `engine_ipc::SERVICE_DEVICE_PROBE_BUDGET` — see that constant -/// for the arithmetic behind the ~18s worst case. The chat tool pays the same -/// bound rather than render a device-less view, which would have the two -/// surfaces disagree while claiming to show the same list. +/// for the ~18s worst case and for which `services` surface pays it. The chat +/// command pays it too, rather than show a device-less view of the same list. pub(crate) fn render_services_text(paths: &AppPaths, all: bool) -> Result { render_services_text_with_devices(paths, all, |ready| fetch_service_devices(paths, ready)) } diff --git a/apps/rocm/src/therock/family.rs b/apps/rocm/src/therock/family.rs index a73f03b573..f0153330f5 100644 --- a/apps/rocm/src/therock/family.rs +++ b/apps/rocm/src/therock/family.rs @@ -12,9 +12,9 @@ use super::source::{FamilyResolution, TheRockChannel}; use anyhow::{Result, bail}; use rocm_core::{ - AppPaths, detect_host_gfx_target, detect_host_gpu_diagnostics, detect_managed_therock_family, - gfx_target_has_no_rocm_build, known_therock_families, normalize_therock_family, - runtime_is_windows, + AppPaths, GpuSupportLevel, VULKAN_ONLY_SUPPORT_CLAIM, detect_host_gfx_target, + detect_host_gpu_diagnostics, detect_managed_therock_family, gpu_support_level, + known_therock_families, normalize_therock_family, runtime_is_windows, }; use std::fmt::Write as _; @@ -99,7 +99,7 @@ pub(super) fn resolve_family( // left over from an earlier session happens to name gfx90c, and the // reverse for an env value once `--family` isn't the one in play. if let Some(value) = family_override { - if gfx_target_has_no_rocm_build(value) { + if target_is_vulkan_only(value) { bail!(unsupported_requested_target_message( value.trim(), "--family" @@ -115,7 +115,7 @@ pub(super) fn resolve_family( } if let Some(value) = env_override.as_deref() { - if gfx_target_has_no_rocm_build(value) { + if target_is_vulkan_only(value) { bail!(unsupported_requested_target_message( value.trim(), "ROCM_CLI_THEROCK_FAMILY" @@ -156,7 +156,7 @@ pub(super) fn resolve_family( // there is nothing to install for this GPU. Other unmapped targets keep the // generic error below, which points at `--family`. if let Some(raw_arch) = host_arch.as_deref() - && gfx_target_has_no_rocm_build(raw_arch) + && target_is_vulkan_only(raw_arch) { let message = unsupported_host_gpu_message(raw_arch); let diagnostics = detect_host_gpu_diagnostics(); @@ -172,6 +172,32 @@ pub(super) fn resolve_family( ) } +/// Whether `value` is a GPU target the CLI recognizes and knows has no ROCm +/// build — [`GpuSupportLevel::VulkanOnly`] — which is what `install sdk` +/// refuses with the two messages below. +/// +/// Asked of [`gpu_support_level`] rather than re-tested here. The install path +/// used to call `gfx_target_has_no_rocm_build` directly, which was *exactly* +/// equivalent (review on #678 checked that it was) and still wrong to keep: it +/// made this a second, independent definition of the tier whose claim these +/// messages then go on to make, so a change to the classification would have +/// left `rocm examine` and `rocm install sdk` disagreeing about the same GPU +/// while both looked correct in isolation. One classifier, one answer. +/// +/// The family argument is computed rather than passed as `None` because this +/// is also asked of a target the *user* named, before any host probe: for +/// `--family gfx1100` the covering family is what makes the answer `RocmHip`, +/// and hard-coding `None` here would classify every requested target as if no +/// family covered it. On the host branch the caller has already established +/// that `normalize_therock_family` answers `None`, so the extra call there is +/// a no-op that costs a string compare. +fn target_is_vulkan_only(value: &str) -> bool { + matches!( + gpu_support_level(Some(value), normalize_therock_family(value).as_deref()), + GpuSupportLevel::VulkanOnly + ) +} + /// The install error for a detected GPU target that no TheRock package family /// covers. It deliberately does not list `--family` values: none of them was /// built for this GPU. @@ -203,8 +229,7 @@ fn unsupported_host_gpu_message(raw_arch: &str) -> String { format!( "no ROCm SDK build exists for the detected GPU target `{raw_arch}`, so \ `rocm install sdk` has nothing to install for it.\n\ - The GPU is detected but is not supported by ROCm; \ - the Vulkan backend can still be used. \ + The GPU is detected but is {VULKAN_ONLY_SUPPORT_CLAIM}. \ Try `rocm serve --engine lemonade` - it runs on Lemonade's \ Vulkan backend instead." ) @@ -232,8 +257,8 @@ fn unsupported_requested_target_message(value: &str, source: &str) -> String { format!( "no ROCm SDK build exists for the GPU target `{value}` requested by \ {source}, so `rocm install sdk` has nothing to install for it.\n\ - That GPU is not supported by ROCm; the Vulkan backend can still be \ - used. Nothing was installed and no other family was substituted. \ + That GPU is {VULKAN_ONLY_SUPPORT_CLAIM}. Nothing was installed and no \ + other family was substituted. \ Run `rocm examine` to see what this host itself supports." ) } @@ -342,6 +367,60 @@ mod tests { assert_eq!(raw_arch_agreeing_with_family(None, "gfx103X-dgpu"), None); } + /// The gate on both install refusals is the classifier's `VulkanOnly` + /// tier, not a private re-derivation of it. + /// + /// Every case is one `gpu_support_level` answers, so a change there has to + /// come through here: `gfx90c` is the recognized-with-no-ROCm-build tier + /// and is refused; a covered target is `RocmHip` and is not; an + /// unrecognized one is `Unknown`, a gap in this build's catalog rather + /// than a fact about the card, and refusing it would be the exact false + /// claim `GpuSupportLevel`'s doc comment exists to prevent. + #[test] + fn the_install_refusal_gate_is_the_classifiers_vulkan_only_tier() { + for refused in ["gfx90c", "GFX90C", "gfx90c:xnack-", " gfx90c "] { + assert!( + target_is_vulkan_only(refused), + "`{refused}` is the Vulkan-only tier and install must refuse it" + ); + } + for allowed in ["gfx1100", "gfx1151", "gfx90a", "gfx950-dcgpu"] { + assert!( + !target_is_vulkan_only(allowed), + "`{allowed}` has a ROCm build; install must not refuse it" + ); + } + for unknown in ["gfx803", "not-a-target", ""] { + assert!( + !target_is_vulkan_only(unknown), + "`{unknown}` is unrecognized, which is a gap in this catalog and \ + not a confirmed absence of a ROCm build: refusing it would claim \ + something about hardware nobody probed" + ); + } + } + + /// Both refusals and `rocm examine`'s `rocm_support` line make the same + /// claim about a Vulkan-only GPU, so they render it from the same string. + /// + /// Asserting the constant is *in* each message is what makes it one + /// claim; the wording itself is pinned word-for-word in `rocm-core` + /// (`the_vulkan_only_claim_is_pinned_word_for_word`), so neither test + /// alone can be satisfied by quietly rewriting the other's expectation. + #[test] + fn both_install_refusals_render_the_shared_vulkan_only_claim() { + for message in [ + unsupported_host_gpu_message("gfx90c"), + unsupported_requested_target_message("gfx90c", "--family"), + ] { + assert!( + message.contains(VULKAN_ONLY_SUPPORT_CLAIM), + "the claim must come from the shared constant, or install and \ + examine can drift apart while both look right: {message}" + ); + } + } + #[test] fn unsupported_gpu_error_names_the_gpu_and_does_not_suggest_families() { let message = unsupported_host_gpu_message("gfx90c"); @@ -392,7 +471,7 @@ mod tests { /// Both refusals used to close with the same "`rocm serve ` runs on /// Lemonade's Vulkan backend" sentence. In `unsupported_host_gpu_message` /// that is a fact: it is only reached after `detect_host_gfx_target` returns - /// a target `gfx_target_has_no_rocm_build` confirms. This one fires from the + /// a target `target_is_vulkan_only` confirms. This one fires from the /// `--family` / `ROCM_CLI_THEROCK_FAMILY` branches before any probe runs — /// see `cli_family_overrides_a_stale_env_value_naming_an_unsupported_gpu` /// for an override that deliberately disagrees with its host — so on a diff --git a/crates/rocm-core/src/gfx_target.rs b/crates/rocm-core/src/gfx_target.rs index 092c288a46..ecbdfb4b0a 100644 --- a/crates/rocm-core/src/gfx_target.rs +++ b/crates/rocm-core/src/gfx_target.rs @@ -573,9 +573,34 @@ pub enum GpuSupportLevel { Unknown, } +/// The one claim every [`GpuSupportLevel::VulkanOnly`] surface makes, written +/// once. +/// +/// `rocm examine`'s `rocm_support` line and `rocm install sdk`'s two refusals +/// (`unsupported_host_gpu_message` / `unsupported_requested_target_message` in +/// `apps/rocm/src/therock/family.rs`) all tell the user the same two things +/// about a Vulkan-only GPU: ROCm will not run on it, and Vulkan will. They are +/// three different sentences around it — each adds its own subject and its own +/// next step, because what they may promise differs — but the claim itself is +/// shared, so it lives here next to the classifier that decides when it is +/// true rather than being retyped at each site. +/// +/// Deliberately just the claim, with no leading subject and no trailing +/// punctuation: a caller supplies both. Changing the wording here changes all +/// three surfaces together, which is the point. +pub const VULKAN_ONLY_SUPPORT_CLAIM: &str = + "not supported by ROCm; the Vulkan backend can still be used"; + /// Classify the host's GPU support level from the same two facts /// `ExamineSummary` and `resolve_family` already compute: the detected gfx /// target and whichever TheRock family (if any) covers it. +/// +/// Not only for a *host*: `resolve_family` also asks this of a target the user +/// named with `--family` or `ROCM_CLI_THEROCK_FAMILY`, to decide whether to +/// refuse it. The two arguments are facts about a GPU target, not about this +/// machine, so that is the same question. What differs is what the caller may +/// then say — a requested target establishes nothing about the hardware here, +/// which is why the two install refusals end differently. pub fn gpu_support_level( detected_gfx_target: Option<&str>, compatible_therock_family: Option<&str>, @@ -680,6 +705,24 @@ mod tests { ); } + /// The shared claim is user-visible text on three surfaces, so its wording + /// is pinned here rather than only asserted as "whatever the constant + /// says". + /// + /// `rocm examine`'s `rocm_support` line and both `rocm install sdk` + /// refusals now interpolate [`VULKAN_ONLY_SUPPORT_CLAIM`], which is what + /// stops them drifting — but it also means a careless edit to the constant + /// silently rewords all three at once, and their own tests would still + /// pass because they assert containment of the same constant. This test is + /// the one place that has to be changed deliberately. + #[test] + fn the_vulkan_only_claim_is_pinned_word_for_word() { + assert_eq!( + VULKAN_ONLY_SUPPORT_CLAIM, + "not supported by ROCm; the Vulkan backend can still be used" + ); + } + #[test] fn gfx_target_has_no_rocm_build_names_only_gfx90c() { assert!(gfx_target_has_no_rocm_build("gfx90c")); diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index dc732c4fa2..aec43dfbfb 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -6,7 +6,7 @@ use super::ExamineSummary; use super::os_facts::format_gib_value; -use crate::gfx_target::{GpuSupportLevel, gpu_support_level}; +use crate::gfx_target::{GpuSupportLevel, VULKAN_ONLY_SUPPORT_CLAIM, gpu_support_level}; impl ExamineSummary { /// The `rocm_support` line of the examine text. @@ -39,9 +39,8 @@ impl ExamineSummary { // fallback below is unreachable; it keeps the arm total without a // panic path. GpuSupportLevel::VulkanOnly => format!( - " rocm_support: GPU detected ({}) but not supported by ROCm; \ - the Vulkan backend can still be used - serve on it with \ - `rocm serve --engine lemonade`\n", + " rocm_support: GPU detected ({}) but {VULKAN_ONLY_SUPPORT_CLAIM} - \ + serve on it with `rocm serve --engine lemonade`\n", detected.unwrap_or_default() ), // The covering family is the fact; it is already printed on its own diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index c5ea6662a8..8120e70340 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -132,7 +132,9 @@ pub use fix::{ apply as apply_fix, catalog_manifest, catalog_manifest_json, exit as fix_exit, list_recipes as list_fix_recipes, }; -pub use gfx_target::{GpuSupportLevel, gpu_support_level, gpu_support_level_for_host_gpu}; +pub use gfx_target::{ + GpuSupportLevel, VULKAN_ONLY_SUPPORT_CLAIM, gpu_support_level, gpu_support_level_for_host_gpu, +}; pub use gfx_target::{ extract_first_gfx_token, gfx_target_has_no_rocm_build, known_therock_families, normalize_therock_family, From d35ac5fea0d36d7f7fbe0d07efae94a6ee541455 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 13 Nov 2026 18:58:37 +0200 Subject: [PATCH 42/59] docs: point the static-catalog note at rocmd's real engine inventory `lemonade_engine_note`'s doc comment named `apps/rocmd/src/common.rs` as the place keeping the unconditional ROCm wording. That file has no engine catalog and no such string; the table is `rocmd_engine_inventory` in `apps/rocmd/src/host_snapshot.rs`. The comment exists to say which surfaces are deliberately left host-unaware, so a wrong pointer defeats its only purpose -- a reader checking the claim finds nothing and cannot tell whether the deferral is real. Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 83558840d1..b6f0e54164 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -1052,7 +1052,8 @@ pub(crate) fn engine_inventory_note( /// /// All three of those have a live `GpuSupportLevel` in scope. The two places /// that still print the note statically — the dashboard TUI's engine-manager -/// pane and `rocmd`'s own catalog in `apps/rocmd/src/common.rs` — keep the +/// pane and `rocmd`'s own `rocmd_engine_inventory` in +/// `apps/rocmd/src/host_snapshot.rs` — keep the /// unconditional ROCm wording, and that is deliberate rather than an oversight: /// the TUI pane renders a hard-coded descriptive list with no probe behind it, /// and `rocmd` is the daemon answering for whichever host a client asks about, From da8437eeace686be18922528ef634b049751d592 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Mon, 16 Nov 2026 12:59:35 +0200 Subject: [PATCH 43/59] fix(examine): stop telling a working Adrenalin host to install the driver it has A Windows machine with a current Adrenalin install lands in `VulkanUnconfirmed` routinely -- the Khronos `Drivers` key the probe reads is legacy there and `vulkaninfo` is rarely on PATH -- yet both surfaces that render that level opened with "no Vulkan driver was found" and told the user to install the AMD display driver they already had. The claim and the next step become `rocm-core` constants so `rocm examine`'s `rocm_support` line and `lemonade_engine_note`, which appear a few lines apart in the same report, cannot contradict each other. The next step is cross-checked against `driver.status`: on Windows the display driver carries the Vulkan ICD, so `amd_display_driver_detected` drops the install advice and leaves `vulkaninfo` as the one step that can still settle it. The cross-check is Windows-only because Linux's `driver_status` reports the `amdgpu` kernel module, which says nothing about a userspace ICD. Each evidence source now travels as a `VulkanEvidence` carrying its own "did this answer at all", so transposing the ICD and device sources is a type error rather than a silent flip between `Inconclusive` and `Absent`. `VulkanStatus::as_str` replaces the hand-written table in `vulkan_line` and is held against what serde writes, so the text and `--json` spellings of one status cannot drift. `VulkanStatus::is_usable` had no production caller and is removed; the tests that asserted it now assert the routing `gpu_support_level` actually produces. The e2e advice arms for the two probed Vulkan tiers now match on `rocm_core::VULKAN_UNCONFIRMED_CLAIM` rather than a copy of its text, so a rewording cannot silently stop them firing, and the tier owes ``run `vulkaninfo` `` -- the step that can clear it on every host that reaches it, including the Windows one with no install left to do. README and manual-testing said `rocm examine --json` reports a `vulkan` object; the host summary is flattened into the document's `summary` member, so both now say `summary.vulkan`. Comments throughout drop review-process narration and state the invariant in the present tense instead. Signed-off-by: Roman Sirokov --- README.md | 9 +- apps/rocm/src/main.rs | 55 +- crates/rocm-core/src/gfx_target.rs | 113 ++-- crates/rocm-core/src/host_gpu.rs | 18 +- crates/rocm-core/src/host_gpu/render.rs | 184 +++++-- crates/rocm-core/src/host_gpu/vulkan.rs | 492 +++++++++++------- crates/rocm-core/src/lib.rs | 17 +- docs/manual-testing.md | 4 +- tests/e2e-cucumber/features/examine.feature | 18 +- tests/e2e-cucumber/tests/e2e/examine_steps.rs | 44 +- 10 files changed, 641 insertions(+), 313 deletions(-) diff --git a/README.md b/README.md index ee3ba480f3..5c07bf1303 100644 --- a/README.md +++ b/README.md @@ -565,10 +565,11 @@ separate from "no GPU backend" for the same reason in the other direction: on Windows the Khronos `Drivers` key is legacy and `vulkaninfo` is rarely on PATH, so a working machine can land there, and it has not earned a confirmed "no". -`rocm examine --json` reports the evidence behind this in a `vulkan` object -(status, ICD manifests found, devices enumerated). The field is optional, so an -older `rocm remote doctor` summary that predates it reads as unknown rather -than as a false "no Vulkan". +`rocm examine --json` reports the evidence behind this in a `summary.vulkan` +object (status, ICD manifests found, devices enumerated). The host summary is +flattened into the document's `summary` member, so the object is there and not +at the top level. The field is optional, so an older `rocm remote doctor` +summary that predates it reads as unknown rather than as a false "no Vulkan". The probe runs wherever the full host summary is gathered, not on `rocm examine` alone: the text and JSON of `rocm examine`, the MCP and bridge snapshots, the diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 01f025f1cc..e586f632ad 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -1105,17 +1105,26 @@ const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str // its reader to run the command they were already reading. It names the // thing that would actually settle the question instead. // - // That thing is NOT "install `vulkaninfo`", for the reason the matching - // arm in `host_gpu::render` spells out: on a real driverless Linux host - // the tool is already installed, prints "Found no drivers!" and exits 1, - // which is exactly what lands here. Both strings appear in the same - // `rocm examine` report, so advising that install would contradict the - // `rocm_support:` line a few lines above it. + // The claim and the next step are `rocm_core`'s + // `VULKAN_UNCONFIRMED_CLAIM` and `VULKAN_UNCONFIRMED_NEXT_STEP`, + // spelled out here only because this is a `const fn` returning a + // `&'static str` and a constant cannot be interpolated into one. + // `the_lemonade_note_and_the_support_line_make_the_same_unconfirmed_claim` + // holds the two in step; both strings appear in the *same* `rocm + // examine` report, a few lines apart, so a divergence is visible to the + // user immediately. + // + // In particular this must not say "install a Vulkan driver" + // unconditionally: a working Windows Adrenalin machine lands in this + // level, since the Khronos key it reads is legacy there. Nor "install + // `vulkaninfo`": on a driverless Linux host that tool is already + // installed, prints "Found no drivers!" and exits 1, which is another + // of the shapes that lands here. rocm_core::GpuSupportLevel::VulkanUnconfirmed => { "default embedded Lemonade server; this GPU has no ROCm/HIP support, and whether a \ - Vulkan driver is present could not be determined - install a Vulkan driver (Mesa \ - RADV on Linux, the AMD display driver on Windows) to run models on this GPU, and \ - run `vulkaninfo` yourself to see what it says" + Vulkan driver is present could not be determined here - run `vulkaninfo` yourself \ + to see what it reports, and install a Vulkan driver (Mesa RADV on Linux, the AMD \ + display driver on Windows) if it finds none" } rocm_core::GpuSupportLevel::NoGpuBackend => { "default embedded Lemonade server; this GPU has no ROCm/HIP support and no Vulkan driver was found, so no GPU backend can run here" @@ -1334,6 +1343,34 @@ pub(crate) mod tests { ); } + /// The lemonade note and `rocm examine`'s `rocm_support` line both render + /// `VulkanUnconfirmed`, and both appear in the same `rocm examine` report a + /// few lines apart. They must make the same claim about the host and offer + /// the same next step, so both are held against the constants `rocm-core` + /// defines; this note spells them out only because a `const fn` returning + /// `&'static str` cannot interpolate a constant. + /// + /// Neither may tell a host to install a Vulkan driver unconditionally: a + /// working Windows Adrenalin machine reaches this level, because the + /// Khronos registry key the probe reads is legacy there. + #[test] + fn the_lemonade_note_and_the_support_line_make_the_same_unconfirmed_claim() { + let note = super::lemonade_engine_note(rocm_core::GpuSupportLevel::VulkanUnconfirmed); + + assert!( + note.contains(rocm_core::VULKAN_UNCONFIRMED_CLAIM), + "the note must make the shared claim verbatim: {note}" + ); + assert!( + note.contains(rocm_core::VULKAN_UNCONFIRMED_NEXT_STEP), + "and offer the shared next step verbatim: {note}" + ); + assert!( + !note.contains("no Vulkan driver was found"), + "a level a working Adrenalin laptop reaches must not deny its driver: {note}" + ); + } + use std::process::ExitCode; /// `Ok(())` must map to a clean exit so `rocm`'s successful commands don't diff --git a/crates/rocm-core/src/gfx_target.rs b/crates/rocm-core/src/gfx_target.rs index ca81d8edb2..e6e6a13767 100644 --- a/crates/rocm-core/src/gfx_target.rs +++ b/crates/rocm-core/src/gfx_target.rs @@ -578,12 +578,12 @@ pub enum GpuSupportLevel { /// `render.rs` that tells the two apart in what the user reads, and it only /// names a driver when one was actually found. /// - /// The second half of that sentence is evidence, not inference. This variant - /// used to be returned from the gfx table alone — "no ROCm build" was taken - /// to mean "Vulkan then", on a machine nobody had looked at. Every surface - /// that renders this level tells the user to go and serve a model on it, so - /// a host with no Vulkan driver installed, or with only a software - /// rasteriser, was sent to a backend that cannot start. + /// The second half of that sentence must be evidence, never inference. + /// Deriving it from the gfx table alone — reading "no ROCm build" as + /// "Vulkan then", on a machine nobody has looked at — sends a host with no + /// Vulkan driver, or with only a software rasteriser, to a backend that + /// cannot start: every surface that renders this level tells the user to go + /// and serve a model on it. VulkanOnly, /// Recognized, confirmed to have no ROCm/HIP build, and the Vulkan probe /// could not reach an answer — no ICD manifest, no enumerated device, and @@ -593,9 +593,9 @@ pub enum GpuSupportLevel { /// as a confirmed fact and is serialized for JSON consumers, and this state /// has not earned it: on Windows the Khronos `Drivers` key is legacy and /// `vulkaninfo` is rarely on PATH, so a working Adrenalin laptop — the - /// machine shape ROCMAI-550 was reported from — can land here. Telling it - /// there is no GPU backend would be a worse answer than the static claim - /// this change replaced. + /// machine shape ROCMAI-550 was reported from — lands here routinely. + /// Telling it there is no GPU backend is a worse answer than the + /// unconditional Vulkan claim this level exists to replace. /// /// Nor is it [`Self::VulkanOnly`], which promises a backend that may not be /// there. It is its own answer: say what was looked for, that nothing was @@ -628,16 +628,16 @@ pub enum GpuSupportLevel { /// classifier that decides when it is true rather than being retyped at each /// site. /// -/// `rocm examine`'s `rocm_support` line used to interpolate this too, and -/// deliberately no longer does. It is the one surface that pays for the Vulkan -/// probe, so it grades the second half of the claim by what the probe actually -/// established — an enumerated device is named, an installed-but-unenumerated -/// driver gets a weaker sentence, an unprobed platform admits it — and the -/// unconditional "can still be used" is exactly the promise that change exists -/// to stop making. See `rocm_support_line` in `host_gpu/render.rs`. The two -/// install refusals keep it because they are asked of a *target*, before and -/// without any host probe (they classify with [`VulkanStatus::Unknown`]), so -/// the graded wording has no evidence to grade on there. +/// `rocm examine`'s `rocm_support` line deliberately does **not** interpolate +/// this. It is the one surface that pays for the Vulkan probe, so it grades the +/// second half of the claim by what the probe established — an enumerated +/// device is named, an installed-but-unenumerated driver gets a weaker +/// sentence, an unprobed platform admits it — and the unconditional "can still +/// be used" is exactly the promise this PR exists to stop making. See +/// `rocm_support_line` in `host_gpu/render.rs`. The two install refusals keep +/// it because they are asked of a *target*, before and without any host probe +/// (they classify with [`VulkanStatus::Unknown`]), so the graded wording has no +/// evidence to grade on there. /// /// Deliberately just the claim, with no leading subject and no trailing /// punctuation: a caller supplies both. Changing the wording here changes both @@ -645,6 +645,54 @@ pub enum GpuSupportLevel { pub const VULKAN_ONLY_SUPPORT_CLAIM: &str = "not supported by ROCm; the Vulkan backend can still be used"; +/// What every [`GpuSupportLevel::VulkanUnconfirmed`] surface says about the +/// host, written once. +/// +/// Two surfaces render this level — `rocm examine`'s `rocm_support` line +/// (`rocm_support_line` in `host_gpu/render.rs`) and the lemonade engine note +/// (`lemonade_engine_note` in `apps/rocm/src/main.rs`) — and they appear in the +/// *same* `rocm examine` report, a few lines apart. Sharing the claim is what +/// stops one of them denying a driver while the other says the question is +/// open. +/// +/// It states a gap in the probe, never a fact about the machine. On Windows the +/// ICD lookup reads only the legacy Khronos `Drivers` key and `vulkaninfo` is +/// rarely on PATH, so a working Adrenalin laptop lands in this level routinely; +/// anything stronger than "could not be determined" is false on that host. +pub const VULKAN_UNCONFIRMED_CLAIM: &str = + "whether a Vulkan driver is present could not be determined here"; + +/// The next step for a host at [`GpuSupportLevel::VulkanUnconfirmed`] when +/// nothing is known about its display driver. +/// +/// `vulkaninfo` comes first and the install is conditional on what it reports, +/// because the level is reached by hosts that already have a working driver. +/// An unconditional "install a Vulkan driver" is advice those hosts cannot act +/// on, and AGENTS.md §3 asks that a named command be able to clear the +/// condition it is printed for — running `vulkaninfo` can, on every host that +/// reaches here. +/// +/// Deliberately not "install `vulkaninfo`": on a driverless Linux host the tool +/// is typically already installed, prints "Found no drivers!" and exits 1, +/// which is one of the shapes that lands in this level. +pub const VULKAN_UNCONFIRMED_NEXT_STEP: &str = "run `vulkaninfo` yourself to see what it reports, and install a Vulkan driver (Mesa RADV on \ + Linux, the AMD display driver on Windows) if it finds none"; + +/// The next step for a Windows host at [`GpuSupportLevel::VulkanUnconfirmed`] +/// whose AMD display driver the summary already found. +/// +/// The Windows AMD display driver *is* the Vulkan driver on that platform — +/// Adrenalin installs the ICD — so `driver_status: amd_display_driver_detected` +/// contradicts the install advice in [`VULKAN_UNCONFIRMED_NEXT_STEP`]. Only +/// `vulkaninfo` is left to settle whether that driver exposes a device here. +/// +/// The cross-check is Windows-only on purpose. Linux's `driver_status` reports +/// the `amdgpu` kernel module, which says nothing about whether a userspace +/// Vulkan ICD such as Mesa RADV is installed, so suppressing the install advice +/// there would withhold the one step that can help. +pub const VULKAN_UNCONFIRMED_NEXT_STEP_WITH_DISPLAY_DRIVER: &str = "the AMD display driver is installed (see `driver_status`), so there is nothing to install \ + here - run `vulkaninfo` yourself to see whether it reports a device"; + /// Classify a GPU target's support level. /// /// Reads the two facts `ExamineSummary` and `resolve_family` already compute — @@ -669,9 +717,9 @@ pub const VULKAN_ONLY_SUPPORT_CLAIM: &str = /// must not be allowed to turn a detection gap into a support claim. /// /// [`VulkanStatus::Unknown`] — the platform could not be probed at all — keeps -/// `VulkanOnly`, because withholding the one path such a host has on the -/// strength of a probe that never ran would be worse than the old assumption, -/// not better. +/// `VulkanOnly`: withholding the one path such a host has, on the strength of a +/// probe that never ran, is a worse answer than the unconditional assumption it +/// replaces. pub fn gpu_support_level( detected_gfx_target: Option<&str>, compatible_therock_family: Option<&str>, @@ -682,19 +730,16 @@ pub fn gpu_support_level( (Some(gfx), None) if gfx_target_has_no_rocm_build(gfx) => { // `Unknown` only — a platform the probe cannot look at, which keeps // the benefit of the doubt. `Inconclusive` is a host that WAS - // looked at and showed nothing, and promising it Vulkan is the - // symptom this change exists to remove; it is the common - // no-driver shape, since most such hosts have no `vulkaninfo` - // either. The wording it gets is hedged rather than a confirmed - // absence — see `render.rs`. + // looked at and showed nothing, and must not be promised Vulkan: it + // is the common no-driver shape, since most such hosts have no + // `vulkaninfo` either. The wording it gets is hedged rather than a + // confirmed absence — see `render.rs`. // Every variant is spelled out, with no `_` and no guard. A guard - // (`v if v.is_usable()`) reads well but does not count toward - // exhaustiveness, so the arms below had to end in a wildcard — and - // a wildcard here silently routes any future `VulkanStatus` to - // `NoGpuBackend`, which renders as a confirmed fact. Listing them - // makes a new variant a compile error instead, the same discipline - // `render.rs` uses. The first arm is `VulkanStatus::is_usable` plus - // `Unknown`; keep them in step. + // reads well but does not count toward exhaustiveness, so the arms + // would have to end in a wildcard — and a wildcard here silently + // routes any future `VulkanStatus` to `NoGpuBackend`, which renders + // as a confirmed fact. Listing every variant makes a new one a + // compile error instead, the same discipline `render.rs` uses. match vulkan { // A device answered, or a driver is installed and nothing // contradicted it. diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index 96fdbeb2bf..c3c6e4b0c1 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -75,9 +75,9 @@ pub use linux::is_amdgpu_device; #[cfg(any(target_os = "linux", test))] pub(crate) use linux::{KfdGpuNode, kfd_gpu_nodes_in}; pub use vulkan::{ - VulkanDevice, VulkanStatus, VulkanSupport, amd_icd_driver_label, classify_vulkan_support, - detect_vulkan_support, icd_driver_label, icd_is_gpu_driver, parse_vulkaninfo_summary, - parse_windows_vulkan_icd_listing, windows_icds_from_capture, + VulkanDevice, VulkanEvidence, VulkanStatus, VulkanSupport, amd_icd_driver_label, + classify_vulkan_support, detect_vulkan_support, icd_driver_label, icd_is_gpu_driver, + parse_vulkaninfo_summary, parse_windows_vulkan_icd_listing, windows_icds_from_capture, }; pub(crate) use windows_inventory::detect_windows_amd_driver_info; #[cfg(test)] @@ -461,6 +461,16 @@ pub fn detect_gpu_driver_version() -> Option { None } +/// The [`DriverSummary::status`] word for a Windows host whose AMD display +/// driver was read. +/// +/// Written once because it is also a *fact about the Vulkan driver*: on Windows +/// the display driver installs the Vulkan ICD, so `render.rs` cross-checks this +/// word before advising an unconfirmed-Vulkan host to install one. Only +/// [`windows_driver_summary`] produces it, which is what makes matching on it a +/// Windows-only test. +pub(super) const AMD_DISPLAY_DRIVER_DETECTED: &str = "amd_display_driver_detected"; + fn windows_driver_summary( detail: Option, windows: Option, @@ -468,7 +478,7 @@ fn windows_driver_summary( DriverSummary { policy: "windows_validate_only".to_owned(), status: if detail.is_some() { - "amd_display_driver_detected".to_owned() + AMD_DISPLAY_DRIVER_DETECTED.to_owned() } else { "not_detected".to_owned() }, diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index 419968b5d9..aad6e12574 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -30,28 +30,41 @@ impl ExamineSummary { let Some(vulkan) = self.vulkan.as_ref() else { return "".to_owned(); }; - let status = match vulkan.status { - VulkanStatus::Available => "available", - VulkanStatus::DriverPresentDevicesUnverified => "driver_present_devices_unverified", - VulkanStatus::Absent => "absent", - VulkanStatus::Inconclusive => "inconclusive", - VulkanStatus::Unknown => "unknown", - }; + // `VulkanStatus::as_str`, not a second table here: this word and the + // one `--json` prints are the same status, and a local mapping lets the + // two renderings of one report drift apart silently. + let status = vulkan.status.as_str(); vulkan.detail.as_deref().map_or_else( || status.to_owned(), |detail| format!("{status} ({detail})"), ) } + /// What a [`GpuSupportLevel::VulkanUnconfirmed`] host is told to do next. + /// + /// `DriverSummary::status` only ever reads `AMD_DISPLAY_DRIVER_DETECTED` + /// on Windows — `windows_driver_summary` is the sole producer of that word + /// — so matching on it *is* the Windows cross-check, with no second read + /// of `os` to keep in step with it. On that platform the display driver + /// carries the Vulkan ICD, so its presence contradicts "install a Vulkan + /// driver" and only `vulkaninfo` is left to settle the question. + fn vulkan_unconfirmed_next_step(&self) -> &'static str { + if self.driver.status == super::AMD_DISPLAY_DRIVER_DETECTED { + crate::gfx_target::VULKAN_UNCONFIRMED_NEXT_STEP_WITH_DISPLAY_DRIVER + } else { + crate::gfx_target::VULKAN_UNCONFIRMED_NEXT_STEP + } + } + /// The `rocm_support` line of the examine text. /// /// ROCMAI-550 asks `rocm examine` to "report the support level with one /// line of what the user can do", so every level gets a line — including - /// the ones where the answer is "this build cannot tell". The earlier draft - /// printed only the `VulkanOnly` line, which made this report unable to - /// answer the very question the other surfaces send people here to ask: - /// `lemonade_engine_note` in `apps/rocm/src/main.rs` tells an `Unknown` - /// host to run `rocm examine`, and silence there is not an answer. + /// the ones where the answer is "this build cannot tell". Printing only the + /// `VulkanOnly` line leaves this report unable to answer the very question + /// the other surfaces send people here to ask: `lemonade_engine_note` in + /// `apps/rocm/src/main.rs` tells an `Unknown` host to run `rocm examine`, + /// and silence there is not an answer. /// /// Without the `VulkanOnly` line specifically, /// `compatible_therock_family: ` reads like a probe failure when @@ -112,23 +125,29 @@ impl ExamineSummary { "the Vulkan backend is unconfirmed".to_owned(), } ), - // Looked at, nothing found, and nothing that could have found it. - // Says what was searched and how to settle it, rather than either - // promising a backend or declaring one absent — neither of which - // the evidence supports. See `GpuSupportLevel::VulkanUnconfirmed`. + // Looked at and nothing settled either way. Says the probe could + // not determine it and how to settle it, rather than promising a + // backend or declaring one absent — neither of which the evidence + // supports. See `GpuSupportLevel::VulkanUnconfirmed`. + // + // The claim comes from `VULKAN_UNCONFIRMED_CLAIM` because + // `lemonade_engine_note` renders the same level a few lines further + // down the same report; one constant is what keeps the two from + // contradicting each other. + // + // The next step is cross-checked against `driver.status`, the one + // other fact on this summary that bears on it. On Windows the AMD + // display driver *is* the Vulkan driver, so a host that already has + // it must not be told to install one — the exact misdirection this + // level exists to avoid, reached through its own advice. The + // cross-check is Windows-only: Linux's `driver_status` reports the + // `amdgpu` kernel module, which says nothing about whether a + // userspace ICD is installed. GpuSupportLevel::VulkanUnconfirmed => format!( - // Deliberately does NOT say "install `vulkaninfo`". Measured on - // a real driverless Linux host: `vulkaninfo --summary` is - // installed, prints "Found no drivers!" and exits 1 — which - // lands here. Telling that user to install the tool they are - // already running is advice that can never clear the state. - // Install a *driver* is the step that can. - " rocm_support: GPU detected ({}) but not supported by ROCm; no Vulkan driver \ - was found, and `vulkaninfo` could not confirm that either (not installed, or \ - it failed to run) - install a Vulkan driver (Mesa RADV on Linux, the AMD \ - display driver on Windows) to run models on this GPU, and run `vulkaninfo` \ - yourself to see what it says\n", - detected.unwrap_or_default() + " rocm_support: GPU detected ({}) but not supported by ROCm; {} - {}\n", + detected.unwrap_or_default(), + crate::gfx_target::VULKAN_UNCONFIRMED_CLAIM, + self.vulkan_unconfirmed_next_step(), ), // Both halves are established facts: the catalog confirmed there is // no ROCm build, and the probe looked for a Vulkan driver and found @@ -517,11 +536,11 @@ mod tests { ); } - /// The defect this guards against: the `VulkanOnly` line used to promise - /// "the Vulkan backend can still be used" on every gfx90c host, derived from - /// the gfx table alone. A Renoir laptop with no Vulkan driver installed was - /// told to go and serve a model on a backend that cannot start — and under - /// the GPU-required policy there is nothing for it to fall back to. + /// The defect this guards against: promising "the Vulkan backend can still + /// be used" on every gfx90c host, from the gfx table alone. That sends a + /// Renoir laptop with no Vulkan driver installed to serve a model on a + /// backend that cannot start — and under the GPU-required policy there is + /// nothing for it to fall back to. #[test] fn examine_render_does_not_promise_vulkan_on_a_host_that_has_none() { let summary = gfx90c_summary_with_vulkan(Some(VulkanSupport { @@ -614,8 +633,9 @@ mod tests { let summary = gfx90c_summary_with_vulkan(Some(VulkanSupport { status: VulkanStatus::Inconclusive, detail: Some( - "no Vulkan driver was found, and `vulkaninfo` could not confirm that either (not \ - installed, or it failed to run), so nothing here confirms one is absent" + "no installed Vulkan driver was listed, and `vulkaninfo` could not confirm \ + anything either (not installed, or it failed to run), so nothing here shows \ + whether a driver is present" .to_owned(), ), ..VulkanSupport::default() @@ -624,8 +644,8 @@ mod tests { let rendered = summary.render_text(); assert!( - rendered.contains("`vulkaninfo` could not confirm"), - "it must name the gap that stopped it confirming: {rendered}" + rendered.contains(crate::gfx_target::VULKAN_UNCONFIRMED_CLAIM), + "it must say the probe could not settle it: {rendered}" ); assert!( !rendered.contains("the Vulkan backend can be used"), @@ -639,16 +659,92 @@ mod tests { assert!(rendered.contains("vulkan: inconclusive"), "{rendered}"); } + /// The Windows machine this level exists for: the Khronos `Drivers` key it + /// reads is legacy and `vulkaninfo` is rarely on PATH, so an **Adrenalin + /// laptop with a working Vulkan driver** lands in `VulkanUnconfirmed`. + /// Telling it to install the AMD display driver is advice it has already + /// taken, and nothing it does can clear the state. + /// + /// `driver_status` is the summary's own evidence against that advice, and + /// on Windows the display driver carries the Vulkan ICD, so its presence is + /// decisive. The line must say so instead, and still name the one step that + /// can settle the question. + #[test] + fn examine_render_does_not_tell_a_windows_host_to_install_the_driver_it_has() { + let mut summary = gfx90c_summary_with_vulkan(Some(VulkanSupport { + status: VulkanStatus::Inconclusive, + detail: Some("the legacy Khronos key listed nothing".to_owned()), + ..VulkanSupport::default() + })); + summary.os = "windows".to_owned(); + summary.driver = DriverSummary { + policy: "windows_validate_only".to_owned(), + status: super::super::AMD_DISPLAY_DRIVER_DETECTED.to_owned(), + detail: Some("AMD display driver 31.0.21925.1001".to_owned()), + windows: None, + }; + + let rendered = summary.render_text(); + let support = rendered + .lines() + .find(|line| line.contains("rocm_support:")) + .expect("every host gets a support line"); + + assert!( + !support.contains("install a Vulkan driver"), + "this host already has the driver that would be installed: {support}" + ); + assert!( + support.contains("the AMD display driver is installed"), + "and it is owed the reason that advice does not apply: {support}" + ); + assert!( + support.contains("run `vulkaninfo`"), + "the one step that can still settle it must stay: {support}" + ); + assert!( + support.contains(crate::gfx_target::VULKAN_UNCONFIRMED_CLAIM), + "the claim is the same on both platforms; only the next step differs: {support}" + ); + + // The Linux shape of the same status, which is what makes the two halves + // a discrimination rather than one assertion twice. `driver` has to be + // overwritten explicitly: the shared fixture is a Windows host and + // already carries `AMD_DISPLAY_DRIVER_DETECTED`, so reusing it unchanged + // would re-enter the branch above and assert nothing about Linux. + // + // There `driver_status` reports the `amdgpu` kernel module, which says + // nothing about whether a userspace ICD such as Mesa RADV is installed, + // so the install advice is the true one and must survive. + let mut linux = gfx90c_summary_with_vulkan(Some(VulkanSupport { + status: VulkanStatus::Inconclusive, + detail: Some("no ICD manifest was found".to_owned()), + ..VulkanSupport::default() + })); + linux.os = "linux".to_owned(); + linux.driver = DriverSummary { + policy: "amdgpu_dkms".to_owned(), + status: "amdgpu_module_loaded".to_owned(), + detail: None, + windows: None, + }; + let linux = linux.render_text(); + assert!( + linux.contains(crate::gfx_target::VULKAN_UNCONFIRMED_NEXT_STEP), + "a host with no display-driver evidence still gets the install step: {linux}" + ); + assert!( + !linux.contains("the AMD display driver is installed"), + "and must not be told about a Windows driver it cannot have: {linux}" + ); + } + /// An unmapped target must not be *claimed* unsupported, even though every /// host now gets a `rocm_support` line. /// - /// This replaces `examine_render_omits_the_support_note_for_supported_or_missing_gpus`, - /// which asserted the line was absent for supported, missing and unmapped - /// GPUs. ROCMAI-550 deliberately reverses the absence: silence could not - /// answer the question `lemonade_engine_note` sends users here to ask. The - /// half of that test worth keeping is the discrimination it protected — - /// "no ROCm build" is a fact about gfx90c, and a gap in this catalog is not - /// the same claim about gfx803. + /// Every host gets the line, so what this protects is the discrimination + /// rather than the silence: "no ROCm build" is a fact about gfx90c, and a + /// gap in this catalog is not the same claim about gfx803. #[test] fn examine_render_never_claims_an_unmapped_target_is_unsupported() { for (gfx, family) in [ diff --git a/crates/rocm-core/src/host_gpu/vulkan.rs b/crates/rocm-core/src/host_gpu/vulkan.rs index 842652f82e..825181647a 100644 --- a/crates/rocm-core/src/host_gpu/vulkan.rs +++ b/crates/rocm-core/src/host_gpu/vulkan.rs @@ -85,10 +85,86 @@ pub enum VulkanStatus { } impl VulkanStatus { - /// Whether a model may be routed through Vulkan on the strength of this. + /// The one spelling of this status, shared by the `vulkan:` text line and + /// the `--json` document. + /// + /// Byte-identical to what `#[serde(rename_all = "snake_case")]` writes, so + /// the two forms of the same report cannot disagree about what the probe + /// found; `the_status_word_is_the_one_serde_writes` holds that. + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Available => "available", + Self::DriverPresentDevicesUnverified => "driver_present_devices_unverified", + Self::Absent => "absent", + Self::Inconclusive => "inconclusive", + Self::Unknown => "unknown", + } + } +} + +/// What one evidence source came back with: whether the lookup **answered**, +/// and what it found. +/// +/// The classification turns on the first half, so the two halves travel +/// together and an empty `Vec` can never be read as an answer by accident. The +/// two sources are different instantiations — `VulkanEvidence` for ICD +/// manifests, `VulkanEvidence` for enumerated devices — so +/// handing [`classify_vulkan_support`] one in the other's place is a type error +/// rather than a silent flip between `Inconclusive` and `Absent`. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct VulkanEvidence { + found: Vec, + answered: bool, +} + +impl VulkanEvidence { + /// The lookup ran and this is what it saw. An empty list from here is a + /// real "nothing is installed", which is the only thing that can earn + /// [`VulkanStatus::Absent`]. + #[must_use] + pub const fn answered(found: Vec) -> Self { + Self { + found, + answered: true, + } + } + + /// The lookup did not answer — the tool is missing, exited non-zero, timed + /// out, or this platform has nothing to read. Nothing it "found" could be + /// evidence, so it carries no findings at all. + #[must_use] + pub const fn unanswered() -> Self { + Self { + found: Vec::new(), + answered: false, + } + } + + /// Whether this source answered at all, as opposed to finding nothing. + #[must_use] + pub const fn did_answer(&self) -> bool { + self.answered + } + + #[must_use] + pub fn found(&self) -> &[T] { + &self.found + } + + /// The findings, for a caller that is done asking whether they are + /// evidence. #[must_use] - pub const fn is_usable(self) -> bool { - matches!(self, Self::Available | Self::DriverPresentDevicesUnverified) + pub fn into_found(self) -> Vec { + self.found + } +} + +impl Default for VulkanEvidence { + /// An unfilled source is one nobody asked, for the same reason + /// [`VulkanStatus`] defaults to `Unknown`. + fn default() -> Self { + Self::unanswered() } } @@ -168,17 +244,17 @@ impl VulkanSupport { // Restricted to AMD-branded manifests, because this fallback // only ever runs once an AMD hardware device has *already* been // confirmed: whatever it names is read as the driver behind - // that device. "Not a known software rasteriser" was too weak a + // that device. "Not a known software rasteriser" is too weak a // filter for that job — `detect_icds` sorts alphabetically on - // Linux, so a dual-vendor host carrying both `nvidia_icd.json` - // and `radeon_icd.x86_64.json` had the NVIDIA manifest win the - // `find_map` and a confirmed AMD Vulkan path was attributed to - // NVIDIA's driver. + // Linux, so on a dual-vendor host carrying both + // `nvidia_icd.json` and `radeon_icd.x86_64.json` the NVIDIA + // manifest wins the `find_map`, and a confirmed AMD Vulkan path + // is attributed to NVIDIA's driver. // - // Narrowing to AMD subsumes the software-rasteriser exclusion - // this replaces (lavapipe and SwiftShader are not AMD labels), - // so the "a software rasteriser never counts as a GPU path" - // invariant still holds in the one place the user actually + // Narrowing to AMD subsumes that weaker filter's + // software-rasteriser exclusion (lavapipe and SwiftShader are + // not AMD labels), so "a software rasteriser never counts as a + // GPU path" still holds in the one place the user actually // reads a driver name. self.icds .iter() @@ -318,16 +394,15 @@ pub fn parse_windows_vulkan_icd_listing(output: &str) -> Vec { #[cfg(any(windows, test))] pub(super) const WINDOWS_VULKAN_ICD_SCRIPT: &str = r#"foreach ($key in 'HKLM:\SOFTWARE\Khronos\Vulkan\Drivers','HKLM:\SOFTWARE\WOW6432Node\Khronos\Vulkan\Drivers') { if (Test-Path $key) { foreach ($name in (Get-Item $key).GetValueNames()) { "ICD`t$name" } } }"#; -/// Every Vulkan ICD manifest this host's loader would read, and whether the -/// lookup **ran** — the ICD-side counterpart of `enumerated`. +/// Every Vulkan ICD manifest this host's loader would read, as the ICD-side +/// [`VulkanEvidence`]. /// /// On Linux the lookup is a read of a fixed list of manifest directories, and /// there is no step in it that can fail in a way distinguishable from an honest /// "nothing is installed here": a directory that does not exist is exactly what -/// a host with no Vulkan driver looks like. So the flag is unconditionally -/// `true` — the lookup always happens — rather than a placeholder. +/// a host with no Vulkan driver looks like. So this source always answers. #[cfg(target_os = "linux")] -fn detect_icds() -> (Vec, bool) { +fn detect_icds() -> VulkanEvidence { let mut icds = Vec::new(); for dir in LINUX_ICD_DIRS { let Ok(entries) = std::fs::read_dir(host_path(dir)) else { @@ -344,18 +419,18 @@ fn detect_icds() -> (Vec, bool) { } icds.sort(); icds.dedup(); - (icds, true) + VulkanEvidence::answered(icds) } /// On Windows the lookup is a PowerShell read of the Khronos registry keys, and /// that genuinely can fail: `capture_optional_command` yields `None` for a -/// missing `powershell`, a non-zero exit **and** a timeout alike. Collapsing -/// that `None` into the empty `Vec` an empty registry key produces made a flaky -/// PowerShell on a host with a working AMD driver read identically to a host -/// with no driver at all, which is the conflation `enumerated` exists to prevent -/// on the `vulkaninfo` side. +/// missing `powershell`, a non-zero exit **and** a timeout alike. That `None` +/// must stay distinct from the empty `Vec` an empty registry key produces, +/// otherwise a flaky PowerShell on a host with a working AMD driver reads +/// identically to a host with no driver at all — the conflation the device +/// side's "did it answer" exists to prevent. #[cfg(windows)] -fn detect_icds() -> (Vec, bool) { +fn detect_icds() -> VulkanEvidence { windows_icds_from_capture(capture_optional_command( "powershell", &[ @@ -369,12 +444,12 @@ fn detect_icds() -> (Vec, bool) { } /// Nothing here can be read on a platform that is neither Linux nor Windows, so -/// the flag says the lookup did not run. The classifier never sees it today — +/// this source never answers. The classifier never sees it today — /// `vulkan_is_probeable()` is false there and returns [`VulkanStatus::Unknown`] /// first — but it must not claim a lookup happened where none exists. #[cfg(not(any(target_os = "linux", windows)))] -const fn detect_icds() -> (Vec, bool) { - (Vec::new(), false) +const fn detect_icds() -> VulkanEvidence { + VulkanEvidence::unanswered() } /// The pure half of the Windows [`detect_icds`]: the manifests the capture @@ -386,17 +461,16 @@ const fn detect_icds() -> (Vec, bool) { /// is in hand, so an empty list from it is a real "the key holds no ICD"; `None` /// is the un-run lookup that must not be reported as one. #[must_use] -pub fn windows_icds_from_capture(captured: Option) -> (Vec, bool) { - captured.map_or_else( - || (Vec::new(), false), - |output| (parse_windows_vulkan_icd_listing(&output), true), - ) +pub fn windows_icds_from_capture(captured: Option) -> VulkanEvidence { + captured.map_or_else(VulkanEvidence::unanswered, |output| { + VulkanEvidence::answered(parse_windows_vulkan_icd_listing(&output)) + }) } /// The devices `vulkaninfo --summary` reports, or an empty list when the tool /// is not installed. Absence of the tool is not absence of a driver, which is /// why this cannot decide [`VulkanStatus`] on its own. -fn detect_vulkan_devices() -> (Vec, bool) { +fn detect_vulkan_devices() -> VulkanEvidence { vulkan_devices_from_capture(capture_optional_command("vulkaninfo", &["--summary"])) } @@ -404,22 +478,22 @@ fn detect_vulkan_devices() -> (Vec, bool) { /// and whether `vulkaninfo` actually **answered**. /// /// The classification turns on "could we ask" versus "we asked and nothing was -/// there", and the second half of that is what this bool carries. It is -/// deliberately *not* "the binary exists": `capture_optional_command` yields +/// there", and the second half of that is what `did_answer` carries. Answering +/// is deliberately *not* "the binary exists": `capture_optional_command` yields /// `Some` only for an exit-0 run, so a `vulkaninfo` that exits non-zero, times -/// out, or is too old to know `--summary` is `None` here — none of which is a -/// host that was looked at and found empty. +/// out, or is too old to know `--summary` arrives here as a source that +/// answered nothing — none of which is a host that was looked at and found +/// empty. /// -/// Using mere presence on `PATH` for this was wrong in the direction that -/// matters: on a host whose ICD sits outside the fixed search directories, a -/// failing `vulkaninfo` produced a *confirmed* `Absent` and told the user "no -/// Vulkan GPU driver was found - install one", which is the unearned claim -/// this probe exists to stop making. -fn vulkan_devices_from_capture(captured: Option) -> (Vec, bool) { - captured.map_or_else( - || (Vec::new(), false), - |output| (parse_vulkaninfo_summary(&output), true), - ) +/// Mere presence on `PATH` is the wrong test, in the direction that matters: on +/// a host whose ICD sits outside the fixed search directories, a failing +/// `vulkaninfo` would then produce a *confirmed* `Absent` and tell the user "no +/// Vulkan GPU driver was found - install one", which is the unearned claim this +/// probe exists to stop making. +fn vulkan_devices_from_capture(captured: Option) -> VulkanEvidence { + captured.map_or_else(VulkanEvidence::unanswered, |output| { + VulkanEvidence::answered(parse_vulkaninfo_summary(&output)) + }) } /// Whether this platform's Vulkan surfaces can be read at all. @@ -430,32 +504,30 @@ const fn vulkan_is_probeable() -> bool { /// What this host offers for Vulkan. #[must_use] pub fn detect_vulkan_support() -> VulkanSupport { - let (icds, icds_listed) = detect_icds(); - let (devices, enumerated) = detect_vulkan_devices(); classify_vulkan_support( - icds, - devices, + detect_icds(), + detect_vulkan_devices(), vulkan_is_probeable(), - icds_listed, - enumerated, ) } /// The pure decision behind [`detect_vulkan_support`], split out so every /// combination of evidence can be tested without a Vulkan install. /// -/// `icds_listed` and `enumerated` are the two "did this source answer at all" -/// flags, one per source, and they are what keep "could not look" apart from -/// "looked and found nothing". Both must be true before an empty result may be +/// Each source arrives as a [`VulkanEvidence`] carrying its own "did this +/// answer at all", which is what keeps "could not look" apart from "looked and +/// found nothing". Both must have answered before an empty result may be /// reported as a confirmed absence. #[must_use] pub fn classify_vulkan_support( - icds: Vec, - devices: Vec, + icd_evidence: VulkanEvidence, + device_evidence: VulkanEvidence, probeable: bool, - icds_listed: bool, - enumerated: bool, ) -> VulkanSupport { + let icds_listed = icd_evidence.did_answer(); + let enumerated = device_evidence.did_answer(); + let icds = icd_evidence.into_found(); + let devices = device_evidence.into_found(); if !probeable { return VulkanSupport { status: VulkanStatus::Unknown, @@ -516,15 +588,22 @@ pub fn classify_vulkan_support( // result may be called an absence. if icds.is_empty() && devices.is_empty() && !(icds_listed && enumerated) { // Which source went unread decides the sentence: a host is owed the - // reason nothing could be concluded about it, and "no Vulkan driver - // was found" is a claim only a lookup that ran may make. + // reason nothing could be concluded about it, and "a Vulkan driver + // is absent" is a claim no single source here may make. + // + // None of the three denies a driver, the arm where the manifest + // lookup *did* answer included. On Windows that lookup reads only + // the legacy Khronos key, which a current Adrenalin install leaves + // empty, so "no manifest was listed" is routinely true of a machine + // with a perfectly good Vulkan driver. let detail = match (icds_listed, enumerated) { // The lookup ran and found no manifest; only `vulkaninfo` is // missing. (`(true, true)` cannot reach here — it is the // confirmed absence handled below.) (true, _) => { - "no Vulkan driver was found, and `vulkaninfo` could not confirm that either \ - (not installed, or it failed to run), so nothing here confirms one is absent" + "no installed Vulkan driver was listed, and `vulkaninfo` could not confirm \ + anything either (not installed, or it failed to run), so nothing here shows \ + whether a driver is present" } (false, true) => { "`vulkaninfo` ran and listed no device, but the installed Vulkan drivers could \ @@ -543,19 +622,13 @@ pub fn classify_vulkan_support( devices, }; } - // A device that enumerated is proof a driver answered, so "no Vulkan - // driver found" would be false here even though no ICD manifest was - // read: the ICD search is not exhaustive, and `vulkaninfo` can see a - // driver it misses. Say what was actually established — hardware - // answered, none of it AMD — rather than denying the driver. - // A device that enumerated is proof a driver answered, so this host is - // not one where "no Vulkan GPU driver was found - install one" is true. - // Fixing only the detail text last round left the *status* saying the - // opposite of the detail beside it: `Absent` renders that install - // advice, while the `vulkan:` line said devices had been enumerated. - // It reports what was actually established instead — a driver answered, - // and none of what it listed is AMD, which is the same evidentiary - // state as an installed driver that enumerated nothing for us. + // A device that enumerated is proof a driver answered, so neither the + // status nor the detail may deny one here, even though no ICD manifest + // was read: the ICD search is not exhaustive, and `vulkaninfo` can see + // a driver it misses. `Absent` is the status that renders the "install + // one" advice, so it must not be used; what was actually established — + // hardware answered, none of it AMD — is the same evidentiary state as + // an installed driver that enumerated nothing for us. if !devices.is_empty() { return VulkanSupport { status: VulkanStatus::DriverPresentDevicesUnverified, @@ -634,6 +707,28 @@ pub fn classify_vulkan_support( mod tests { use super::*; + /// The `vulkan:` text line and the `--json` document are two renderings of + /// one status, so they must spell it the same way. Holding `as_str` against + /// what serde writes is what makes that true by construction rather than by + /// two hand-maintained tables. + #[test] + fn the_status_word_is_the_one_serde_writes() { + for status in [ + VulkanStatus::Available, + VulkanStatus::DriverPresentDevicesUnverified, + VulkanStatus::Absent, + VulkanStatus::Inconclusive, + VulkanStatus::Unknown, + ] { + let json = serde_json::to_string(&status).expect("a status always serializes"); + assert_eq!( + json, + format!("\"{}\"", status.as_str()), + "the text line and the JSON must name {status:?} identically" + ); + } + } + /// The real `vulkaninfo --summary` output from a Ryzen 7 4700U (gfx90c, /// Renoir) on Windows 11 with Adrenalin 31.0.21925.1001 -- the machine /// ROCMAI-517 is about. Note the loader warnings ahead of the report and @@ -692,10 +787,18 @@ GPU0: fn a_real_renoir_host_reports_vulkan_available_and_names_its_driver() { let devices = parse_vulkaninfo_summary(RENOIR_WINDOWS_SUMMARY); - let support = classify_vulkan_support(Vec::new(), devices, true, true, true); + let support = classify_vulkan_support( + VulkanEvidence::answered(Vec::new()), + VulkanEvidence::answered(devices), + true, + ); assert_eq!(support.status, VulkanStatus::Available); - assert!(support.status.is_usable()); + assert_eq!( + crate::gfx_target::gpu_support_level(Some("gfx90c"), None, support.status), + crate::gfx_target::GpuSupportLevel::VulkanOnly, + "a confirmed AMD device is the one thing that earns this GPU a Vulkan path" + ); assert_eq!( support.driver_label().as_deref(), Some("AMD proprietary driver"), @@ -723,10 +826,8 @@ GPU0: assert!(!devices[0].is_hardware()); let support = classify_vulkan_support( - vec!["lvp_icd.x86_64.json".to_owned()], - devices, - true, - true, + VulkanEvidence::answered(vec!["lvp_icd.x86_64.json".to_owned()]), + VulkanEvidence::answered(devices), true, ); @@ -735,7 +836,11 @@ GPU0: VulkanStatus::Absent, "a CPU rasteriser must not read as a Vulkan GPU path: {support:?}" ); - assert!(!support.status.is_usable()); + assert_eq!( + crate::gfx_target::gpu_support_level(Some("gfx90c"), None, support.status), + crate::gfx_target::GpuSupportLevel::NoGpuBackend, + "and the level the user is shown must not offer one either: {support:?}" + ); assert!( support .detail @@ -748,10 +853,8 @@ GPU0: #[test] fn a_lavapipe_only_install_is_absent_even_before_any_device_is_enumerated() { let support = classify_vulkan_support( - vec!["lvp_icd.x86_64.json".to_owned()], - Vec::new(), - true, - true, + VulkanEvidence::answered(vec!["lvp_icd.x86_64.json".to_owned()]), + VulkanEvidence::answered(Vec::new()), true, ); @@ -761,25 +864,23 @@ GPU0: } /// Both halves of this test reach the same *status*; what they must not - /// share is the *sentence*. `enumerated` is the module's central piece of - /// evidence everywhere else, and this arm used to ignore it: a host whose - /// `vulkaninfo` had already run and already listed nothing was still told - /// to "install `vulkaninfo` to confirm". Asserting the two texts differ, - /// rather than asserting each in isolation, is what keeps the advice tied - /// to the evidence. + /// share is the *sentence*. A host whose `vulkaninfo` has already run and + /// listed nothing must not be told to "install `vulkaninfo` to confirm" — + /// that advice can never clear the state it is printed for. Asserting the + /// two texts differ, rather than asserting each in isolation, is what keeps + /// the advice tied to the evidence. #[test] fn an_installed_driver_with_no_enumerated_device_stays_unverified() { let asked = classify_vulkan_support( - vec!["radeon_icd.x86_64.json".to_owned()], - Vec::new(), - true, - true, + VulkanEvidence::answered(vec!["radeon_icd.x86_64.json".to_owned()]), + VulkanEvidence::answered(Vec::new()), true, ); assert_eq!(asked.status, VulkanStatus::DriverPresentDevicesUnverified); - assert!( - asked.status.is_usable(), + assert_eq!( + crate::gfx_target::gpu_support_level(Some("gfx90c"), None, asked.status), + crate::gfx_target::GpuSupportLevel::VulkanOnly, "an installed GPU driver is still worth routing to" ); assert_eq!(asked.driver_label().as_deref(), Some("Mesa RADV")); @@ -797,11 +898,9 @@ GPU0: // Same ICD, same (empty) device list, but nothing asked: here the tool // really may be missing, so the install advice is the true one. let unasked = classify_vulkan_support( - vec!["radeon_icd.x86_64.json".to_owned()], - Vec::new(), - true, + VulkanEvidence::answered(vec!["radeon_icd.x86_64.json".to_owned()]), + VulkanEvidence::unanswered(), true, - false, ); assert_eq!(unasked.status, VulkanStatus::DriverPresentDevicesUnverified); @@ -827,7 +926,11 @@ GPU0: fn no_evidence_either_way_is_not_reported_as_a_confirmed_absence() { // `vulkaninfo` absent: nothing could have found a driver, so nothing // may claim there is none. - let unasked = classify_vulkan_support(Vec::new(), Vec::new(), true, true, false); + let unasked = classify_vulkan_support( + VulkanEvidence::answered(Vec::new()), + VulkanEvidence::unanswered(), + true, + ); assert_eq!( unasked.status, VulkanStatus::Inconclusive, @@ -844,7 +947,11 @@ GPU0: // `vulkaninfo` present and it found nothing: that IS the answer, and // it is the shape a host genuinely without a driver takes. - let nothing = classify_vulkan_support(Vec::new(), Vec::new(), true, true, true); + let nothing = classify_vulkan_support( + VulkanEvidence::answered(Vec::new()), + VulkanEvidence::answered(Vec::new()), + true, + ); assert_eq!( nothing.status, VulkanStatus::Absent, @@ -854,21 +961,28 @@ GPU0: // What `Absent` is now reserved for: something WAS found and ruled out. let software_only = classify_vulkan_support( - vec!["lvp_icd.x86_64.json".to_owned()], - Vec::new(), - true, - true, + VulkanEvidence::answered(vec!["lvp_icd.x86_64.json".to_owned()]), + VulkanEvidence::answered(Vec::new()), true, ); assert_eq!(software_only.status, VulkanStatus::Absent); // macOS and friends: no evidence either way, which must not read as // "no Vulkan" any more than it reads as "Vulkan works". - // An unprobeable platform is the one place `icds_listed` is false + // An unprobeable platform is the one place a source goes unanswered // without a failure: the `cfg` stub there has nothing it could read. - let unprobeable = classify_vulkan_support(Vec::new(), Vec::new(), false, false, false); + let unprobeable = classify_vulkan_support( + VulkanEvidence::unanswered(), + VulkanEvidence::unanswered(), + false, + ); assert_eq!(unprobeable.status, VulkanStatus::Unknown); - assert!(!unprobeable.status.is_usable()); + assert_eq!( + crate::gfx_target::gpu_support_level(Some("gfx90c"), None, unprobeable.status), + crate::gfx_target::GpuSupportLevel::VulkanOnly, + "a platform nobody could look at keeps the benefit of the doubt, which is \ + what tells `Unknown` apart from `Inconclusive`" + ); } #[test] @@ -879,10 +993,8 @@ GPU0: assert_eq!(icd_driver_label("some_new_vendor_icd.x86_64.json"), None); let support = classify_vulkan_support( - vec!["some_new_vendor_icd.x86_64.json".to_owned()], - Vec::new(), - true, - true, + VulkanEvidence::answered(vec!["some_new_vendor_icd.x86_64.json".to_owned()]), + VulkanEvidence::answered(Vec::new()), true, ); assert_eq!(support.status, VulkanStatus::DriverPresentDevicesUnverified); @@ -903,10 +1015,8 @@ GPU1: \tdriverName = radv "; let support = classify_vulkan_support( - Vec::new(), - parse_vulkaninfo_summary(mixed), - true, - true, + VulkanEvidence::answered(Vec::new()), + VulkanEvidence::answered(parse_vulkaninfo_summary(mixed)), true, ); @@ -924,11 +1034,10 @@ GPU1: /// Pins a **known limitation**, not a desired behavior: an enumeration that /// returned only non-AMD hardware is not treated as evidence against an AMD /// Vulkan path, so such a host is still routed to `VulkanOnly` and told to - /// serve a model. An earlier name for this test claimed the opposite ("does - /// not make Vulkan an AMD path") while asserting exactly this, which hid the - /// limitation behind a reassuring name. Asserting the routing too, rather - /// than only the status, is what keeps that from happening again: if this is - /// ever tightened, this test fails and has to be rewritten deliberately. + /// serve a model. The name has to say that rather than imply the opposite, + /// and the routing is asserted alongside the status so the limitation is + /// visible here: if it is ever tightened, this test fails and has to be + /// rewritten deliberately. #[test] fn a_non_amd_only_enumeration_is_still_routed_to_vulkan_today() { let nvidia = "\ @@ -939,15 +1048,13 @@ GPU0: \tdriverName = NVIDIA "; let support = classify_vulkan_support( - vec!["nvidia_icd.json".to_owned()], - parse_vulkaninfo_summary(nvidia), - true, - true, + VulkanEvidence::answered(vec!["nvidia_icd.json".to_owned()]), + VulkanEvidence::answered(parse_vulkaninfo_summary(nvidia)), true, ); // The driver is installed and is not software, so the status stops short - // of "available" — but it is still `is_usable()`. + // of "available" while still routing this host to Vulkan. assert_eq!(support.status, VulkanStatus::DriverPresentDevicesUnverified); assert_eq!( crate::gfx_target::gpu_support_level(Some("gfx90c"), None, support.status), @@ -962,32 +1069,31 @@ GPU0: /// /// `capture_optional_command` collapses "not installed", "exited non-zero", /// "timed out" and "too old for `--summary`" into one `None`, and **none** - /// of those is a host that was looked at. Reading presence on `PATH` as - /// "it answered" meant a failing `vulkaninfo`, on a host whose ICD sits - /// outside the fixed search directories, produced a confirmed `Absent` and - /// told the user to install a driver they already had. + /// of those is a host that was looked at. Presence on `PATH` is therefore + /// not the test: were it, a failing `vulkaninfo` on a host whose ICD sits + /// outside the fixed search directories would produce a confirmed `Absent` + /// and tell the user to install a driver they already have. #[test] fn only_a_vulkaninfo_that_answered_counts_as_having_looked() { - let (devices, enumerated) = vulkan_devices_from_capture(None); + let devices = vulkan_devices_from_capture(None); assert!( - !enumerated && devices.is_empty(), + !devices.did_answer() && devices.found().is_empty(), "a run that produced no output did not look at this host" ); assert_eq!( - classify_vulkan_support(Vec::new(), devices, true, true, enumerated).status, + classify_vulkan_support(VulkanEvidence::answered(Vec::new()), devices, true).status, VulkanStatus::Inconclusive, "so it must not be reported as a confirmed absence" ); // The genuine "looked and found none": exit 0, empty device list. - let (devices, enumerated) = - vulkan_devices_from_capture(Some("Devices:\n=======\n".to_owned())); + let devices = vulkan_devices_from_capture(Some("Devices:\n=======\n".to_owned())); assert!( - enumerated && devices.is_empty(), + devices.did_answer() && devices.found().is_empty(), "an empty answer is still an answer" ); assert_eq!( - classify_vulkan_support(Vec::new(), devices, true, true, enumerated).status, + classify_vulkan_support(VulkanEvidence::answered(Vec::new()), devices, true).status, VulkanStatus::Absent, "and that one has earned the confirmed absence" ); @@ -995,8 +1101,8 @@ GPU0: /// A device that enumerated is proof a driver answered, so the report must /// not deny one just because no ICD manifest was read: the ICD search is a - /// fixed directory list that `vulkaninfo` can see past. The earlier wording - /// said "no Vulkan driver found" on exactly this host, which is false. + /// fixed directory list that `vulkaninfo` can see past. "No Vulkan driver + /// found" is false on exactly this host. #[test] fn an_enumerated_non_amd_device_is_not_reported_as_a_missing_driver() { let nvidia = "\ @@ -1006,13 +1112,11 @@ GPU0: \tdeviceName = NVIDIA GeForce GTX 1050 Ti \tdriverName = NVIDIA "; - // No ICD manifest at all, so the pre-fix code fell to "no Vulkan driver - // found" despite the enumeration above. + // No ICD manifest at all: the one shape in which a denial of the driver + // could slip past the enumeration above. let support = classify_vulkan_support( - Vec::new(), - parse_vulkaninfo_summary(nvidia), - true, - true, + VulkanEvidence::answered(Vec::new()), + VulkanEvidence::answered(parse_vulkaninfo_summary(nvidia)), true, ); @@ -1110,9 +1214,17 @@ GPU0: #[test] fn a_failed_icd_lookup_is_not_reported_as_a_host_without_a_driver() { // Nothing read the registry, and `vulkaninfo` did not answer either. - let unread = classify_vulkan_support(Vec::new(), Vec::new(), true, false, false); + let unread = classify_vulkan_support( + VulkanEvidence::unanswered(), + VulkanEvidence::unanswered(), + true, + ); // Same host, except the lookup ran and genuinely found no manifest. - let read = classify_vulkan_support(Vec::new(), Vec::new(), true, true, false); + let read = classify_vulkan_support( + VulkanEvidence::answered(Vec::new()), + VulkanEvidence::unanswered(), + true, + ); assert_eq!(unread.status, VulkanStatus::Inconclusive); assert_eq!(read.status, VulkanStatus::Inconclusive); @@ -1127,8 +1239,14 @@ GPU0: "the host is owed the reason nothing could be concluded about it: {unread:?}" ); assert!( - read_detail.contains("no Vulkan driver was found"), - "a lookup that ran and found nothing is entitled to say so: {read:?}" + read_detail.contains("no installed Vulkan driver was listed"), + "a lookup that ran and came back empty is entitled to say what it saw: {read:?}" + ); + assert!( + !read_detail.contains("no Vulkan driver was found"), + "but it may not deny the driver itself: on Windows this lookup reads only the legacy \ + Khronos key, which a current Adrenalin install leaves empty, so an empty list is \ + routinely true of a machine with a working Vulkan driver: {read:?}" ); assert_ne!( unread_detail, read_detail, @@ -1140,8 +1258,11 @@ GPU0: // `Absent` is the confirmed claim, and it takes *both* sources having // answered. A `vulkaninfo` that ran and listed nothing is not enough on // its own while the driver list went unread. - let only_vulkaninfo_answered = - classify_vulkan_support(Vec::new(), Vec::new(), true, false, true); + let only_vulkaninfo_answered = classify_vulkan_support( + VulkanEvidence::unanswered(), + VulkanEvidence::answered(Vec::new()), + true, + ); assert_eq!( only_vulkaninfo_answered.status, VulkanStatus::Inconclusive, @@ -1155,7 +1276,12 @@ GPU0: "{only_vulkaninfo_answered:?}" ); assert_eq!( - classify_vulkan_support(Vec::new(), Vec::new(), true, true, true).status, + classify_vulkan_support( + VulkanEvidence::answered(Vec::new()), + VulkanEvidence::answered(Vec::new()), + true + ) + .status, VulkanStatus::Absent, "both sources answered and both came back empty: that one has earned the claim" ); @@ -1168,43 +1294,43 @@ GPU0: #[test] fn a_powershell_that_never_answered_is_not_an_empty_registry_key() { // Missing `powershell`, a non-zero exit, or a timeout: all one `None`. - let (icds, listed) = windows_icds_from_capture(None); + let icds = windows_icds_from_capture(None); assert!( - icds.is_empty() && !listed, + icds.found().is_empty() && !icds.did_answer(), "a run that produced no output did not read the registry" ); assert_eq!( - classify_vulkan_support(icds, Vec::new(), true, listed, false).status, + classify_vulkan_support(icds, VulkanEvidence::unanswered(), true).status, VulkanStatus::Inconclusive, "so it must not be reported as a confirmed absence" ); // Exit 0 with no `ICD` row: the key really is empty, and that answer is // what a host with no Vulkan driver installed looks like. - let (icds, listed) = windows_icds_from_capture(Some(String::new())); + let icds = windows_icds_from_capture(Some(String::new())); assert!( - icds.is_empty() && listed, + icds.found().is_empty() && icds.did_answer(), "an empty answer is still an answer" ); assert_eq!( - classify_vulkan_support(icds, Vec::new(), true, listed, true).status, + classify_vulkan_support(icds, VulkanEvidence::answered(Vec::new()), true).status, VulkanStatus::Absent, "and that one has earned the confirmed absence" ); - let (icds, listed) = windows_icds_from_capture(Some( + let icds = windows_icds_from_capture(Some( "ICD\tC:\\Windows\\System32\\amdvlk64.json\n".to_owned(), )); - assert!(listed); - assert_eq!(icds.len(), 1, "{icds:?}"); - assert_eq!(amd_icd_driver_label(&icds[0]), Some("AMDVLK")); + assert!(icds.did_answer()); + assert_eq!(icds.found().len(), 1, "{icds:?}"); + assert_eq!(amd_icd_driver_label(&icds.found()[0]), Some("AMDVLK")); } /// A confirmed AMD Vulkan path must never be attributed to another vendor's /// driver. `detect_icds` sorts alphabetically on Linux, so on a host with - /// both manifests installed `nvidia_icd.json` came first and won the old - /// "any manifest that is not a software rasteriser" fallback — naming NVIDIA - /// as the driver behind an AMD device the probe had already confirmed. + /// both manifests installed `nvidia_icd.json` comes first: any fallback + /// looser than "AMD only" names NVIDIA as the driver behind an AMD device + /// the probe has already confirmed. #[test] fn a_dual_vendor_host_names_the_amd_driver_behind_a_confirmed_amd_device() { // The AMD entry omits the optional `driverName`, which is the only @@ -1229,13 +1355,11 @@ GPU1: let support = classify_vulkan_support( // Alphabetical, as `detect_icds` returns them on Linux. - vec![ + VulkanEvidence::answered(vec![ "nvidia_icd.json".to_owned(), "radeon_icd.x86_64.json".to_owned(), - ], - devices, - true, - true, + ]), + VulkanEvidence::answered(devices), true, ); @@ -1250,14 +1374,12 @@ GPU1: // The sibling instance: the sentence that justifies offering a host a // Vulkan path when no AMD device was confirmed at all. let unconfirmed = classify_vulkan_support( - vec![ + VulkanEvidence::answered(vec![ "nvidia_icd.json".to_owned(), "radeon_icd.x86_64.json".to_owned(), - ], - Vec::new(), - true, + ]), + VulkanEvidence::unanswered(), true, - false, ); let unconfirmed_detail = unconfirmed.detail.as_deref().unwrap_or_default(); assert!( @@ -1270,11 +1392,9 @@ GPU1: // that status claims nothing about AMD. But `driver_label` — which only // ever speaks for a confirmed AMD device — must stay silent. let nvidia_only = classify_vulkan_support( - vec!["nvidia_icd.json".to_owned()], - Vec::new(), + VulkanEvidence::answered(vec!["nvidia_icd.json".to_owned()]), + VulkanEvidence::unanswered(), true, - true, - false, ); assert!( nvidia_only @@ -1300,8 +1420,8 @@ GPU1: assert_eq!( amd_icd_driver_label("lvp_icd.x86_64.json"), None, - "a software rasteriser is never an AMD GPU driver, so narrowing to AMD keeps the \ - exclusion the old filter provided" + "a software rasteriser is never an AMD GPU driver, so the AMD-only test still \ + excludes one" ); assert_eq!(amd_icd_driver_label("some_new_vendor_icd.json"), None); } diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 4d415edc5f..4f0d5b2f55 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -133,7 +133,9 @@ pub use fix::{ list_recipes as list_fix_recipes, }; pub use gfx_target::{ - GpuSupportLevel, VULKAN_ONLY_SUPPORT_CLAIM, gpu_support_level, gpu_support_level_for_host_gpu, + GpuSupportLevel, VULKAN_ONLY_SUPPORT_CLAIM, VULKAN_UNCONFIRMED_CLAIM, + VULKAN_UNCONFIRMED_NEXT_STEP, VULKAN_UNCONFIRMED_NEXT_STEP_WITH_DISPLAY_DRIVER, + gpu_support_level, gpu_support_level_for_host_gpu, }; pub use gfx_target::{ extract_first_gfx_token, gfx_target_has_no_rocm_build, known_therock_families, @@ -141,12 +143,13 @@ pub use gfx_target::{ }; pub use hardware_root::host_path; pub use host_gpu::{ - DriverSummary, ExamineSummary, HostGpuSummary, VulkanDevice, VulkanStatus, VulkanSupport, - WSL_ROCDXG_READY, WindowsGpuListing, WslHostDriverProbe, WslSummary, default_engine_for_host, - default_engine_for_platform, detect_gpu_driver_version, detect_host_gfx_target, - detect_host_gpu_diagnostics, detect_host_gpu_summary, detect_system_ram_gib, - detect_vulkan_support, has_usable_amd_gpu, is_amdgpu_device, is_wsl_host, pinned_gpu_name, - preferred_serve_engine_for_host_gpu_summary, usable_amd_gpu_indices, windows_amd_gpu_listing, + DriverSummary, ExamineSummary, HostGpuSummary, VulkanDevice, VulkanEvidence, VulkanStatus, + VulkanSupport, WSL_ROCDXG_READY, WindowsGpuListing, WslHostDriverProbe, WslSummary, + default_engine_for_host, default_engine_for_platform, detect_gpu_driver_version, + detect_host_gfx_target, detect_host_gpu_diagnostics, detect_host_gpu_summary, + detect_system_ram_gib, detect_vulkan_support, has_usable_amd_gpu, is_amdgpu_device, + is_wsl_host, pinned_gpu_name, preferred_serve_engine_for_host_gpu_summary, + usable_amd_gpu_indices, windows_amd_gpu_listing, }; #[cfg(any(target_os = "linux", test))] use host_gpu::{KfdGpuNode, kfd_gpu_nodes_in}; diff --git a/docs/manual-testing.md b/docs/manual-testing.md index e5b60c98a8..c9ea4ef7e5 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -323,8 +323,8 @@ checks, which cannot be made anywhere else: - With the AMD display driver (Windows) or Mesa RADV (Linux) installed, the line says models can still run through Vulkan and **names the driver**, and - `rocm examine --json` shows `vulkan.status` as `available` with the device - under `vulkan.devices`. Then actually run + `rocm examine --json` shows `summary.vulkan.status` as `available` with the + device under `summary.vulkan.devices`. Then actually run `rocm serve --engine lemonade` — the point of the claim is that it is true, and this is the only place it can be confirmed against real hardware. diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index 049d2e91dc..30e0835b16 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -753,16 +753,16 @@ Feature: GPU detection and system inspection And the support level agrees with the GPU the inspection detected And the support level leaves a command that fits this host - # The three below close the gap this PR's own notes call unclosable: the - # `VulkanOnly` door needs a Renoir/Cezanne Vega APU, and no runner has one. - # It does not need the hardware, only the two signals the CLI reads for it -- - # a KFD node naming the target, and the host's Vulkan driver state -- both of - # which the harness can plant. So these run on the ungated mock lane, which is - # the lane that has neither. + # The `VulkanOnly` door needs a Renoir/Cezanne Vega APU and no runner has one, + # but it does not need the hardware: only the two signals the CLI reads for it + # -- a KFD node naming the target, and the host's Vulkan driver state -- both + # of which the harness can plant. So these run on the ungated mock lane, which + # is the lane that has neither. # - # They exist because the support level used to be inferred: "no ROCm build" - # was taken to mean "Vulkan then", on a machine nobody had looked at. Each - # scenario pins one of the three answers that inference could not tell apart. + # Each pins one of the three answers a probe can reach where inference from the + # gfx table reaches only one: "no ROCm build" does not mean "Vulkan then" on a + # machine nobody has looked at, and the three scenarios are what keep those + # answers distinguishable. @id:examine-names-the-vulkan-driver-an-unsupported-gpu-can-use @requires-os:linux Scenario: examine-50 - A GPU the official stack rejects is told which way it can still run models Given a machine whose GPU has no official support but has a working Vulkan driver diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 8723a8443a..b5a0fbf730 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -1711,9 +1711,10 @@ async fn assert_no_amdgpu_fault_evidence(world: &mut E2eWorld) { /// It has to be its own step rather than reusing "the user inspects the /// system", because that one runs through `run_rocm`, which does not consume /// `World::command_env` -- so a planted host root never reaches the child and -/// the CLI silently describes the *runner* instead. Not hypothetical: it is -/// what made the first draft of examine-50/51/52 report the lane's own MI300X -/// as `gfx942` while claiming to be about a Renoir APU. +/// the CLI silently describes the *runner* instead. A scenario wired to that +/// step still passes while reporting the lane's own GPU (`gfx942` on the MI300X +/// runners) under a claim about a planted Renoir APU, so examine-50/51/52 must +/// use this one. #[when("the user inspects that machine")] async fn user_inspects_planted_machine_text(world: &mut E2eWorld) { let (stdout, stderr, rc) = @@ -3355,18 +3356,25 @@ async fn assert_support_level_advice_fits(world: &mut E2eWorld) { // this GPU, so naming an engine would promise a run the evidence does // not support. A driver is what clears either condition. // - // Split out because the catch-all below demands `--engine lemonade`, - // which neither tier's text contains. No lane reaches that assertion - // today only because none carries a detected, ROCm-unsupported GPU in - // these tiers; dropping these arms again makes the first such runner - // red rather than failing here. This arm has been lost to a merge once - // already. + // Both arms are load-bearing and must stay: the catch-all below demands + // `--engine lemonade`, which neither tier's text contains, so a host in + // either tier fails examine-49 the moment one of them is dropped. No CI + // lane carries a detected, ROCm-unsupported GPU, so that regression is + // silent here and only shows up on the first such runner. // // Told apart by each tier's own wording, as the catalog arm above is: // `VulkanUnconfirmed` says the probe could not settle it, `NoGpuBackend` - // says it looked and found nothing. - (Some(_), None, _) if line.contains("could not confirm") => ( - "install a Vulkan driver", + // says it looked and found nothing. The `VulkanUnconfirmed` guard reads + // `rocm_core`'s own constant rather than a copy of its text, so the two + // cannot drift apart when the sentence is reworded. + // + // What that tier owes is ``run `vulkaninfo` ``, not "install a Vulkan + // driver": `rocm_support_line` drops the install advice on a Windows + // host whose AMD display driver the summary already found, since that + // driver *is* the Vulkan driver there. `vulkaninfo` is the one step that + // can clear this condition on every host that reaches it. + (Some(_), None, _) if line.contains(rocm_core::VULKAN_UNCONFIRMED_CLAIM) => ( + "run `vulkaninfo`", &["rocm install sdk", "--engine lemonade"], ), (Some(_), None, _) if line.contains("no Vulkan GPU driver was found") => { @@ -3557,11 +3565,19 @@ async fn unsupported_gpu_without_vulkaninfo(world: &mut E2eWorld) { async fn assert_vulkan_unconfirmed(world: &mut E2eWorld) { let output = world.cli_output.as_ref().expect("no command was run"); let line = rocm_support_line(output).expect("no rocm_support line"); + // The constant itself, not a copy of its text: this is the same sentence + // `lemonade_engine_note` renders a few lines further down the same report, + // and reading it from `rocm_core` is what stops this assertion drifting away + // from the wording it is meant to pin. assert!( - line.contains("`vulkaninfo` could not confirm"), - "the support line must name the gap that stopped it confirming, not assert \ + line.contains(rocm_core::VULKAN_UNCONFIRMED_CLAIM), + "the support line must say the probe could not settle it, not assert \ a driver is absent: {line}\n{output}" ); + assert!( + line.contains("run `vulkaninfo`"), + "and still name the one step that can settle it: {line}\n{output}" + ); assert!( !line.contains("Vulkan backend") && !line.contains("runs on"), "and it must not promise a Vulkan path it never established: {line}\n{output}" From e7b65138bea35ef03f140819ee45569cc91242c2 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Mon, 16 Nov 2026 15:18:01 +0200 Subject: [PATCH 44/59] fix(services): stop the device probe losing a just-published anthropic endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `render_services_text_with_devices` snapshots every managed-service record once, up front, through `load_managed_services`. That used to be a local read, so the snapshot was effectively taken at print time. This branch put the live `Healthcheck` device probe in between, and that probe can spend seconds — up to `engine_ipc::SERVICE_DEVICE_PROBE_BUDGET`, about 18 of them — before the first row is written. The serving process publishes the Anthropic endpoint to the engine state file at about the moment the engine first lists its model, which is the same moment the device probe starts getting an answer. So the gap this branch opened is exactly the window in which a freshly ready service renders with no `anthropic endpoint:` row at all, from a snapshot that predates the publication. The Strix Halo Windows e2e lane caught it on content-identical commit d35ac5fe (run 43468147450): five scenarios regressed — `serve-anthropic-endpoint-advertised`, `-messages-inference`, `-endpoint-readiness`, `-messages-tool-use` and `-messages-streaming` — on a lane that passes on three consecutive recent `main` commits. Re-read the one field that moves, after the probe and before printing, with `rocm_core::wait_for_published_anthropic_endpoint` — the same read `serve_cmd.rs` already uses for the same race in the post-launch summary, rather than a second idiom. With no wait: a wait is right there, where the command has just launched the server and is standing by for it, and wrong here, where `rocm services list` reports on whatever exists and must not start blocking on a service that may never publish. Reading once restores the timing the listing had before the probe. Gated on the same statuses as `ManagedServiceRecord::refresh_from_engine_state`, which is what put the field on the snapshot to begin with; for anything else the state file is a finished run's leftovers. `services_listing_shows_an_anthropic_endpoint_published_while_the_probe_ran` pins it, and discriminates on the staleness rather than on the row's presence: its probe closure publishes the endpoint while it runs, so rendering from the pre-probe snapshot shows no row. Verified by removing the production call — the test fails, with the rendered listing in the message. Signed-off-by: Roman Sirokov --- apps/rocm/src/services_text.rs | 141 ++++++++++++++++++++++++++++++++- 1 file changed, 140 insertions(+), 1 deletion(-) diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index 782ab1dcc6..91c1745382 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -439,7 +439,7 @@ fn render_services_text_with_devices( .iter() .find(|record| !managed_service_is_live(record)) .map(|record| record.service_id.clone()); - let records = all_records + let mut records = all_records .into_iter() .filter(|record| all || managed_service_is_live(record)) .collect::>(); @@ -486,6 +486,7 @@ fn render_services_text_with_devices( .map(|record| (record.service_id.as_str(), record.engine.as_str())) .collect(), ); + refresh_published_anthropic_endpoints(&mut records); let _ = writeln!(output, "Servers"); for record in &records { @@ -538,6 +539,49 @@ fn render_services_text_with_devices( Ok(output) } +/// Re-read each live record's published Anthropic endpoint from the engine +/// state file, after the device probe and before anything is printed. +/// +/// [`render_services_text_with_devices`] snapshots every record once, up front, +/// via `load_managed_services`. That was a local read, so the snapshot was +/// effectively taken at print time. Since ROCMAI-550 the device probe sits +/// between the two and can spend seconds (up to +/// `engine_ipc::SERVICE_DEVICE_PROBE_BUDGET`) on live `Healthcheck` RPCs, and +/// the serving process publishes the Anthropic endpoint at roughly the moment +/// the engine first lists its model — the same moment the device probe starts +/// getting an answer. So the gap this PR opened is exactly the window in which +/// a freshly ready service renders with no `anthropic endpoint:` row, from a +/// snapshot that predates the publication. +/// +/// `serve_cmd.rs`'s post-launch summary hit the same race and answers it with +/// `rocm_core::wait_for_published_anthropic_endpoint`; this is the same read, +/// with no wait. A wait is right there — that command has just launched the +/// server and is standing by for it — and wrong here: `rocm services list` +/// reports on whatever exists, and must not start blocking on a service that +/// has not published yet and may never. Reading once restores the timing the +/// listing had before the probe existed. +/// +/// Gated on the same statuses as `ManagedServiceRecord::refresh_from_engine_state`, +/// which is what put the field on the snapshot in the first place: for anything +/// else the state file is a finished run's leftovers and the record's own value +/// is the one to keep. +fn refresh_published_anthropic_endpoints(records: &mut [ManagedServiceRecord]) { + for record in records { + if !matches!( + record.status.as_str(), + "starting" | "running" | "recovering" | "ready" + ) { + continue; + } + if let Some(url) = rocm_core::wait_for_published_anthropic_endpoint( + &record.engine_state_path, + Duration::ZERO, + ) { + record.anthropic_endpoint_url = Some(url); + } + } +} + /// The machine-readable counterpart to [`render_services_text`]. /// /// Applies the same liveness filter as the text form so `--json` and the table @@ -1903,6 +1947,101 @@ pub(crate) mod tests { Ok(()) } + /// The `anthropic endpoint:` row must survive the device probe's gap. + /// + /// `render_services_text_with_devices` snapshots every record once, up + /// front. That used to be a local read taken effectively at print time; + /// since the device probe went in between, the snapshot can be seconds old + /// by the time a row is written. The serving process publishes the + /// Anthropic endpoint to the engine state file at about the moment the + /// engine first lists its model — the same moment the probe starts getting + /// an answer — so that gap is precisely the window in which a freshly + /// ready service renders with no `anthropic endpoint:` row at all. + /// + /// The probe closure here *is* that window: it publishes the endpoint + /// while it runs. Rendering from the pre-probe snapshot therefore shows no + /// row and this test fails; only a re-read after the probe passes it. The + /// fixture cannot be "the endpoint is simply on the record", which the + /// stopped-service test above already covers and which passes either way. + #[test] + fn services_listing_shows_an_anthropic_endpoint_published_while_the_probe_ran() -> Result<()> { + use std::io::{Read, Write}; + use std::net::TcpListener; + + let (root, paths) = test_paths("services-anthropic-after-probe"); + paths.ensure()?; + let listener = TcpListener::bind(("127.0.0.1", 0))?; + let ready_port = listener.local_addr()?.port(); + let server = thread::spawn(move || -> Result<()> { + for _ in 0..2 { + let (mut stream, _) = listener.accept()?; + stream.set_read_timeout(Some(Duration::from_secs(2)))?; + let mut request = [0_u8; 512]; + let read = stream.read(&mut request)?; + let request = String::from_utf8_lossy(&request[..read]).into_owned(); + let body = if request.starts_with("POST /v1/chat/completions ") { + r#"{"choices":[{"message":{"content":"ok"}}]}"# + } else { + r#"{"data":[{"id":"Qwen/Qwen3.5"}]}"# + }; + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + )?; + } + Ok(()) + }); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-anthropic-late", + "vllm", + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + ready_port, + "managed", + std::process::id(), + Some("therock-release".to_owned()), + None, + Some("gpu_required".to_owned()), + ); + record.status = "ready".to_owned(); + record.write()?; + // Nothing is published when the snapshot is taken: no state file exists, + // so `refresh_from_engine_state` has nothing to adopt. + assert!(!record.engine_state_path.exists()); + let state_path = record.engine_state_path; + + let rendered = render_services_text_with_devices(&paths, false, |_ready| { + // What the serving process does while the probe is in flight. + if let Some(parent) = state_path.parent() { + fs::create_dir_all(parent).expect("engine state dir"); + } + fs::write( + &state_path, + r#"{"status": "ready", "anthropic_endpoint_url": "http://127.0.0.1:1", "anthropic_endpoint_mode": "proxy"}"#, + ) + .expect("publish the anthropic endpoint mid-probe"); + HashMap::new() + })?; + server + .join() + .expect("fake models server should not panic")?; + let _ = fs::remove_dir_all(root); + + // Port 1 is closed, so the row says "(not responding)". What is pinned + // is that the row exists and names the published URL — the five Strix + // Halo scenarios that regressed look for exactly this prefix. + assert!( + rendered.contains(" anthropic endpoint: http://127.0.0.1:1"), + "an endpoint published during the device probe must still reach the \ + row, not be lost to the pre-probe snapshot:\n{rendered}" + ); + Ok(()) + } + /// The default view lists only live servers. The counts used to be taken /// from that already-filtered list, so a host whose local servers had all /// failed was told "none ready" and shown an empty list - the records were From fa9c1fc8dd374f7ffcbc3d89db5114a059fd2d91 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Mon, 16 Nov 2026 15:18:34 +0200 Subject: [PATCH 45/59] docs(services): scope the device-probe wait to the table, not `--json` `ServicesCommand::List`'s `--help` said each running server is asked which backend and GPU it is on, "so this takes a few seconds... about 18 seconds worst case", with no qualification. `--json` does none of that: `services_cmd.rs` routes it to `render_services_json`, which reads `load_managed_services` and serializes the stored `ManagedServiceRecord`s verbatim. There is no call into the probing path and no `device` field on the record to fill, so the claim was simply untrue of the form a script is likeliest to run. Scope the sentence to the default table and say what `--json` does instead. README.md carries the same paragraph and gets the same qualification. Signed-off-by: Roman Sirokov --- README.md | 4 +++- apps/rocm/src/cli.rs | 11 ++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 7376bb322f..a3cf826983 100644 --- a/README.md +++ b/README.md @@ -1530,7 +1530,9 @@ can take a few seconds rather than returning instantly. The query is capped in total — about 18 seconds worst case, however many servers are listed — and a server that does not answer in time simply has no `device` line rather than holding up the rest of the list. The same applies to asking about servers in -`rocm chat`, which prints the same list. +`rocm chat`, which prints the same list. `list --json` is unaffected: it emits +the stored service records, which have no `device` field, asks no server +anything, and returns as fast as it always did. `restart` starts a server again only once its stop has confirmed that every recorded process is gone; if the stop cannot confirm that, it says so, starts diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index 8a56b5eab9..9bbb3bf482 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -1185,11 +1185,12 @@ pub(crate) enum ServicesCommand { /// Show currently running local model servers, and count the records that /// are no longer running. /// - /// Each running server is asked which backend and GPU it is actually on, so - /// this takes a few seconds rather than returning instantly. The asking is - /// capped in total (about 18 seconds worst case, however many servers are - /// listed), and a server that does not answer in time simply has no device - /// line. + /// In the default table, each running server is asked which backend and GPU + /// it is actually on, so this takes a few seconds rather than returning + /// instantly. The asking is capped in total (about 18 seconds worst case, + /// however many servers are listed), and a server that does not answer in + /// time simply has no device line. `--json` asks nothing and carries no + /// device field: it is the stored records, returned as fast as before. List { /// Include failed, stopped, and old service records. #[arg(short, long)] From 0c7e432faf23a8c4135bc68c3e43dfb536774dff Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Mon, 16 Nov 2026 15:19:13 +0200 Subject: [PATCH 46/59] fix(examine): stop an SDK for another family being called this GPU's MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `rocm_support_line`'s `RocmHip` arm printed "the managed ROCm SDK for {installed} is already installed - serve on it" whenever ANY managed runtime was on disk, without comparing it with what the detected GPU needs. The two fields are computed from unrelated facts and really can differ. `compatible_therock_family` is `normalize_therock_family` applied to the gfx target detected right now; `detect_managed_therock_family` walks the runtime registry and the engine manifests for the most recently installed TheRock family and never looks at the GPU at all — `managed_therock_family_uses_runtime_manifest_not_host_mapping` pins exactly that independence. A GPU swap, an eGPU, or a second card is enough to make the install on disk one this GPU was never built for, and the user was then told it was ready for their card. Gate the "already installed" branch on the two families matching and give the mismatch its own wording, which names both families and points at `rocm install sdk` — still an install with something left to do, so the advice can clear the condition it is printed for. The existing test planted `gfx110X-dgpu` against a gfx1103 GPU, whose compatible family is `gfx110X-all`: it was asserting the "already installed" sentence on a MISMATCHED pair, which is the bug. It now uses the GPU's own family, and `an_sdk_for_another_family_is_not_reported_as_ready_for_this_gpu` covers the mismatch. Verified it discriminates: with the family comparison removed the new test fails. The e2e step `the support level leaves a command that fits this host` expected `rocm serve ` and forbade `rocm install sdk` for any host with both a compatible family and an installed SDK, which would now fail against correct output on a mismatched host. It gains the matching condition and an arm for the mismatch. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/host_gpu/render.rs | 109 +++++++++++++++--- tests/e2e-cucumber/tests/e2e/examine_steps.rs | 14 ++- 2 files changed, 104 insertions(+), 19 deletions(-) diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index aec43dfbfb..d2de61a01f 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -55,19 +55,44 @@ impl ExamineSummary { // `detected_therock_family` is the installed managed runtime (see // `detect_managed_therock_family`), as opposed to // `compatible_therock_family`, which is what this GPU *could* run. - GpuSupportLevel::RocmHip => match self.detected_therock_family.as_deref() { - Some(installed) => format!( - " rocm_support: GPU detected ({}) with full ROCm/HIP support, and the \ - managed ROCm SDK for {installed} is already installed - serve on it with \ - `rocm serve `\n", - detected.unwrap_or("") - ), - None => format!( - " rocm_support: GPU detected ({}) with full ROCm/HIP support; \ - install the SDK for it with `rocm install sdk`\n", - detected.unwrap_or("") - ), - }, + // + // The two are computed from unrelated facts and really can differ. + // `compatible_therock_family` is `normalize_therock_family` applied + // to the gfx target detected right now; `detect_managed_therock_family` + // is the family of the most recently installed managed runtime on + // disk and never looks at the GPU at all. Swap the card, plug in an + // eGPU, or keep two cards and boot the other one, and the install + // on disk is for a family this GPU was never built for. Saying + // "already installed - serve on it" there would promise a serve + // this host cannot do, which is exactly the kind of unestablished + // claim ROCMAI-550 exists to remove, so the match is a condition of + // that sentence rather than an assumption behind it. + GpuSupportLevel::RocmHip => { + let compatible = self.compatible_therock_family.as_deref(); + match self.detected_therock_family.as_deref() { + Some(installed) if Some(installed) == compatible => format!( + " rocm_support: GPU detected ({}) with full ROCm/HIP support, and the \ + managed ROCm SDK for {installed} is already installed - serve on it with \ + `rocm serve `\n", + detected.unwrap_or("") + ), + // An install is present but for another family, so + // `rocm install sdk` still has something to do and is still + // advice that can clear the condition it is printed for. + Some(installed) => format!( + " rocm_support: GPU detected ({}) with full ROCm/HIP support, but the \ + managed ROCm SDK installed here is for {installed}, not {} - install the \ + one this GPU needs with `rocm install sdk`\n", + detected.unwrap_or(""), + compatible.unwrap_or("") + ), + None => format!( + " rocm_support: GPU detected ({}) with full ROCm/HIP support; \ + install the SDK for it with `rocm install sdk`\n", + detected.unwrap_or("") + ), + } + } // The command names `rocm services list` rather than the serve // summary on purpose. The summary reads the device once, right // after `ready`, and lemonade answers `unknown` until its own @@ -680,16 +705,18 @@ mod tests { // Same GPU, but the SDK it was being told to install is already there: // the advice has to be something that is still left to do (AGENTS.md - // wants a named command to be able to clear the condition). + // wants a named command to be able to clear the condition). The + // installed family must be the GPU's own - see + // `an_sdk_for_another_family_is_not_reported_as_ready_for_this_gpu`. let mut installed = supported; - installed.detected_therock_family = Some("gfx110X-dgpu".to_owned()); + installed.detected_therock_family = family; let rendered = installed.render_text(); assert!( rendered.contains("rocm_support: GPU detected (gfx1103) with full ROCm/HIP support"), "{rendered}" ); assert!( - rendered.contains("managed ROCm SDK for gfx110X-dgpu is already installed"), + rendered.contains("managed ROCm SDK for gfx110X-all is already installed"), "{rendered}" ); assert!( @@ -728,4 +755,54 @@ mod tests { "nor as supported: {rendered}" ); } + + /// A managed SDK on disk is not, by itself, an SDK for *this* GPU. + /// + /// `detected_therock_family` is the most recently installed managed runtime + /// (`detect_managed_therock_family` reads the registry and never looks at + /// the GPU); `compatible_therock_family` is normalised from the gfx target + /// detected right now. A GPU swap, an eGPU or a second card leaves them + /// pointing at different families, and the `already installed - serve on + /// it` sentence then promises a serve this host cannot do. + /// + /// The matching half is asserted by + /// `examine_render_reports_a_support_level_for_every_host`; only this one + /// fails if the two families stop being compared, because the wording of + /// the matching case is unchanged. + #[test] + fn an_sdk_for_another_family_is_not_reported_as_ready_for_this_gpu() { + let compatible = normalize_therock_family("gfx1103").expect("gfx1103 has a family"); + let mut host = examine_summary_with_gpu(Some("gfx1103"), Some(&compatible)); + // A real family, and not this GPU's: the gfx110X split between the + // discrete and the all-targets build is exactly the pair a user is + // likeliest to have both of. + host.detected_therock_family = Some("gfx110X-dgpu".to_owned()); + assert_ne!( + host.detected_therock_family.as_deref(), + Some(compatible.as_str()), + "precondition: the installed family is not the compatible one" + ); + + let rendered = host.render_text(); + let line = rendered + .lines() + .find(|line| line.trim_start().starts_with("rocm_support:")) + .expect("every host gets a support line") + .to_owned(); + + assert!( + !line.contains("already installed"), + "an SDK built for another family must not be reported as this GPU's: {line}" + ); + assert!( + line.contains("gfx110X-dgpu") && line.contains(compatible.as_str()), + "the line has to name both families, or the user cannot see which \ + one is missing: {line}" + ); + assert!( + line.contains("rocm install sdk"), + "there is still an install left to do, so the advice must name the \ + command that does it: {line}" + ); + } } diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 41285d1810..b8845ce883 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -3307,9 +3307,17 @@ async fn assert_support_level_advice_fits(world: &mut E2eWorld) { .filter(|value| *value != "" && *value != ""); let (expected, forbidden): (&str, &[&str]) = match (detected, family, installed_sdk) { - // ROCm-capable with the managed SDK already in place: installing one - // clears nothing, so the line must point at serving instead. - (_, Some(_), Some(_)) => ("rocm serve ", &["rocm install sdk"]), + // ROCm-capable with the managed SDK for THIS GPU's family already in + // place: installing one clears nothing, so the line must point at + // serving instead. + (_, Some(family), Some(installed)) if installed == family => { + ("rocm serve ", &["rocm install sdk"]) + } + // ROCm-capable with a managed SDK installed, but for another family — + // a GPU swap, an eGPU, or a second card. The install on disk was never + // built for this GPU, so the one it needs is still the next step and + // the line must not offer to serve on what is there. + (_, Some(_), Some(_)) => ("rocm install sdk", &["serve on it"]), // ROCm-capable with no managed SDK yet: that install is the next step. (_, Some(_), None) => ("rocm install sdk", &[]), // A GPU no package family covers. Two different tiers land here and From 3d59dbcb883f0c73df389f6599a7ec3191e55db4 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 17 Nov 2026 10:58:23 +0200 Subject: [PATCH 47/59] fix(serve): stop the engine's post-spawn state rewrite dropping the Anthropic endpoint The Anthropic proxy is a thread of the engine's own process, and it publishes exactly once -- as soon as the server lists a model. The engine adapter rewrites the same state file *whole* immediately after it spawns that server, and since ROCMAI-550 it spends a host GPU probe building the document first (`pinned_gpu_name`: pnputil plus a PowerShell CIM query on Windows, seconds). The publication lands inside that window and is replaced, and because nothing publishes twice the `anthropic endpoint:` row never comes back -- five Windows e2e scenarios saw a ready service with a working `device:` row and no endpoint. The file lock main added in ROCMAI-453 orders the two writers but does not stop the later one replacing what the earlier recorded; that is what a full rewrite is for. `publish_anthropic_endpoint` now notes the publication for this process before it touches the file, and `write_json_state_file` carries that note into the document it is about to write. Scoped to this process and this state file, so a dead endpoint a previous run left on disk is still cleared by the new run's first write, which `reset_for_restart` relies on. Also puts the engine state file into the e2e failure message: the row being absent because nothing published and the row being absent because something replaced it look identical in `services list`, and telling them apart cost a Windows lane run per guess. Signed-off-by: Roman Sirokov Signed-off-by: Roman Sirokov --- apps/rocmd/src/anthropic_proxy.rs | 27 +- crates/rocm-core/src/anthropic_endpoint.rs | 243 +++++++++++++++++- crates/rocm-core/src/endpoint_readiness.rs | 19 +- crates/rocm-core/src/lib.rs | 2 +- engines/lemonade/src/state.rs | 83 ++++++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 60 ++++- 6 files changed, 405 insertions(+), 29 deletions(-) diff --git a/apps/rocmd/src/anthropic_proxy.rs b/apps/rocmd/src/anthropic_proxy.rs index 73df2dceb3..51937bb6db 100644 --- a/apps/rocmd/src/anthropic_proxy.rs +++ b/apps/rocmd/src/anthropic_proxy.rs @@ -46,9 +46,8 @@ use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::{Json, Router}; use rocm_core::{ - ANTHROPIC_ENDPOINT_MODE_STATE_KEY, ANTHROPIC_ENDPOINT_URL_STATE_KEY, AnthropicEndpointMode, - JsonStateFileRefused, format_http_base_url, http_get_with_auth, http_post_json_with_auth, - merge_json_state_file, openai_models_endpoint_has_model, + AnthropicEndpointMode, JsonStateFileRefused, format_http_base_url, http_get_with_auth, + http_post_json_with_auth, openai_models_endpoint_has_model, publish_anthropic_endpoint, }; use serde_json::{Value, json}; @@ -153,11 +152,17 @@ const PUBLISH_RETRY_PAUSE: Duration = Duration::from_millis(200); /// Record the endpoint in the engine's state file. /// -/// `merge_json_state_file` refuses a state file it cannot merge into rather -/// than replace it. The vLLM and AIM engines still rewrite their state files -/// in place, so a file refused as empty or not valid JSON can be one caught -/// mid-write and whole again a moment later: that refusal is tried again, up -/// to [`PUBLISH_ATTEMPTS`] attempts [`PUBLISH_RETRY_PAUSE`] apart. Any other +/// Through `rocm_core::publish_anthropic_endpoint`, not a bare merge: the +/// engine adapter running in this same process rewrites that file *whole* a +/// moment after it spawns the server, and only a publication that function has +/// noted survives such a rewrite. See its documentation for why a bare merge +/// loses the endpoint for good. +/// +/// It refuses a state file it cannot merge into rather than replace it. The +/// vLLM and AIM engines still rewrite their state files in place, so a file +/// refused as empty or not valid JSON can be one caught mid-write and whole +/// again a moment later: that refusal is tried again, up to +/// [`PUBLISH_ATTEMPTS`] attempts [`PUBLISH_RETRY_PAUSE`] apart. Any other /// failure (a whole document that is not an object, or failing to lock, read /// or write the file) is returned at once. fn publish( @@ -178,13 +183,9 @@ fn publish_with_pause( ) -> anyhow::Result<()> { // The listener URL carries no `/v1`: Anthropic SDKs append `/v1/messages`. let base = url.trim_end_matches('/').trim_end_matches("/v1"); - let patch = json!({ - ANTHROPIC_ENDPOINT_URL_STATE_KEY: base, - ANTHROPIC_ENDPOINT_MODE_STATE_KEY: mode.as_str(), - }); let mut attempt = 1; loop { - let error = match merge_json_state_file(&config.state_path, &patch) { + let error = match publish_anthropic_endpoint(&config.state_path, base, mode) { Ok(()) => return Ok(()), Err(error) => error, }; diff --git a/crates/rocm-core/src/anthropic_endpoint.rs b/crates/rocm-core/src/anthropic_endpoint.rs index 744ae7d023..6d0314d33b 100644 --- a/crates/rocm-core/src/anthropic_endpoint.rs +++ b/crates/rocm-core/src/anthropic_endpoint.rs @@ -12,10 +12,21 @@ //! engine publishes the proxy's address under the state-file keys below, and [`crate::ManagedServiceRecord::refresh_from_engine_state`] adopts //! it, so every reader (`rocm serve`, `rocm services list`, the readiness probe) //! sees the same endpoint. +//! +//! The publisher is a *thread of the engine's own process*, not a separate +//! writer: `rocm`'s `run_builtin_engine_serve_http` starts the proxy and then +//! runs the engine adapter in the same process. Both write the one engine state +//! file, and the engine adapters write it whole. [`publish_anthropic_endpoint`] +//! and `carry_published_anthropic_endpoint` are the two halves that keep the +//! whole-file write from dropping what the sibling thread published. -use std::path::Path; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, OnceLock, PoisonError}; use std::time::{Duration, Instant}; +use anyhow::Result; + use crate::openai_models_endpoint_has_model; /// Engine state-file key naming the Anthropic-compatible endpoint's base URL. @@ -56,6 +67,117 @@ pub fn anthropic_endpoint_ready( .unwrap_or(false) } +/// What *this process* has published, keyed by the engine state file it went +/// into, so [`carry_published_anthropic_endpoint`] can put it back. +/// +/// Deliberately process-local rather than read back off disk. A previous run of +/// the same service id left its own endpoint in that file, and that URL is dead +/// — its proxy died with the process that bound the port. The first state write +/// of a new run is what clears it (`ManagedServiceRecord::reset_for_restart` +/// relies on exactly that), so a disk-backed carry-over would resurrect a dead +/// endpoint on every restart. Only the publication this process made is ours to +/// preserve. +/// +/// Keyed by path rather than held as a single value because the key is per +/// service: nothing today runs two engines in one process, but a map costs +/// nothing and a global would quietly cross-contaminate if one ever did (and +/// does cross-contaminate between tests sharing a process, which is the same +/// bug in miniature). +static PUBLISHED_ENDPOINTS: OnceLock>> = OnceLock::new(); + +fn published_endpoints() -> &'static Mutex> { + PUBLISHED_ENDPOINTS.get_or_init(|| Mutex::new(HashMap::new())) +} + +/// Record `base_url` as this service's Anthropic endpoint in `state_path`, and +/// remember it for the rest of this process. +/// +/// `base_url` carries no `/v1`: Anthropic SDKs append `/v1/messages` to it. +/// +/// The in-memory note is taken *before* the file is touched, and the note's +/// mutex is released before the file lock is asked for. Both matter: +/// +/// * noting first leaves no window — a whole-file rewrite that gets the file +/// lock first already sees the note and carries the endpoint, and one that +/// gets it after the merge below is carrying it too; +/// * releasing first keeps the two locks from ever being held in both orders, +/// since `carry_published_anthropic_endpoint` is called with the file lock +/// held. +/// +/// Called more than once for one endpoint when the merge has to be retried +/// (`rocmd`'s proxy retries a state file caught mid-write); the note is the +/// same each time. +pub fn publish_anthropic_endpoint( + state_path: &Path, + base_url: &str, + mode: AnthropicEndpointMode, +) -> Result<()> { + { + let mut published = published_endpoints() + .lock() + .unwrap_or_else(PoisonError::into_inner); + published.insert( + state_path.to_path_buf(), + (base_url.to_owned(), mode.as_str().to_owned()), + ); + } + crate::merge_json_state_file( + state_path, + &serde_json::json!({ + ANTHROPIC_ENDPOINT_URL_STATE_KEY: base_url, + ANTHROPIC_ENDPOINT_MODE_STATE_KEY: mode.as_str(), + }), + ) +} + +/// `state`, with the Anthropic endpoint this process published for `state_path` +/// put back into it — or `None` when there is nothing to put back, so the +/// caller writes the document it already has and pays no clone. +/// +/// [`crate::write_json_state_file`] is the only caller. A full rewrite replaces +/// every key, by design: it serializes a document its caller built rather than +/// patching the file, which is how a new run of a service clears what the +/// previous run left. The one thing it must not replace is a publication made +/// by *its own process*, because the publisher is a thread of that process +/// (`rocm`'s `run_builtin_engine_serve_http` starts the Anthropic proxy and +/// then runs the engine adapter) and nothing publishes a second time. +/// +/// This is not a theoretical race. The engine adapter's post-spawn `running` +/// write happens immediately after the server process is started, and since +/// ROCMAI-550 it spends a host GPU probe (`crate::pinned_gpu_name`: `pnputil` +/// plus a PowerShell CIM query on Windows, seconds) building the document +/// before writing it. That is long enough for the server to come up, for the +/// proxy's poll to see a listed model, and for the publication to land inside +/// the window and be replaced — leaving a ready service that never shows an +/// `anthropic endpoint:` row. +/// +/// The cross-process lock [`crate::merge_json_state_file`] and +/// [`crate::write_json_state_file`] share does not cover this. It orders the +/// two writers so neither sees a half-written file; it does not stop the +/// later one replacing what the earlier one recorded, which is exactly what a +/// full rewrite is for. +pub(crate) fn carry_published_anthropic_endpoint( + state_path: &Path, + state: &serde_json::Value, +) -> Option { + let (url, mode) = published_endpoints() + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(state_path) + .cloned()?; + let mut carried = state.clone(); + let object = carried.as_object_mut()?; + object.insert( + ANTHROPIC_ENDPOINT_URL_STATE_KEY.to_owned(), + serde_json::Value::String(url), + ); + object.insert( + ANTHROPIC_ENDPOINT_MODE_STATE_KEY.to_owned(), + serde_json::Value::String(mode), + ); + Some(carried) +} + /// The Anthropic endpoint the serving process has published in `state_path`, /// waiting up to `wait` for it to appear. /// @@ -120,4 +242,123 @@ mod tests { fn mode_strings_are_the_values_published_in_state() { assert_eq!(AnthropicEndpointMode::Proxy.as_str(), "proxy"); } + + /// The regression ROCMAI-550's Windows lane hit, driven through the real + /// writers: publish the endpoint, then have the engine adapter rewrite the + /// state file whole — which is what `write_running_state` does right after + /// it spawns the server, with a multi-second host GPU probe inside it. + /// + /// The rewritten document is deliberately *not* a copy of what is on disk: + /// a "fix" that merely re-read the file before writing would not be what + /// the adapters do, and would not satisfy this. + #[test] + fn a_whole_file_state_write_keeps_the_endpoint_this_process_published() { + let dir = unique_dir("carry"); + let state_path = dir.join("state.json"); + crate::write_json_state_file(&state_path, &serde_json::json!({ "status": "starting" })) + .unwrap(); + publish_anthropic_endpoint( + &state_path, + "http://127.0.0.1:41000", + AnthropicEndpointMode::Proxy, + ) + .unwrap(); + + crate::write_json_state_file( + &state_path, + &serde_json::json!({ "status": "running", "server_pid": 7 }), + ) + .unwrap(); + + let state: serde_json::Value = + serde_json::from_slice(&std::fs::read(&state_path).unwrap()).unwrap(); + assert_eq!( + state[ANTHROPIC_ENDPOINT_URL_STATE_KEY], "http://127.0.0.1:41000", + "an engine rewriting the whole state file must not drop the \ + endpoint its own process published: nothing publishes twice, so \ + the `anthropic endpoint:` row never comes back" + ); + assert_eq!(state[ANTHROPIC_ENDPOINT_MODE_STATE_KEY], "proxy"); + // The rewrite still replaces everything else, which is what it is for. + assert_eq!(state["status"], "running"); + assert_eq!(state["server_pid"], 7); + let _ = std::fs::remove_dir_all(dir); + } + + /// The carry is scoped to what *this* process published for *this* file. + /// + /// Both halves matter. A previous run of the same service left a dead + /// endpoint in that file — its proxy died with the process that bound the + /// port — and the new run's first whole-file write is what clears it, which + /// `ManagedServiceRecord::reset_for_restart` relies on; carrying anything + /// read off disk would resurrect it. And a process that published for one + /// service must not hand that endpoint to another. + #[test] + fn a_rewrite_still_clears_an_endpoint_this_process_did_not_publish() { + let dir = unique_dir("unpublished"); + let published_to = dir.join("published.json"); + let other = dir.join("other.json"); + publish_anthropic_endpoint( + &published_to, + "http://127.0.0.1:41001", + AnthropicEndpointMode::Proxy, + ) + .unwrap(); + std::fs::write( + &other, + r#"{"anthropic_endpoint_url": "http://127.0.0.1:1", "anthropic_endpoint_mode": "proxy"}"#, + ) + .unwrap(); + + crate::write_json_state_file(&other, &serde_json::json!({ "status": "starting" })).unwrap(); + + let state: serde_json::Value = + serde_json::from_slice(&std::fs::read(&other).unwrap()).unwrap(); + assert_eq!( + state, + serde_json::json!({ "status": "starting" }), + "only this process's own publication for this exact state file may \ + survive a whole-file write; a dead endpoint from a previous run \ + must not be restored" + ); + let _ = std::fs::remove_dir_all(dir); + } + + /// Publishing still writes the keys through to the file, which is what + /// every out-of-process reader (`rocm services list`, `rocm serve`'s + /// summary) actually looks at — the note is an addition, not a substitute. + #[test] + fn publishing_writes_the_endpoint_into_the_state_file_it_names() { + let dir = unique_dir("publish"); + let state_path = dir.join("state.json"); + std::fs::write(&state_path, r#"{"status": "running"}"#).unwrap(); + publish_anthropic_endpoint( + &state_path, + "http://127.0.0.1:41002", + AnthropicEndpointMode::Proxy, + ) + .unwrap(); + assert_eq!( + wait_for_published_anthropic_endpoint(&state_path, Duration::ZERO).as_deref(), + Some("http://127.0.0.1:41002") + ); + let state: serde_json::Value = + serde_json::from_slice(&std::fs::read(&state_path).unwrap()).unwrap(); + assert_eq!(state[ANTHROPIC_ENDPOINT_MODE_STATE_KEY], "proxy"); + // The merge keeps what was already recorded for the service. + assert_eq!(state["status"], "running"); + let _ = std::fs::remove_dir_all(dir); + } + + /// A directory no other test in this process shares: the publication note + /// is keyed by path and lives for the whole test binary. + fn unique_dir(tag: &str) -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!( + "rocm-core-anthropic-{tag}-{}-{}", + std::process::id(), + crate::unix_time_millis() + )); + std::fs::create_dir_all(&dir).unwrap(); + dir + } } diff --git a/crates/rocm-core/src/endpoint_readiness.rs b/crates/rocm-core/src/endpoint_readiness.rs index 52895a8bec..b19f9a95e9 100644 --- a/crates/rocm-core/src/endpoint_readiness.rs +++ b/crates/rocm-core/src/endpoint_readiness.rs @@ -124,6 +124,16 @@ fn merge_json_state_file_with( /// Takes the same lock as [`merge_json_state_file`] and publishes the same /// atomic way, so a full rewrite cannot land in the middle of another /// writer's merge. +/// +/// It replaces every key merged before it — that is what a full rewrite is for, +/// and how a new run of a service clears what the previous run recorded — with +/// one exception: an Anthropic endpoint *this process* published +/// (`crate::publish_anthropic_endpoint`) is carried into the document first. +/// The publisher is a thread of the same process as the engine adapter calling +/// this, it publishes exactly once, and since ROCMAI-550 the adapter's +/// post-spawn rewrite spends a multi-second host GPU probe building its +/// document — wide enough for the publication to land inside it and be lost. +/// See `anthropic_endpoint::carry_published_anthropic_endpoint`. pub fn write_json_state_file(path: &Path, value: &serde_json::Value) -> Result<()> { write_json_state_file_with(path, value, || {}) } @@ -137,8 +147,15 @@ fn write_json_state_file_with( before_publish: impl FnOnce(), ) -> Result<()> { let _lock = FileLock::acquire(json_state_lock_path(path))?; + // Read inside the file lock, which leaves no gap: a publisher notes the + // endpoint before it asks for this lock, so one that is waiting here has + // already been noted and is carried, and one that has not noted yet merges + // after this rewrite and lands on its own. `publish_anthropic_endpoint` + // releases its in-memory note before taking the file lock, so the two locks + // are never held in both orders. + let carried = crate::anthropic_endpoint::carry_published_anthropic_endpoint(path, value); before_publish(); - publish_json_state(path, value) + publish_json_state(path, carried.as_ref().unwrap_or(value)) } /// Why [`merge_json_state_file`] refused an existing state file. diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 56305bb8bb..31bdac110b 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -73,7 +73,7 @@ pub mod windows_driver; pub use amd_smi::{amd_smi_json, device_readable_bounded, resolve_amd_smi_binary}; pub use anthropic_endpoint::{ ANTHROPIC_ENDPOINT_MODE_STATE_KEY, ANTHROPIC_ENDPOINT_URL_STATE_KEY, AnthropicEndpointMode, - anthropic_endpoint_ready, wait_for_published_anthropic_endpoint, + anthropic_endpoint_ready, publish_anthropic_endpoint, wait_for_published_anthropic_endpoint, }; pub use automation::{ AuditEventRecord, AutomationEventRecord, AutomationProposalRecord, AutomationRuntimeState, diff --git a/engines/lemonade/src/state.rs b/engines/lemonade/src/state.rs index d35e2d5992..5a0bd94b86 100644 --- a/engines/lemonade/src/state.rs +++ b/engines/lemonade/src/state.rs @@ -150,6 +150,13 @@ pub(crate) fn read_service_state(path: &Path) -> Result { /// with every merge (`rocm_core::write_json_state_file`), so a full rewrite /// never lands inside another writer's merge, and publishes atomically, so a /// lock-free reader such as `rocm services list` never sees half a document. +/// +/// It replaces every key merged before it, with one exception that +/// `write_json_state_file` owns: an Anthropic endpoint published by this very +/// process is carried through. The proxy that publishes it is a thread of this +/// process and publishes exactly once, so the rewrite that happens moments +/// after the server is spawned would otherwise lose it for good — see +/// `a_running_state_write_keeps_the_anthropic_endpoint_this_process_published`. fn write_state(path: &Path, value: &Value) -> Result<()> { rocm_core::write_json_state_file(path, value) } @@ -310,6 +317,82 @@ mod tests { use std::io::{Read, Write}; use std::time::Duration; + /// ROCMAI-550 regression, on the real writer: the `running` state write + /// happens immediately after `serve_router` spawns the server, and since + /// ROCMAI-550 it spends a host GPU probe (`rocm_core::pinned_gpu_name`, + /// `pnputil` plus a PowerShell CIM query on Windows) building its document + /// before writing it. The Anthropic proxy `rocm` starts on a thread of this + /// same process publishes its endpoint as soon as the server lists a model + /// — inside that window — and nothing publishes twice, so a rewrite that + /// drops the endpoint drops it for the life of the service. That is what + /// left five Windows e2e scenarios with a ready service and no + /// `anthropic endpoint:` row. + /// + /// This drives the production call, not the primitive underneath it, so it + /// also holds the routing: a `write_state` that went back to a plain + /// `fs::write` would fail here while `rocm-core`'s own tests stayed green. + #[test] + fn a_running_state_write_keeps_the_anthropic_endpoint_this_process_published() { + let dir = scratch_dir("anthropic-endpoint-survives-rewrite"); + let request = ServeHttpRequest { + service_id: "svc".to_owned(), + model_ref: "Qwen3-0.6B-GGUF".to_owned(), + host: "127.0.0.1".to_owned(), + port: 11435, + device_policy: rocm_engine_protocol::DevicePolicy::GpuRequired, + gpu_indices: Vec::new(), + runtime_id: None, + env_id: None, + state_path: dir.join("svc.json"), + log_path: None, + engine_recipe: None, + }; + let runtime = LemonadeRuntime { + manifest: LemonadeInstallManifest { + env_id: "test".to_owned(), + version: "test".to_owned(), + runtime_dir: dir.join("runtime"), + lemond: dir.join("lemond"), + lemonade: dir.join("lemonade"), + backend_recipe: crate::LLAMACPP_RECIPE.to_owned(), + backend_name: "rocm".to_owned(), + installed_at_unix_ms: 0, + }, + }; + + // The serve's own first write, then the proxy thread's publication. + write_running_state(&request, &runtime, std::process::id(), None, "starting").unwrap(); + rocm_core::publish_anthropic_endpoint( + &request.state_path, + "http://127.0.0.1:41000", + rocm_core::AnthropicEndpointMode::Proxy, + ) + .unwrap(); + + // The post-spawn rewrite, which replaces the whole document. + write_running_state( + &request, + &runtime, + std::process::id(), + Some(4242), + "running", + ) + .unwrap(); + + let state = read_service_state(&request.state_path).unwrap(); + assert_eq!( + state["anthropic_endpoint_url"], "http://127.0.0.1:41000", + "the rewrite dropped the endpoint its own process published, so \ + `rocm services list` will never show an `anthropic endpoint:` row \ + for this service: {state}" + ); + assert_eq!(state["anthropic_endpoint_mode"], "proxy"); + // The rewrite still does its own job. + assert_eq!(state["status"], "running"); + assert_eq!(state["server_pid"], 4242); + fs::remove_dir_all(dir).ok(); + } + #[test] fn identity_from_state_prefers_server_pid_and_its_start_ticks() { let state = json!({ diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 844631e023..56cd3c4b55 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -2648,12 +2648,42 @@ async fn listed_anthropic_endpoint(world: &mut E2eWorld) -> Option { } } +/// What a missing `anthropic endpoint:` row has to be explained by: the +/// listing, and the engine state file the row is read out of. +/// +/// The row is absent for two very different reasons — the serving process never +/// published, or something wrote the file whole and replaced the key — and the +/// listing alone cannot tell them apart. That cost a round trip to the Windows +/// GPU lane per guess while ROCMAI-550's regression was being chased, so the +/// file goes in the failure message. `--json` is the records verbatim, so it +/// carries `engine_state_path`; it also asks no device probe, so reading it +/// costs nothing a failing step cannot afford. +fn missing_anthropic_endpoint_evidence(world: &mut E2eWorld) -> String { + let listing = world.cli_output.clone().unwrap_or_default(); + let (records, _, _) = crate::run_rocm(world, &["services", "list", "--json"]); + let states = serde_json::from_str::(&records) + .ok() + .and_then(|records| records.as_array().cloned()) + .unwrap_or_default() + .iter() + .filter_map(|record| record.get("engine_state_path")?.as_str().map(str::to_owned)) + .map(|path| { + let body = std::fs::read_to_string(&path) + .unwrap_or_else(|error| format!("")); + format!("{path}:\n{body}") + }) + .collect::>() + .join("\n"); + format!("{listing}\nservices list --json:\n{records}\nengine state:\n{states}") +} + #[then("the service details include an Anthropic-compatible messages endpoint")] async fn assert_service_lists_anthropic_endpoint(world: &mut E2eWorld) { let listed = listed_anthropic_endpoint(world).await; - let output = world.cli_output.clone().unwrap_or_default(); - let endpoint = listed - .unwrap_or_else(|| panic!("no `anthropic endpoint:` row in `services list`:\n{output}")); + let Some(endpoint) = listed else { + let evidence = missing_anthropic_endpoint_evidence(world); + panic!("no `anthropic endpoint:` row in `services list`:\n{evidence}"); + }; assert!( endpoint.starts_with("http://") && !endpoint.contains("/v1"), "the Anthropic endpoint is a base URL clients append `/v1/messages` to, got `{endpoint}`" @@ -2674,9 +2704,11 @@ async fn assert_service_still_lists_openai_endpoint(world: &mut E2eWorld) { /// POST `body` to the served model's Anthropic endpoint and return the JSON /// reply, whatever its status. async fn post_anthropic_messages(world: &mut E2eWorld, body: serde_json::Value) { - let base = listed_anthropic_endpoint(world) - .await - .expect("no Anthropic endpoint is listed for the served model"); + let listed = listed_anthropic_endpoint(world).await; + let Some(base) = listed else { + let evidence = missing_anthropic_endpoint_evidence(world); + panic!("no Anthropic endpoint is listed for the served model:\n{evidence}"); + }; let budget = Duration::from_secs(crate::inference_timeout_for(world)); let reply = tokio::spawn(async move { reqwest::Client::builder() @@ -2823,9 +2855,11 @@ async fn assert_anthropic_response_model_correct(world: &mut E2eWorld) { #[when("the user sends a streaming request to the Anthropic messages endpoint")] async fn user_sends_streaming_anthropic_messages_request(world: &mut E2eWorld) { - let base = listed_anthropic_endpoint(world) - .await - .expect("no Anthropic endpoint is listed for the served model"); + let listed = listed_anthropic_endpoint(world).await; + let Some(base) = listed else { + let evidence = missing_anthropic_endpoint_evidence(world); + panic!("no Anthropic endpoint is listed for the served model:\n{evidence}"); + }; let budget = Duration::from_secs(crate::inference_timeout_for(world)); // Same budget and model name as the non-streaming request above. let body = serde_json::json!({ @@ -2961,11 +2995,11 @@ async fn assert_both_endpoints_healthy(world: &mut E2eWorld) { // `services list` probes each endpoint it shows and marks one that does not // answer with `(not responding)`; a healthy service carries neither mark. let listed = listed_anthropic_endpoint(world).await; + if listed.is_none() { + let evidence = missing_anthropic_endpoint_evidence(world); + panic!("no `anthropic endpoint:` row in `services list`:\n{evidence}"); + } let output = world.cli_output.clone().unwrap_or_default(); - assert!( - listed.is_some(), - "no `anthropic endpoint:` row in `services list`:\n{output}" - ); assert!( output.contains("status: ready"), "the OpenAI-compatible endpoint is not reported ready:\n{output}" From c97969d7c8da409218e3f68cc1ada4f340c1b947 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 17 Nov 2026 17:03:33 +0200 Subject: [PATCH 48/59] fix(vulkan): do not let a software-only enumeration deny an installed GPU driver `classify_vulkan_support` returned a confirmed `Absent` from `devices` alone, before `gpu_icds` was computed on the next line. A host with `radeon_icd.x86_64.json` installed whose `vulkaninfo` lists only `llvmpipe` -- the ordinary shape of a missing `render`-group membership -- was told "no Vulkan GPU driver was found - install one", contradicting the `icds` field of the same report with advice it cannot act on. The ICD evidence is now consulted first, and such a host is `DriverPresentDevicesUnverified` with the software-only enumeration named as the reason nothing was confirmed. The ICD-based software-only fallback could likewise confirm `Absent` without `vulkaninfo` ever running: a lavapipe-only manifest list skips the `icds.is_empty()` guard, so the documented "both sources answered" precondition held for its sibling branch only. That fallback now requires `enumerated`, which makes `vulkaninfo`'s answer an invariant of every `Absent` path rather than a property of most of them. `absent_always_has_vulkaninfos_answer_behind_it` sweeps every reachable combination of the two sources to hold that invariant mechanically. `examine-60` is the e2e half: it plants `examine-58`'s enumeration with a GPU manifest beside it, so reading the devices before the manifests fails exactly one of the pair. Signed-off-by: Roman Sirokov Signed-off-by: Roman Sirokov --- crates/rocm-core/src/host_gpu/vulkan.rs | 291 +++++++++++++++++- docs/testing.md | 16 +- tests/e2e-cucumber/features/examine.feature | 32 +- tests/e2e-cucumber/tests/e2e/examine_steps.rs | 62 ++++ 4 files changed, 376 insertions(+), 25 deletions(-) diff --git a/crates/rocm-core/src/host_gpu/vulkan.rs b/crates/rocm-core/src/host_gpu/vulkan.rs index f7ec68e6c0..69c1f1dd6a 100644 --- a/crates/rocm-core/src/host_gpu/vulkan.rs +++ b/crates/rocm-core/src/host_gpu/vulkan.rs @@ -63,10 +63,19 @@ pub enum VulkanStatus { /// The probe looked and found no GPU Vulkan driver. A software rasteriser /// alone lands here, not in [`Self::Available`]. /// - /// Reachable only when both sources *answer* and answer empty — in - /// practice a `vulkaninfo` that exits 0 having listed no device. A real - /// driverless Linux host does not take this path: there `vulkaninfo` fails - /// to create an instance and exits non-zero, which + /// `vulkaninfo` must have *answered* on every path that reaches this + /// status, and `absent_always_has_vulkaninfos_answer_behind_it` holds that + /// against every combination of evidence. It is the only source strong + /// enough to rule a GPU path out on its own: neither the Linux ICD + /// directory list nor the Windows Khronos key is exhaustive, so "no GPU + /// manifest was listed" never denies a driver by itself. Where nothing at + /// all was found, the ICD listing must have answered too, because an empty + /// `icds` is also what a PowerShell read that failed or timed out produces. + /// + /// In practice that means a `vulkaninfo` that exits 0 having listed no + /// device, or one that listed only software devices with no GPU manifest + /// beside them. A real driverless Linux host does not take this path: there + /// `vulkaninfo` fails to create an instance and exits non-zero, which /// [`vulkan_devices_from_capture`] reads as a source that did not answer, /// landing such a host in [`Self::Inconclusive`]. That is deliberate — see /// [`vulkan_devices_from_capture`] for why a failing probe must not be @@ -253,10 +262,14 @@ impl VulkanSupport { .find(|device| device.is_amd() && device.is_hardware()) .and_then(|device| device.driver_name.clone()) .or_else(|| { - // Restricted to AMD-branded manifests, because this fallback - // only ever runs once an AMD hardware device has *already* been - // confirmed: whatever it names is read as the driver behind - // that device. "Not a known software rasteriser" is too weak a + // Restricted to AMD-branded manifests, because whatever this + // fallback names is read as the driver behind an AMD Vulkan + // path. It runs whenever no AMD hardware device supplied a + // `driverName` of its own — either because none was confirmed + // at all, or because the one that was confirmed went unnamed — + // so the manifest it picks must not be allowed to attribute + // that path to another vendor. "Not a known software + // rasteriser" is too weak a // filter for that job — `detect_icds` sorts alphabetically on // Linux, so on a dual-vendor host carrying both // `nvidia_icd.json` and `radeon_icd.x86_64.json` the NVIDIA @@ -562,8 +575,20 @@ pub fn classify_vulkan_support( }; } + // An enumeration of nothing but software devices denies a GPU path only + // while no GPU driver manifest contradicts it. A host carrying + // `radeon_icd.x86_64.json` whose `vulkaninfo` lists only `llvmpipe` is the + // ordinary shape of a permission problem (no `render` group), not of a + // missing driver — and "no Vulkan GPU driver was found - install one" is + // then both false and unactionable, since the driver it names is already + // installed and printed in `icds` of the same report. The ICD evidence is + // therefore consulted *before* this early return, not after it: such a host + // falls through to the driver-present-devices-unverified arm at the end, + // which names the software-only enumeration as the reason nothing was + // confirmed. let software_only = !devices.is_empty() && devices.iter().all(|device| !device.is_hardware()); - if software_only { + let gpu_icds: Vec<&String> = icds.iter().filter(|icd| icd_is_gpu_driver(icd)).collect(); + if software_only && gpu_icds.is_empty() { return VulkanSupport { status: VulkanStatus::Absent, detail: Some( @@ -574,7 +599,6 @@ pub fn classify_vulkan_support( }; } - let gpu_icds: Vec<&String> = icds.iter().filter(|icd| icd_is_gpu_driver(icd)).collect(); if gpu_icds.is_empty() { // Nothing found at all, and nothing that could have found it: the ICD // search is not exhaustive (a fixed directory list on Linux that @@ -651,9 +675,36 @@ pub fn classify_vulkan_support( devices, }; } - // Reaching the first arm means `icds` is empty *and* both sources - // answered (the branch above returned otherwise), so this is the only - // place entitled to deny the driver outright. + // Two different hosts are left, and only one of them has answered + // enough to be denied a driver. + // + // `icds.is_empty()` here means both sources answered: the guard above + // returned otherwise. That is the empty-everything confirmed absence. + // + // A non-empty `icds` at this point is a software-rasteriser manifest + // and nothing else, and that fact arrives from the *weaker* source. The + // ICD search is documented as non-exhaustive — a fixed directory list + // that ignores `VK_ICD_FILENAMES`/`VK_DRIVER_FILES` on Linux, the + // legacy Khronos key alone on Windows — so "lavapipe is the only + // manifest I can see" does not exclude a GPU driver whose manifest sits + // somewhere this probe never reads. `vulkaninfo` is the source that + // would have seen it, and when it never answered nothing here has ruled + // a GPU path out. Routed to `Inconclusive` for the same reason as the + // guard above, so that every `Absent` this function returns has + // `vulkaninfo`'s answer behind it. + if !icds.is_empty() && !enumerated { + return VulkanSupport { + status: VulkanStatus::Inconclusive, + detail: Some( + "only a software (CPU) Vulkan driver was listed, and `vulkaninfo` could not \ + confirm anything either (not installed, or it failed to run), so nothing here \ + rules out a GPU driver this probe cannot see" + .to_owned(), + ), + icds, + devices, + }; + } let detail = if icds.is_empty() { "no Vulkan driver found".to_owned() } else { @@ -682,8 +733,18 @@ pub fn classify_vulkan_support( || "a Vulkan driver is installed".to_owned(), |label| format!("{label} is installed"), ); - // Three different hosts reach this status and they are not owed the same - // sentence. A non-empty `devices` means `vulkaninfo` answered and the + // Four different hosts reach this status and they are not owed the same + // sentence. + // + // `software_only` is the one that would otherwise be described wrongly by + // the arm below it: saying "none of the devices it enumerated is an AMD + // GPU" of a host that enumerated nothing but `llvmpipe` is true and + // useless, because it reads as "your other vendor's card answered" rather + // than "no GPU answered at all". It is also the arm that must not say a + // driver is absent — a GPU manifest is installed and named in `icds` — so + // it names the exclusion and the likeliest cause instead. + // + // A non-empty `devices` otherwise means `vulkaninfo` answered and the // hardware it listed simply was not AMD — saying no device was enumerated // there is false, and it is the shape an NVIDIA box with an AMD iGPU // disabled lands in. @@ -697,7 +758,13 @@ pub fn classify_vulkan_support( // `render` group, not a missing tool). Only `enumerated == false` is the // case where the tool is genuinely missing or failed to run, and only // there is that advice true. - let detail = if !devices.is_empty() { + let detail = if software_only { + format!( + "{named}, but `vulkaninfo` enumerated only a software (CPU) Vulkan device, which is \ + not a GPU path, so nothing confirms the driver can reach a GPU here (on Linux, check \ + that your user is in the `render` group)" + ) + } else if !devices.is_empty() { format!("{named}, but none of the devices it enumerated is an AMD GPU") } else if enumerated { format!( @@ -875,6 +942,198 @@ GPU0: assert!(icd_is_gpu_driver("radeon_icd.x86_64.json")); } + /// The same lavapipe-only manifest list as the test above, with the one + /// difference that decides whether anything was actually established: + /// `vulkaninfo` never answered. + /// + /// The ICD search is documented as non-exhaustive — a fixed directory list + /// that ignores `VK_ICD_FILENAMES`/`VK_DRIVER_FILES` on Linux, the legacy + /// Khronos key alone on Windows — so "lavapipe is the only manifest I can + /// see" rules out nothing on its own; a proprietary AMD ICD sitting in a + /// directory this probe never reads looks identical. `vulkaninfo` is the + /// source that would have seen it, and here it was not there to ask. + /// + /// Asserted as a pair against the answered case, because the status is the + /// whole finding: an isolated assertion that this is `Inconclusive` would + /// still pass if the sibling were quietly weakened to match it, and the + /// point is that the two hosts earn different answers. + #[test] + fn a_lavapipe_only_install_is_not_a_confirmed_absence_until_vulkaninfo_answers() { + let unasked = classify_vulkan_support( + VulkanEvidence::answered(vec!["lvp_icd.x86_64.json".to_owned()]), + VulkanEvidence::unanswered(), + true, + ); + + assert_eq!( + unasked.status, + VulkanStatus::Inconclusive, + "no source here has looked where a GPU manifest could still be: {unasked:?}" + ); + assert_eq!( + crate::gfx_target::gpu_support_level(Some("gfx90c"), None, unasked.status), + crate::gfx_target::GpuSupportLevel::VulkanUnconfirmed, + "and the level must not render the confirmed-absence advice: {unasked:?}" + ); + assert!( + unasked + .detail + .as_deref() + .is_some_and(|detail| detail.contains("`vulkaninfo` could not confirm")), + "the host is owed which source went unread: {unasked:?}" + ); + + let asked = classify_vulkan_support( + VulkanEvidence::answered(vec!["lvp_icd.x86_64.json".to_owned()]), + VulkanEvidence::answered(Vec::new()), + true, + ); + assert_eq!( + asked.status, + VulkanStatus::Absent, + "while the same manifest list with `vulkaninfo`'s answer behind it has \ + earned the confirmed absence: {asked:?}" + ); + } + + /// `VulkanStatus::Absent` renders as "no Vulkan GPU driver was found - + /// install one", a confirmed fact, so its documented precondition is worth + /// holding mechanically rather than arm by arm: `vulkaninfo` must have + /// answered on every path that reaches it. + /// + /// Swept over every combination of the two sources' answers against a + /// representative set of findings, so a future arm that denies a driver on + /// the ICD listing alone fails here even if it is added somewhere none of + /// the hand-written cases look. + #[test] + fn absent_always_has_vulkaninfos_answer_behind_it() { + let lavapipe = VulkanDevice { + name: "llvmpipe (LLVM 15.0.7, 256 bits)".to_owned(), + driver_name: Some("llvmpipe".to_owned()), + vendor_id: Some("0x10005".to_owned()), + device_type: Some("PHYSICAL_DEVICE_TYPE_CPU".to_owned()), + ..VulkanDevice::default() + }; + let icd_sets: [Vec; 3] = [ + Vec::new(), + vec!["lvp_icd.x86_64.json".to_owned()], + vec!["radeon_icd.x86_64.json".to_owned()], + ]; + let device_sets: [Vec; 2] = [Vec::new(), vec![lavapipe]]; + + for icds in &icd_sets { + for devices in &device_sets { + for icds_listed in [false, true] { + for enumerated in [false, true] { + // An unanswered source has nothing to report, which is + // what `VulkanEvidence::unanswered` encodes; pairing a + // "did not answer" with findings would test a state the + // constructors cannot build. + let icd_evidence = if icds_listed { + VulkanEvidence::answered(icds.clone()) + } else if icds.is_empty() { + VulkanEvidence::unanswered() + } else { + continue; + }; + let device_evidence = if enumerated { + VulkanEvidence::answered(devices.clone()) + } else if devices.is_empty() { + VulkanEvidence::unanswered() + } else { + continue; + }; + + let support = classify_vulkan_support(icd_evidence, device_evidence, true); + + assert!( + support.status != VulkanStatus::Absent || enumerated, + "a confirmed absence without `vulkaninfo`'s answer, from \ + icds={icds:?} devices={devices:?} icds_listed={icds_listed} \ + enumerated={enumerated}: {support:?}" + ); + } + } + } + } + } + + /// The `render`-group host, and the combination neither layer covered: a + /// real GPU driver manifest is installed and `vulkaninfo` enumerates only + /// `llvmpipe`. + /// + /// Telling this host "no Vulkan GPU driver was found - install one" is both + /// false and unactionable — `radeon_icd.x86_64.json` is printed in the + /// `icds` field of the same report — so the device evidence may not deny a + /// driver the ICD evidence is holding. What was actually established is + /// that a driver is installed and nothing confirmed it can reach a GPU, + /// which is exactly `DriverPresentDevicesUnverified`. + /// + /// Held against the software-ICD host beside it, because the two differ + /// only in which manifest is planted: folding the ICD check back out of the + /// `software_only` early return would turn this half into `Absent` while + /// the other half kept passing. + #[test] + fn a_software_only_enumeration_does_not_deny_an_installed_gpu_driver() { + let lavapipe = VulkanDevice { + name: "llvmpipe (LLVM 15.0.7, 256 bits)".to_owned(), + driver_name: Some("llvmpipe".to_owned()), + vendor_id: Some("0x10005".to_owned()), + device_type: Some("PHYSICAL_DEVICE_TYPE_CPU".to_owned()), + ..VulkanDevice::default() + }; + + let with_gpu_driver = classify_vulkan_support( + VulkanEvidence::answered(vec!["radeon_icd.x86_64.json".to_owned()]), + VulkanEvidence::answered(vec![lavapipe.clone()]), + true, + ); + + assert_eq!( + with_gpu_driver.status, + VulkanStatus::DriverPresentDevicesUnverified, + "an installed GPU driver is not absent just because the enumeration was \ + software-only: {with_gpu_driver:?}" + ); + let detail = with_gpu_driver + .detail + .as_deref() + .expect("every classification carries a detail"); + assert!( + detail.contains("Mesa RADV is installed"), + "the driver the report names in `icds` must be named here too: {detail}" + ); + assert!( + detail.contains("software (CPU) Vulkan device"), + "and the reason nothing was confirmed must be the software-only \ + enumeration, not a vendor mismatch: {detail}" + ); + assert!( + detail.contains("`render` group"), + "the likeliest cause is the one step that can clear it: {detail}" + ); + assert_eq!( + crate::gfx_target::gpu_support_level(Some("gfx90c"), None, with_gpu_driver.status), + crate::gfx_target::GpuSupportLevel::VulkanOnly, + "so the user must not reach the level that renders \"install one\": \ + {with_gpu_driver:?}" + ); + + // The same enumeration with only a software manifest beside it: nothing + // contradicts the devices, so this one keeps the confirmed absence. + let software_throughout = classify_vulkan_support( + VulkanEvidence::answered(vec!["lvp_icd.x86_64.json".to_owned()]), + VulkanEvidence::answered(vec![lavapipe]), + true, + ); + assert_eq!( + software_throughout.status, + VulkanStatus::Absent, + "a software rasteriser top to bottom is still not a GPU path: \ + {software_throughout:?}" + ); + } + /// Both halves of this test reach the same *status*; what they must not /// share is the *sentence*. A host whose `vulkaninfo` has already run and /// listed nothing must not be told to "install `vulkaninfo` to confirm" — diff --git a/docs/testing.md b/docs/testing.md index 23c23231c8..e4ec4c93c0 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -4451,7 +4451,7 @@ levels (`gpu_support_level`): ROCm/HIP-supported, Vulkan-only, Vulkan-unconfirmed, no-GPU-backend, or unknown. Unknown covers both "nothing detected" and "a target this build's catalog does not recognize" — conflating the latter with "confirmed to have no ROCm build" would turn a detection gap -into a claim about the hardware. `examine-49` holds the line against the +into a claim about the hardware. `examine-55` holds the line against the `detected_gfx_target` and `compatible_therock_family` the same report prints, so every lane checks the level its own hardware implies: @@ -4469,17 +4469,27 @@ evidentiary state rather than a different wording of one: | `examine-57` | A `vulkaninfo` that runs and reports nothing. | no GPU backend | | `examine-58` | Only a software rasteriser (lavapipe). | no GPU backend | | `examine-59` | No ICD, no device, and no `vulkaninfo` to ask. | Vulkan-unconfirmed | +| `examine-60` | A GPU ICD whose `vulkaninfo` enumerates only a software rasteriser. | Vulkan-only, through `DriverPresentDevicesUnverified` | ```bash -cargo xtask e2e -- -n '(examine-56|examine-57|examine-58|examine-59)' +cargo xtask e2e -- -n '(examine-56|examine-57|examine-58|examine-59|examine-60)' ``` -`examine-51` and `examine-53` are the pair worth keeping straight: both end +`examine-57` and `examine-59` are the pair worth keeping straight: both end with nothing found, and they differ only in whether a tool was there to look. That difference is the whole reason `VulkanUnconfirmed` exists, so a change that collapses them will pass one of these and fail the other rather than silently reporting a confirmed absence to a host nobody probed. +`examine-58` and `examine-60` are the second such pair, and they differ only in +which ICD manifest is planted beside the same software-only enumeration. A +lavapipe manifest leaves nothing to contradict the devices, so that host is a +confirmed absence; `radeon_icd.x86_64.json` is a GPU driver the report itself +lists, so telling that host "no Vulkan GPU driver was found - install one" +would contradict the same report's `icds` field with advice it cannot act on. +`examine-60` is also the only scenario that reaches +`VulkanStatus::DriverPresentDevicesUnverified` at all. + ## Windows Tool Notes The TheRock SDK wheel install path should not require users to install global diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index 372622d838..74c7bcdeb4 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -821,9 +821,9 @@ Feature: GPU detection and system inspection # of which the harness can plant. So these run on the ungated mock lane, which # is the lane that has neither. # - # Each pins one of the three answers a probe can reach where inference from the - # gfx table reaches only one: "no ROCm build" does not mean "Vulkan then" on a - # machine nobody has looked at, and the three scenarios are what keep those + # Each pins one of the answers a probe can reach where inference from the gfx + # table reaches only one: "no ROCm build" does not mean "Vulkan then" on a + # machine nobody has looked at, and these scenarios are what keep those # answers distinguishable. @id:examine-names-the-vulkan-driver-an-unsupported-gpu-can-use @requires-os:linux Scenario: examine-56 - A GPU the official stack rejects is told which way it can still run models @@ -859,17 +859,17 @@ Feature: GPU detection and system inspection Then the inspection does not claim models can run through Vulkan And the inspection says the only renderer it found runs on the CPU - # The shape examine-51 does not reach. examine-51 plants a `vulkaninfo` that + # The shape examine-57 does not reach. examine-57 plants a `vulkaninfo` that # runs and reports no devices, which is the only shape that earns a confirmed # absence; this host has no `vulkaninfo` to ask at all, and must be told the # check could not be made rather than either promised Vulkan or declared to # have nothing. # # Known gap, measured rather than assumed: a real `vulkaninfo` on a Linux host - # with no reachable driver does NOT produce examine-51's shape. It prints + # with no reachable driver does NOT produce examine-57's shape. It prints # "Found no drivers!" and exits 1, which this probe classifies as # inconclusive — so such a host lands at *this* scenario's level rather than - # examine-51's. Telling "no drivers" apart from "the tool is broken" needs the + # examine-57's. Telling "no drivers" apart from "the tool is broken" needs the # loader's own message, which the shared capture discards. @id:examine-cannot-confirm-vulkan-without-vulkaninfo @requires-os:linux Scenario: examine-59 - A machine that cannot be checked for Vulkan is not told either way @@ -878,3 +878,23 @@ Feature: GPU detection and system inspection Then the inspection reports the GPU as unsupported by the official stack And the inspection does not claim models can run through Vulkan And the inspection says it could not confirm whether a Vulkan driver is there + + # examine-58 with one thing changed: the manifest beside the software renderer + # names a real GPU driver. That is the common `render`-group permission shape + # -- the driver is installed, and the user's own account cannot open the + # device, so the loader falls back to llvmpipe -- and it is the one case where + # the device evidence must NOT be allowed to deny the driver. "No Vulkan GPU + # driver was found - install one" would contradict the `icds` field of the + # same report and name an install the host has already done. + # + # The pair is the test: examine-58 and this scenario plant the same + # enumeration and differ only in the manifest, so an implementation that reads + # the devices before the manifests passes one and fails the other instead of + # collapsing both into a confirmed absence. + @id:examine-does-not-deny-an-installed-vulkan-driver-a-software-renderer-masks @requires-os:linux + Scenario: examine-60 - An installed GPU driver masked by a software renderer is not reported as missing + Given a machine whose GPU has no official support and a Vulkan driver masked by a software renderer + When the user inspects that machine + Then the inspection reports the GPU as unsupported by the official stack + And the inspection does not say a Vulkan driver is missing + And the inspection names the installed Vulkan driver it could not confirm diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 3fd6865ebf..828ea5a4cf 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -3855,6 +3855,68 @@ async fn unsupported_gpu_with_software_renderer(world: &mut E2eWorld) { ); } +/// The `render`-group shape: a real GPU driver manifest the loader would find, +/// and a `vulkaninfo` that enumerates only the CPU rasteriser because this user +/// cannot open the device node. +/// +/// Deliberately the *same* enumeration as the step above, with only the +/// manifest changed. The two scenarios are a discrimination rather than two +/// assertions: the device evidence alone cannot tell the hosts apart, so an +/// implementation that reads it before the ICD evidence reports both as a +/// confirmed absence and fails exactly one of the pair. +#[given( + "a machine whose GPU has no official support and a Vulkan driver masked by a software renderer" +)] +async fn unsupported_gpu_with_masked_vulkan_driver(world: &mut E2eWorld) { + plant_unsupported_gpu( + world, + Some("radeon_icd.x86_64.json"), + Some(LAVAPIPE_VULKANINFO_SUMMARY), + ); +} + +#[then("the inspection does not say a Vulkan driver is missing")] +async fn assert_vulkan_driver_not_denied(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + assert!( + !output.contains("no Vulkan GPU driver was found"), + "a driver this very report lists in `icds` must not be declared missing:\n{output}" + ); + assert!( + !output.contains("install one"), + "nor may the host be sent to install what it already has:\n{output}" + ); + assert!( + output.contains("vulkan: driver_present_devices_unverified"), + "the machine-readable status must say the driver is there and unconfirmed:\n{output}" + ); +} + +#[then("the inspection names the installed Vulkan driver it could not confirm")] +async fn assert_masked_vulkan_driver_named(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + let line = rocm_support_line(output).expect("no rocm_support line"); + assert!( + line.contains("a Vulkan driver is installed"), + "the support sentence must say what was actually established: {line}\n{output}" + ); + // The evidence line is where a reader can check that claim: it has to name + // the driver the manifest implies and the reason nothing was confirmed, or + // "a driver is installed" is just another unbacked promise. + assert!( + output.contains("Mesa RADV is installed"), + "the evidence line must name the driver the planted manifest implies:\n{output}" + ); + assert!( + output.contains("software (CPU) Vulkan device"), + "and why it could not be confirmed:\n{output}" + ); + assert!( + !output.contains("the Vulkan backend can be used"), + "but an unconfirmed driver is not a confirmed device:\n{output}" + ); +} + #[then("the inspection reports the GPU as unsupported by the official stack")] async fn assert_gpu_unsupported_by_rocm(world: &mut E2eWorld) { let output = world.cli_output.as_ref().expect("no command was run"); From 21b238392b9d66dc85b7ca58eaf4dc50fd4e3646 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 17 Nov 2026 17:03:34 +0200 Subject: [PATCH 49/59] fix(engines): stop AIM claiming a GPU it was never given, and write state under the lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AIM's `reported_device` hardcoded `rocm` and probed the host for a GPU name whatever the launch's `device_policy` was, so a `cpu_only` container — which `resolve_use_gpu`/`docker::RunSpec` correctly give no /dev/kfd, no /dev/dri and no HIP_VISIBLE_DEVICES — still rendered e.g. `rocm (AMD Instinct MI300X)`. That is the invented device claim this branch exists to remove, shipped in compiled code even though `parse_device_policy` currently refuses `cpu` at the CLI. The policy the launch recorded now decides, and the probe is not made at all when no GPU was attached. AIM also recorded no `gpu_indices`, so on a multi-GPU host it answered a bare `rocm` where lemonade and vLLM answer `rocm gpu ()` for the same hardware — the engine does pin an ordinal, `plan_gpus` resolves it and `HIP_VISIBLE_DEVICES` narrows the container to it. `run_container` now reports the ordinals it attached so the state file and the container cannot disagree. Both engines' `write_state` were a lock-free `fs::write` that bypassed `carry_published_anthropic_endpoint`. The unlocked write predates this branch, but the slow host GPU probe that makes the race practically reachable is added to these two state writes by this branch itself, so the exposure is this branch's to close: routing them through `rocm_core::write_json_state_file` orders them against every merge and carries a sibling thread's publication through the rewrite, as lemonade's already does. The status writers patch through `merge_json_state_file` instead of rewriting the document whole. `healthcheck_service_at` is split out of both handlers, the way lemonade's already was, so a test can drive the real RPC entry point: every device test in the three engines called the private formatter, so reverting a handler's wiring to a hardcoded literal kept the suite green. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/anthropic_endpoint.rs | 8 + crates/rocm-core/src/endpoint_readiness.rs | 5 +- engines/aim/src/process.rs | 47 +++- engines/aim/src/state.rs | 302 ++++++++++++++++----- engines/vllm/src/state.rs | 136 ++++++++-- 5 files changed, 393 insertions(+), 105 deletions(-) diff --git a/crates/rocm-core/src/anthropic_endpoint.rs b/crates/rocm-core/src/anthropic_endpoint.rs index 6d0314d33b..93e8eb7a3d 100644 --- a/crates/rocm-core/src/anthropic_endpoint.rs +++ b/crates/rocm-core/src/anthropic_endpoint.rs @@ -19,6 +19,14 @@ //! file, and the engine adapters write it whole. [`publish_anthropic_endpoint`] //! and `carry_published_anthropic_endpoint` are the two halves that keep the //! whole-file write from dropping what the sibling thread published. +//! +//! The carry reaches a whole-file write only when that write goes through +//! [`crate::write_json_state_file`], which is +//! `carry_published_anthropic_endpoint`'s single caller. All three built-in +//! adapters' `write_state` do (`engines/lemonade`, `engines/vllm`, +//! `engines/aim`). An out-of-tree engine plugin that writes the state file +//! itself takes neither this nor the state directory's lock, and would drop +//! the publication. use std::collections::HashMap; use std::path::{Path, PathBuf}; diff --git a/crates/rocm-core/src/endpoint_readiness.rs b/crates/rocm-core/src/endpoint_readiness.rs index b19f9a95e9..63335b490a 100644 --- a/crates/rocm-core/src/endpoint_readiness.rs +++ b/crates/rocm-core/src/endpoint_readiness.rs @@ -70,8 +70,9 @@ pub fn json_state_lock_path(path: &Path) -> PathBuf { /// run under one cross-process lock ([`json_state_lock_path`]) and no merge /// can drop a key another merge recorded. The lock orders writers rather than /// combining them: a full rewrite through [`write_json_state_file`] still -/// replaces every key merged before it, and a writer that bypasses both -/// functions (the vLLM and AIM engines' own state writes) takes no lock at all. +/// replaces every key merged before it. Every built-in engine adapter's state +/// writes go through one of the two; a writer that bypasses both (an +/// out-of-tree engine plugin writing the file itself) takes no lock at all. /// /// The result is published atomically /// ([`crate::atomic_write::write_file_atomically`]), so a reader that takes no diff --git a/engines/aim/src/process.rs b/engines/aim/src/process.rs index b6460269f4..10681d5e69 100644 --- a/engines/aim/src/process.rs +++ b/engines/aim/src/process.rs @@ -164,7 +164,7 @@ pub(crate) fn launch_service(request: LaunchRequest) -> Result { let state_path = crate::state::state_path(&request.service_id); - let served_model_id = run_container( + let run = run_container( &request.service_id, &request.model_ref, &request.host, @@ -178,10 +178,11 @@ pub(crate) fn launch_service(request: LaunchRequest) -> Result { &state_path, &request.service_id, &request.model_ref, - served_model_id.as_deref(), + run.canonical_name.as_deref(), &request.host, request.port, device_policy, + &run.gpu_indices, engine_recipe.as_ref(), )?; @@ -246,8 +247,22 @@ fn plan_gpus(requested: &[u32], usable: Option>, windows: bool) -> Resu }) } +/// What [`run_container`] established about the container it left running. +#[derive(Debug, Default, PartialEq, Eq)] +struct ContainerRun { + /// The model name the image says it serves under, if known. + canonical_name: Option, + /// The GPU ordinals the container was narrowed to through + /// `HIP_VISIBLE_DEVICES`, for [`crate::state::write_running_state`] to + /// record so the `device:` row can name one (ROCMAI-550). Empty when no + /// GPU was attached, and when the container was already running — that + /// launch chose the ordinals and this call cannot know which. + gpu_indices: Vec, +} + /// Starts the named service's container unless it is already running, and -/// returns the model name the image says it serves under, if known. +/// reports the model name the image says it serves under together with the +/// GPUs the container was given. /// /// Container names are unique in Docker, so `docker run --name X` against a /// name still held by an exited container (a crash, a reboot, a failed @@ -261,15 +276,18 @@ fn run_container( device_policy: &DevicePolicy, requested_gpus: &[u32], extra_env: Option<&BTreeMap>, -) -> Result> { +) -> Result { let name = crate::docker::container_name(service_id); // Normalize once so the pull, the metadata and the run all use the same image. let image_ref = crate::registry::normalize_image_ref(model_ref); match crate::docker::container_state(&name)? { ContainerState::Running => { - return Ok(crate::registry::local_image_metadata(&image_ref) - .and_then(|metadata| metadata.canonical_name)); + return Ok(ContainerRun { + canonical_name: crate::registry::local_image_metadata(&image_ref) + .and_then(|metadata| metadata.canonical_name), + gpu_indices: Vec::new(), + }); } ContainerState::Exited(_) => crate::docker::remove(&name)?, ContainerState::Missing => {} @@ -313,6 +331,9 @@ fn run_container( if let Some(api_key) = rocm_engine_protocol::resolve_endpoint_api_key() { env.insert("VLLM_API_KEY".to_owned(), api_key); } + // The one place the attached ordinals are decided, so the container and the + // state file cannot disagree about them. + let gpu_indices = if use_gpu { gpus.indices } else { Vec::new() }; crate::docker::run(&crate::docker::RunSpec { name: &name, image: &image_ref, @@ -320,11 +341,14 @@ fn run_container( port, container_port: crate::DEFAULT_AIM_PORT, use_gpu, - gpu_indices: if use_gpu { &gpus.indices } else { &[] }, + gpu_indices: &gpu_indices, render_gid, env: &env, })?; - Ok(metadata.canonical_name) + Ok(ContainerRun { + canonical_name: metadata.canonical_name, + gpu_indices, + }) } /// Enforces `device_policy` instead of treating GPU use as a soft heuristic: @@ -357,7 +381,7 @@ fn resolve_use_gpu( pub(crate) fn serve_http(request: ServeHttpRequest) -> Result<()> { let name = crate::docker::container_name(&request.service_id); - let label_name = run_container( + let run = run_container( &request.service_id, &request.model_ref, &request.host, @@ -377,10 +401,11 @@ pub(crate) fn serve_http(request: ServeHttpRequest) -> Result<()> { &request.state_path, &request.service_id, &request.model_ref, - label_name.as_deref(), + run.canonical_name.as_deref(), &request.host, request.port, request.device_policy, + &run.gpu_indices, request.engine_recipe.as_ref(), )?; @@ -391,7 +416,7 @@ pub(crate) fn serve_http(request: ServeHttpRequest) -> Result<()> { &name, &request.host, request.port, - label_name.as_deref(), + run.canonical_name.as_deref(), aim_ready_timeout(), ) { Ok(served_model_id) => { diff --git a/engines/aim/src/state.rs b/engines/aim/src/state.rs index 500a4508b5..b7e1ac6077 100644 --- a/engines/aim/src/state.rs +++ b/engines/aim/src/state.rs @@ -17,8 +17,19 @@ const HEALTHCHECK_TIMEOUT_MS: u64 = 700; pub(crate) fn healthcheck_service(request: HealthcheckRequest) -> Result { require_nonempty(&request.service_id, "service_id")?; - let state_path = state_path(&request.service_id); - let state = read_service_state(&state_path).ok(); + healthcheck_service_at(&state_path(&request.service_id)) +} + +/// [`healthcheck_service`] against an explicit state path. +/// +/// Split out for the same reason `engines/lemonade`'s `healthcheck_service_at` +/// is: [`state_path`] resolves through the caller's real `AppPaths`, so the +/// RPC handler itself could not be driven from a test, and the `device` field +/// it answers with was only ever covered by calling [`reported_device`] +/// directly — a test that stays green if this handler stops consulting it at +/// all (see `healthcheck_rpc_answers_with_the_device_the_state_file_describes`). +pub(crate) fn healthcheck_service_at(state_path: &Path) -> Result { + let state = read_service_state(state_path).ok(); let endpoint_url = state .as_ref() @@ -42,7 +53,7 @@ pub(crate) fn healthcheck_service(request: HealthcheckRequest) -> Result Result String { + if recorded_device_policy(state) == Some(DevicePolicy::CpuOnly) { + return rocm_engine_protocol::format_reported_device("cpu", None, None); + } rocm_engine_protocol::format_reported_device( "rocm", - None, + rocm_engine_protocol::first_gpu_index_from_state(state), value_string(state, "gpu_name").as_deref(), ) } +/// The `device_policy` the launch recorded, or `None` when the state file +/// predates the key or holds something that is not one of the three policies. +fn recorded_device_policy(state: &Value) -> Option { + serde_json::from_value(state.get("device_policy")?.clone()).ok() +} + pub(crate) fn endpoint_response(request: EndpointRequest) -> Result { require_nonempty(&request.service_id, "service_id")?; let state = read_service_state(&state_path(&request.service_id)) @@ -131,9 +158,13 @@ pub(crate) fn write_running_state( host: &str, port: u16, device_policy: DevicePolicy, + gpu_indices: &[u32], engine_recipe: Option<&EngineRecipeHint>, ) -> Result<()> { let endpoint = endpoint_url(host, port); + // No GPU was attached, so there is no device to probe for or name. See + // [`reported_device`] for why this is gated rather than probed anyway. + let uses_gpu = device_policy != DevicePolicy::CpuOnly; // `extra_env` can carry secrets (e.g. an HF token passed via `--engine-env`); // this state file is status/introspection only (never read back to relaunch // the container, see `crate::process::launch_service`), so redact values @@ -152,16 +183,23 @@ pub(crate) fn write_running_state( "port": port, "endpoint_url": endpoint, "device_policy": device_policy, + // The ordinals `process::plan_gpus` resolved and `docker::RunSpec` + // narrowed the container to with `HIP_VISIBLE_DEVICES` — the same + // key, read the same way, as `engines/lemonade` and + // `engines/vllm`, so one multi-GPU host does not report + // `rocm gpu 1 (…)` for those two and a bare `rocm` for this one. + // Empty for a CPU-only launch, and for a container this call found + // already running (that launch pinned it, and this one cannot know + // what it chose). + "gpu_indices": gpu_indices, // The host's GPU name, so the `device:` row names a device and not - // only a backend (ROCMAI-550). This engine records no GPU ordinal, - // so without it its row would be the bare word `rocm` on every - // host. Captured once here rather than probed per healthcheck: - // `rocm services list` healthchecks every ready service on its - // render path. The empty ordinal list is the literal truth — the - // container is handed a device list, not an ordinal — and - // `rocm_core::pinned_gpu_name` still declines on any host where - // more than one AMD GPU could be the one that answered. - "gpu_name": rocm_core::pinned_gpu_name(&[]), + // only a backend (ROCMAI-550). Captured once here rather than + // probed per healthcheck: `rocm services list` healthchecks every + // ready service on its render path. `rocm_core::pinned_gpu_name` + // declines on any host where more than one AMD GPU could be the + // one that answered, and is not asked at all when no GPU was + // attached. + "gpu_name": uses_gpu.then(|| rocm_core::pinned_gpu_name(gpu_indices)).flatten(), "engine_recipe": engine_recipe, "started_at_unix_ms": current_unix_millis(), }), @@ -182,24 +220,26 @@ pub(crate) fn read_status(state_path: &Path) -> Option { value_string(&read_service_state(state_path).ok()?, "status") } +/// Patch one key into the service's state file under the state directory's +/// lock ([`rocm_core::merge_json_state_file`]) — the same lock the +/// inference-verification latch and `rocmd`'s Anthropic-endpoint publish take, +/// so no writer drops a key another writer recorded. +/// +/// It used to read, patch and `fs::write` with no lock at all, which could +/// drop either of those. A state file that is empty, torn or not a JSON object +/// is now an error and left as it was, rather than replaced wholesale. fn merge_state_field(state_path: &Path, key: &str, value: Value) -> Result<()> { - let mut state = read_service_state(state_path).unwrap_or_else(|_| json!({})); - if let Some(object) = state.as_object_mut() { - object.insert(key.to_owned(), value); - } - write_state(state_path, &state) + rocm_core::merge_json_state_file(state_path, &json!({ key: value })) } pub(crate) fn write_terminal_state(state_path: &Path, status: &str) -> Result<()> { - let mut state = read_service_state(state_path).unwrap_or_else(|_| json!({})); - if let Some(object) = state.as_object_mut() { - object.insert("status".to_owned(), json!(status)); - object.insert( - "stopped_at_unix_ms".to_owned(), - json!(current_unix_millis()), - ); - } - write_state(state_path, &state) + rocm_core::merge_json_state_file( + state_path, + &json!({ + "status": status, + "stopped_at_unix_ms": current_unix_millis(), + }), + ) } pub(crate) fn read_service_state(path: &Path) -> Result { @@ -208,22 +248,20 @@ pub(crate) fn read_service_state(path: &Path) -> Result { serde_json::from_str(&text).with_context(|| format!("failed to parse {}", path.display())) } -/// Serializes `value` to `path` as pretty JSON, creating the parent directory -/// if needed. The low-level primitive [`write_running_state`], -/// [`merge_state_field`] and [`write_terminal_state`] all go through — it is -/// not the only way this module's state files are written: -/// inference-verification latches go through -/// `rocm_core::merge_json_state_file` instead (see [`inference_verified`]), -/// which patches the file under the state directory's lock rather than -/// routing through here. This function takes no such lock and writes in -/// place, so the lock does not order it against those latches. +/// Replace the service's state file whole, under the state directory's lock +/// and published atomically ([`rocm_core::write_json_state_file`]). +/// +/// [`write_running_state`] is the only caller; the status writers patch +/// ([`merge_state_field`]) instead. It used to be a lock-free `fs::write`, +/// which is a problem this PR's own change made reachable: the document being +/// written now includes a host GPU probe (`rocm_core::pinned_gpu_name`, up to +/// seconds on Windows), and `rocmd`'s Anthropic-proxy thread — a thread of +/// *this* process — publishes its endpoint into the same file in that window. +/// Going through `write_json_state_file` both orders this against every merge +/// and carries that publication through the rewrite, the way +/// `engines/lemonade`'s `write_state` does. fn write_state(path: &Path, value: &Value) -> Result<()> { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .with_context(|| format!("failed to create {}", parent.display()))?; - } - fs::write(path, serde_json::to_vec_pretty(value)?) - .with_context(|| format!("failed to write {}", path.display())) + rocm_core::write_json_state_file(path, value) } pub(crate) fn endpoint_url(host: &str, port: u16) -> String { @@ -389,6 +427,7 @@ mod tests { "127.0.0.1", 8000, DevicePolicy::GpuPreferred, + &[0], Some(&hint), ) .unwrap(); @@ -410,8 +449,8 @@ mod tests { /// is covered by /// `healthcheck_reports_unknown_for_a_service_with_no_state_file`). /// - /// Reverting the string to `"rocm_gpu"`, or dropping the `gpu_name` field - /// from the state write, fails this. + /// Reverting the string to `"rocm_gpu"`, dropping the `gpu_name` field or + /// dropping the `gpu_indices` field from the state write fails this. #[test] fn reported_device_for_existing_state_matches_the_shared_engine_shape() { let dir = tempfile::tempdir().unwrap(); @@ -424,42 +463,179 @@ mod tests { "127.0.0.1", 8000, DevicePolicy::GpuRequired, + &[1], None, ) .unwrap(); // The probed name is whatever the test host has, so assert the shape - // rather than a value: `rocm`, optionally followed by `()`, and - // never the old `rocm_gpu` or a `gpu` noun with no ordinal behind it. + // rather than a value: `rocm gpu 1`, optionally followed by + // `()`, and never the old `rocm_gpu`. let state = read_service_state(&path).unwrap(); let device = reported_device(&state); assert!( - device == "rocm" || (device.starts_with("rocm (") && device.ends_with(')')), - "AIM must report the shared ` [()]` shape: {device}" + device == "rocm gpu 1" || (device.starts_with("rocm gpu 1 (") && device.ends_with(')')), + "AIM must report the shared ` gpu [()]` shape, with the \ + ordinal the container was narrowed to: {device}" ); assert!( !device.contains("rocm_gpu"), "the pre-ROCMAI-550 literal must not come back: {device}" ); - assert!( - !device.split_whitespace().any(|word| word == "gpu"), - "this engine pins no ordinal, so a bare `gpu` noun would name \ - nothing: {device}" - ); - // The state file must actually carry the key the row is built from, + // The state file must actually carry both keys the row is built from, // even when the host probe found no name. assert!( state.get("gpu_name").is_some(), "write_running_state must record gpu_name: {state}" ); + assert_eq!( + state.get("gpu_indices"), + Some(&json!([1])), + "write_running_state must record the ordinals the container was \ + given, the way lemonade and vLLM do: {state}" + ); // And the name, when the host has one, reaches the row. + assert_eq!( + reported_device(&json!({ "gpu_indices": [0], "gpu_name": "AMD Instinct MI300X" })), + "rocm gpu 0 (AMD Instinct MI300X)" + ); + } + + /// A `cpu_only` launch attaches no GPU at all (`process::resolve_use_gpu` + /// returns `false`, and `docker::RunSpec` then passes no `/dev/kfd`, no + /// `/dev/dri` and no `HIP_VISIBLE_DEVICES`), so neither the `rocm` backend + /// nor a GPU name is a fact about it. Before ROCMAI-550's fix this path + /// still probed the host and rendered e.g. + /// `rocm (AMD Instinct MI300X)` for a container that used no GPU — the + /// invented claim the whole ticket exists to remove. + /// + /// Drops to red if `write_running_state` probes for a name regardless of + /// the policy, or if `reported_device` goes back to hardcoding `rocm`. + #[test] + fn a_cpu_only_launch_claims_neither_rocm_nor_a_gpu_name() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("svc-cpu.json"); + write_running_state( + &path, + "svc-cpu", + "some/model", + Some("Org/Served"), + "127.0.0.1", + 8000, + DevicePolicy::CpuOnly, + &[], + None, + ) + .unwrap(); + + let state = read_service_state(&path).unwrap(); + assert_eq!( + state.get("gpu_name"), + Some(&Value::Null), + "a cpu-only launch must not have the host GPU probed for it: {state}" + ); + assert_eq!(reported_device(&state), "cpu"); + + // And the rendering is driven by the recorded policy, not by the + // absence of a name: a state file that somehow carries both still + // reports `cpu`. + assert_eq!( + reported_device(&json!({ + "device_policy": "cpu_only", + "gpu_indices": [0], + "gpu_name": "AMD Instinct MI300X", + })), + "cpu" + ); + // A state file written before the key existed keeps the old reading. assert_eq!( reported_device(&json!({ "gpu_name": "AMD Instinct MI300X" })), "rocm (AMD Instinct MI300X)" ); } + /// The `device` the Healthcheck RPC *itself* answers with, not the private + /// formatter it is supposed to call. + /// + /// Every other device test in the three engines calls `reported_device` + /// directly, so reverting the handler's `Some(state) => reported_device` + /// wiring to a hardcoded literal kept the whole suite green. This drives + /// the handler. + /// + /// The planted state carries no endpoint, so no HTTP probe is made and + /// `listed`/`ready` are false — which is the point: this engine reports + /// the device as soon as it has state, independently of readiness (see + /// `apps/rocm/src/engine_ipc.rs`'s `device_worth_reporting`). + #[test] + fn healthcheck_rpc_answers_with_the_device_the_state_file_describes() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("svc-rpc.json"); + write_state( + &path, + &json!({ + "service_id": "svc-rpc", + "status": "running", + "device_policy": "gpu_required", + "gpu_indices": [2], + "gpu_name": "AMD Instinct MI300X", + }), + ) + .unwrap(); + + let response = healthcheck_service_at(&path).unwrap(); + assert_eq!(response.device, "rocm gpu 2 (AMD Instinct MI300X)"); + assert_eq!(response.status, "running"); + assert!(!response.model_loaded); + + // And a service with no state on disk is the `"unknown"` the CLI + // filters out rather than prints. + let response = healthcheck_service_at(&dir.path().join("absent.json")).unwrap(); + assert_eq!(response.device, "unknown"); + } + + /// The post-spawn whole-file rewrite must not drop the Anthropic endpoint + /// a sibling thread of this same process published moments earlier. + /// + /// `rocm`'s `run_builtin_engine_serve_http` runs `rocmd`'s Anthropic proxy + /// as a thread of the process running this adapter, and ROCMAI-550 put a + /// host GPU probe (seconds, on Windows) inside the rewrite — wide enough + /// for the publish to land inside it. Routing [`write_state`] through + /// `rocm_core::write_json_state_file` is what carries it through; putting + /// the raw `fs::write` back drops the key and fails this. + #[test] + fn a_running_state_write_keeps_the_anthropic_endpoint_this_process_published() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("svc-anthropic.json"); + rocm_core::publish_anthropic_endpoint( + &path, + "http://127.0.0.1:9931", + rocm_core::AnthropicEndpointMode::Proxy, + ) + .unwrap(); + + write_running_state( + &path, + "svc-anthropic", + "some/model", + None, + "127.0.0.1", + 8000, + DevicePolicy::GpuRequired, + &[0], + None, + ) + .unwrap(); + + let state = read_service_state(&path).unwrap(); + assert_eq!( + value_string(&state, rocm_core::ANTHROPIC_ENDPOINT_URL_STATE_KEY).as_deref(), + Some("http://127.0.0.1:9931"), + "the whole-file rewrite dropped the endpoint the proxy thread \ + published: {state}" + ); + } + #[test] fn status_helpers_round_trip_and_keep_other_fields() { let dir = tempfile::tempdir().unwrap(); diff --git a/engines/vllm/src/state.rs b/engines/vllm/src/state.rs index 540a6dae44..7acaf5ae0f 100644 --- a/engines/vllm/src/state.rs +++ b/engines/vllm/src/state.rs @@ -34,7 +34,19 @@ pub(crate) struct ServiceFiles { pub(crate) fn healthcheck_service(request: HealthcheckRequest) -> Result { require_nonempty(&request.service_id, "service_id")?; let files = service_files(&request.service_id)?; - let state = read_service_state(&files.state_path).ok(); + healthcheck_service_at(&files.state_path) +} + +/// [`healthcheck_service`] against an explicit state path. +/// +/// Split out the way `engines/lemonade`'s `healthcheck_service_at` already is: +/// [`service_files`] resolves through the caller's real `AppPaths`, so the RPC +/// handler itself could not be driven from a test, and the `device` it answers +/// with was only covered by calling [`reported_device`] directly — a test that +/// stays green if this handler stops consulting it at all (see +/// `healthcheck_rpc_answers_with_the_device_the_state_file_describes`). +pub(crate) fn healthcheck_service_at(state_path: &Path) -> Result { + let state = read_service_state(state_path).ok(); let endpoint_url = state.as_ref().and_then(endpoint_url_from_state); let model_ref = state .as_ref() @@ -51,7 +63,7 @@ pub(crate) fn healthcheck_service(request: HealthcheckRequest) -> Result Result<()> { - let mut state = read_service_state(state_path).unwrap_or_else(|_| json!({})); - if let Some(object) = state.as_object_mut() { - object.insert("status".to_owned(), Value::String(status.to_owned())); - object.insert( - "stopped_at_unix_ms".to_owned(), - Value::from(current_unix_millis() as u64), - ); - } - write_state(state_path, &state) + rocm_core::merge_json_state_file( + state_path, + &json!({ + "status": status, + "stopped_at_unix_ms": current_unix_millis() as u64, + }), + ) } pub(crate) fn read_service_state(path: &Path) -> Result { @@ -223,25 +237,20 @@ pub(crate) fn read_service_state(path: &Path) -> Result { serde_json::from_str(&text).with_context(|| format!("failed to parse {}", path.display())) } -/// Serializes `value` to `path` as pretty JSON, creating the parent directory -/// if needed. The low-level primitive [`write_running_state`] and -/// [`write_terminal_state`] both go through — it is not the only way this -/// module's state files are written: inference-verification latches go -/// through `rocm_core::merge_json_state_file` instead (see -/// [`inference_verified`]), which patches the file under the state -/// directory's lock rather than routing through here. This function takes no -/// such lock and writes in place, so the lock does not order it against those -/// latches. +/// Replace the service's state file whole, under the state directory's lock +/// and published atomically ([`rocm_core::write_json_state_file`]). +/// +/// [`write_running_state`] is the only caller; the terminal status is a patch +/// ([`write_terminal_state`]). It used to be a lock-free `fs::write`, which is +/// a problem this PR's own change made reachable: the document being written +/// now includes a host GPU probe (`rocm_core::pinned_gpu_name`, up to seconds +/// on Windows), and `rocmd`'s Anthropic-proxy thread — a thread of *this* +/// process — publishes its endpoint into the same file in that window. Going +/// through `write_json_state_file` both orders this against every merge and +/// carries that publication through the rewrite, as `engines/lemonade`'s +/// `write_state` does. fn write_state(path: &Path, value: &Value) -> Result<()> { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .with_context(|| format!("failed to create {}", parent.display()))?; - } - fs::write( - path, - serde_json::to_vec_pretty(value).context("failed to serialize vLLM state")?, - ) - .with_context(|| format!("failed to write {}", path.display())) + rocm_core::write_json_state_file(path, value) } pub(crate) fn endpoint_url(host: &str, port: u16) -> String { @@ -769,4 +778,73 @@ mod tests { "rocm (AMD Instinct MI300X)" ); } + + /// The `device` the Healthcheck RPC *itself* answers with, not the private + /// formatter it is supposed to call. + /// + /// Every other device test in the three engines calls `reported_device` + /// directly, so reverting this handler's `Some(state) => reported_device` + /// wiring to the old hardcoded literal kept the whole suite green. This + /// drives the handler. + /// + /// The planted state carries no endpoint, so no HTTP probe is made and + /// `listed`/`ready` are false — which is the point: this engine reports + /// the device as soon as it has state, independently of readiness (unlike + /// `engines/lemonade`, see `apps/rocm/src/engine_ipc.rs`'s + /// `device_worth_reporting`). + #[test] + fn healthcheck_rpc_answers_with_the_device_the_state_file_describes() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("svc-rpc.json"); + write_state( + &path, + &json!({ + "status": "running", + "gpu_indices": [3], + "gpu_name": "AMD Instinct MI300X", + }), + ) + .unwrap(); + + let response = healthcheck_service_at(&path).unwrap(); + assert_eq!(response.device, "rocm gpu 3 (AMD Instinct MI300X)"); + assert_eq!(response.status, "running"); + assert!(!response.model_loaded); + + // And a service with no state on disk is the `"unknown"` the CLI + // filters out rather than prints. + let response = healthcheck_service_at(&dir.path().join("absent.json")).unwrap(); + assert_eq!(response.device, "unknown"); + } + + /// The post-spawn whole-file rewrite must not drop the Anthropic endpoint + /// a sibling thread of this same process published moments earlier. + /// + /// `rocm`'s `run_builtin_engine_serve_http` runs `rocmd`'s Anthropic proxy + /// as a thread of the process running this adapter, and ROCMAI-550 put a + /// host GPU probe (seconds, on Windows) inside the rewrite — wide enough + /// for the publish to land inside it. Routing [`write_state`] through + /// `rocm_core::write_json_state_file` is what carries it through; putting + /// the raw `fs::write` back drops the key and fails this. + #[test] + fn a_running_state_write_keeps_the_anthropic_endpoint_this_process_published() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("svc-anthropic.json"); + rocm_core::publish_anthropic_endpoint( + &path, + "http://127.0.0.1:9932", + rocm_core::AnthropicEndpointMode::Proxy, + ) + .unwrap(); + + write_state(&path, &json!({ "status": "running", "gpu_indices": [0] })).unwrap(); + + let state = read_service_state(&path).unwrap(); + assert_eq!( + value_string(&state, rocm_core::ANTHROPIC_ENDPOINT_URL_STATE_KEY).as_deref(), + Some("http://127.0.0.1:9932"), + "the whole-file rewrite dropped the endpoint the proxy thread \ + published: {state}" + ); + } } From fbf0ad33d40feb72d20b13c877e5c05f21945b36 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 17 Nov 2026 17:04:18 +0200 Subject: [PATCH 50/59] test(core): reach the two branches the GPU classifier's own suite could not MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `gpu_support_level`'s `(_, Some(_)) => RocmHip` arm ignores the detected target on purpose — `resolve_family` asks it about a `--family` the user named, where there may be no detected target — but every call in the suite passed Some/Some, None/None or Some/None, so narrowing the arm to `(Some(_), Some(_))` was invisible. `gpu_name_for_single_gpu_host`'s ordinal guard was in the same position: the only test with a non-zero ordinal also supplied a host count of 2, which the check two lines below already declines on, so the guard could be deleted outright with the suite still green. Both mutations now turn it red. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/gfx_target.rs | 18 ++++++++++++++++++ crates/rocm-core/src/host_gpu.rs | 23 ++++++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/crates/rocm-core/src/gfx_target.rs b/crates/rocm-core/src/gfx_target.rs index ecbdfb4b0a..a73e36e97d 100644 --- a/crates/rocm-core/src/gfx_target.rs +++ b/crates/rocm-core/src/gfx_target.rs @@ -1066,12 +1066,30 @@ mod tests { ); } + /// A covering TheRock family is the whole answer, which is why that arm + /// ignores `detected_gfx_target` — and `resolve_family` depends on it: + /// it asks this about a target named with `--family`, where there may be + /// no detected target at all. Until the `None`/`Some(_)` case below, every + /// call in this suite passed `Some`/`Some`, `None`/`None` or `Some`/`None`, + /// so the arm could be narrowed to `(Some(_), Some(_))` with nothing + /// noticing. #[test] fn gpu_support_level_is_rocm_hip_when_a_family_covers_it() { assert_eq!( gpu_support_level(Some("gfx90a"), Some("gfx90a")), GpuSupportLevel::RocmHip ); + assert_eq!( + gpu_support_level(None, Some("gfx90a")), + GpuSupportLevel::RocmHip + ); + // Even for a target this file's own table says has no ROCm build: the + // family the caller resolved is a statement about ROCm support, and a + // detected target cannot outvote it. + assert_eq!( + gpu_support_level(Some("gfx90c"), Some("gfx90a")), + GpuSupportLevel::RocmHip + ); } #[test] diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index 38ac668e56..b287351c06 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -513,7 +513,7 @@ pub fn pinned_gpu_name(gpu_indices: &[u32]) -> Option { // uses it: off Windows nothing is probed here at all, and on Windows the // count already needed it before any of the decisions below could run. // - // The cost, stated rather than implied (raised in review on #678): this + // The cost, stated rather than implied: this // deduplicates *within* a call and is **not** cached across calls. Each // engine's `write_running_state` calls it, and that runs more than once per // launch (`starting`, then `running`), so a Windows launch pays that many @@ -986,6 +986,27 @@ mod tests { ); } + /// The ordinal guard on its own, with the host count agreeing there is one + /// GPU — the disagreement its doc comment names (a `--gpu 1` that got past + /// validation on a host now reporting a single card). + /// + /// Every other case here supplies a count that already declines, so + /// deleting the ordinal branch left the whole suite green. This is the + /// only call that reaches it. + #[test] + fn a_pinned_ordinal_the_host_count_contradicts_withholds_the_name() { + let mut probed = false; + let name = gpu_name_for_single_gpu_host(&[1], Some(1), false, || { + probed = true; + Some("AMD Instinct MI300X".to_owned()) + }); + assert_eq!(name, None); + assert!( + !probed, + "an ordinal the host's own count contradicts must not be probed" + ); + } + #[test] fn a_single_unmasked_gpu_is_named() { for indices in [&[][..], &[0][..]] { From b7e77e811fdd7c3ce754f9616717cfa4a1249a1a Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 17 Nov 2026 17:04:22 +0200 Subject: [PATCH 51/59] fix(examine): cross-check the no-GPU-backend install advice against driver_status The `NoGpuBackend` arm emitted "install one (Mesa RADV on Linux, the AMD display driver on Windows)" unconditionally, while its sibling `VulkanUnconfirmed` arm already consulted `driver.status` first -- because on Windows the AMD display driver *is* the Vulkan ICD, so its presence contradicts that advice. The same report printed `driver_status: amd_display_driver_detected` a few lines above the sentence telling the reader to go and install it. The check is now one predicate both arms call, which is the shape that would have kept the second arm from shipping without it. The confirmed absence is unchanged -- the probe did look and found no Vulkan GPU device -- only the step offered differs, and `vulkaninfo` is not offered there the way it is under `VulkanUnconfirmed`, since reaching `Absent` already takes its answer. `examine_render_does_not_promise_vulkan_on_a_host_that_has_none` asserted "install one" against the shared Windows fixture, which defaults to exactly the contradictory driver status; it now plants the host it is actually about. Signed-off-by: Roman Sirokov Signed-off-by: Roman Sirokov --- crates/rocm-core/src/host_gpu/render.rs | 179 ++++++++++++++++++++++-- 1 file changed, 167 insertions(+), 12 deletions(-) diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index 50d05465d9..929e2e77f6 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -40,22 +40,69 @@ impl ExamineSummary { ) } - /// What a [`GpuSupportLevel::VulkanUnconfirmed`] host is told to do next. + /// Whether this summary's own evidence already contradicts "install a + /// Vulkan driver". /// /// `DriverSummary::status` only ever reads `AMD_DISPLAY_DRIVER_DETECTED` /// on Windows — `windows_driver_summary` is the sole producer of that word /// — so matching on it *is* the Windows cross-check, with no second read /// of `os` to keep in step with it. On that platform the display driver - /// carries the Vulkan ICD, so its presence contradicts "install a Vulkan - /// driver" and only `vulkaninfo` is left to settle the question. + /// carries the Vulkan ICD, so its presence contradicts that advice: the + /// report would print `driver_status: amd_display_driver_detected` a few + /// lines above a sentence telling the reader to go and install what it + /// names. + /// + /// The cross-check is Windows-only for that reason, and not by accident: + /// Linux's `driver_status` reports the `amdgpu` kernel module, which says + /// nothing about whether a userspace ICD such as Mesa RADV is installed, + /// so suppressing the install advice there would withhold the one step + /// that can help. + /// + /// One predicate rather than a test spelled out in each arm that gives + /// install advice. Two arms now need it — `VulkanUnconfirmed` and + /// `NoGpuBackend` — and the second one shipped without it precisely + /// because the first one's check lived inside that first one's `match` + /// arm. Each arm's own test pins its half: + /// `examine_render_does_not_tell_a_windows_host_to_install_the_driver_it_has` + /// and + /// `examine_render_does_not_tell_a_no_backend_windows_host_to_install_the_driver_it_has`. + fn install_advice_is_contradicted_by_the_driver(&self) -> bool { + self.driver.status == super::AMD_DISPLAY_DRIVER_DETECTED + } + + /// What a [`GpuSupportLevel::VulkanUnconfirmed`] host is told to do next. fn vulkan_unconfirmed_next_step(&self) -> &'static str { - if self.driver.status == super::AMD_DISPLAY_DRIVER_DETECTED { + if self.install_advice_is_contradicted_by_the_driver() { crate::gfx_target::VULKAN_UNCONFIRMED_NEXT_STEP_WITH_DISPLAY_DRIVER } else { crate::gfx_target::VULKAN_UNCONFIRMED_NEXT_STEP } } + /// What a [`GpuSupportLevel::NoGpuBackend`] host is told to do next. + /// + /// The unconditional branch is the whole story on a host with nothing + /// installed: both halves of the level are established facts, there is + /// nothing left to go and find out, and installing a driver is the one + /// thing that can clear it. + /// + /// The other branch is a Windows host that already has the driver that + /// sentence names. Its probe answered — `VulkanStatus::Absent` is never + /// reached without `vulkaninfo`'s answer behind it — so "run `vulkaninfo`" + /// is not left either, the way it is under `VulkanUnconfirmed`. What is + /// left is the display driver itself: it is installed and is exposing no + /// Vulkan GPU device here, which a repair or update of that driver can + /// change and a fresh install cannot. + fn no_gpu_backend_next_step(&self) -> &'static str { + if self.install_advice_is_contradicted_by_the_driver() { + "the AMD display driver is installed (see `driver_status`) but is exposing no Vulkan \ + GPU device here, so repairing or updating it is the step that could open one" + } else { + "install one (Mesa RADV on Linux, the AMD display driver on Windows) to run models on \ + this GPU" + } + } + /// The `rocm_support` line of the examine text. /// /// ROCMAI-550 asks `rocm examine` to "report the support level with one @@ -152,15 +199,24 @@ impl ExamineSummary { // Both halves are established facts: the catalog confirmed there is // no ROCm build, and the probe looked for a Vulkan driver and found // none. Unlike `VulkanUnconfirmed` above, there is nothing left to - // go and find out on this host, so "install a driver" is the whole - // story. Only `Absent` reaches this level — `gpu_support_level` - // routes the probed-but-inconclusive host to `VulkanUnconfirmed` - // precisely so that this sentence is never said to one. + // go and find out on this host. Only `Absent` reaches this level — + // `gpu_support_level` routes the probed-but-inconclusive host to + // `VulkanUnconfirmed` precisely so that this sentence is never said + // to one. + // + // The next step is cross-checked against `driver.status` for the + // same reason `VulkanUnconfirmed`'s is, and through the same + // predicate: a Windows host whose report already says + // `driver_status: amd_display_driver_detected` must not be told, a + // few lines below, to install the driver it is showing. The + // confirmed absence itself is unaffected — the probe did look and + // did find no Vulkan GPU device — only what the host can do about + // it differs. GpuSupportLevel::NoGpuBackend => format!( " rocm_support: GPU detected ({}) but not supported by ROCm, and no Vulkan GPU \ - driver was found - install one (Mesa RADV on Linux, the AMD display driver on \ - Windows) to run models on this GPU\n", - detected.unwrap_or_default() + driver was found - {}\n", + detected.unwrap_or_default(), + self.no_gpu_backend_next_step(), ), // The covering family is the fact; it is already printed on its own // line above, so this one says what it means rather than repeating @@ -603,11 +659,25 @@ mod tests { /// nothing for it to fall back to. #[test] fn examine_render_does_not_promise_vulkan_on_a_host_that_has_none() { - let summary = gfx90c_summary_with_vulkan(Some(VulkanSupport { + let mut summary = gfx90c_summary_with_vulkan(Some(VulkanSupport { status: VulkanStatus::Absent, detail: Some("no Vulkan driver found".to_owned()), ..VulkanSupport::default() })); + // The shared fixture is a Windows host carrying + // `AMD_DISPLAY_DRIVER_DETECTED`, which is the one state where "install + // one" is the wrong advice (see + // `examine_render_does_not_tell_a_no_backend_windows_host_to_install_the_driver_it_has`). + // This test is about the host that genuinely has nothing, so it plants + // that host rather than asserting the install sentence on a summary + // whose own `driver_status` line contradicts it. + summary.os = "linux".to_owned(); + summary.driver = DriverSummary { + policy: "amdgpu_dkms".to_owned(), + status: "amdgpu_module_loaded".to_owned(), + detail: None, + windows: None, + }; let rendered = summary.render_text(); @@ -799,6 +869,91 @@ mod tests { ); } + /// The same misdirection one level down, and the one the + /// `VulkanUnconfirmed` cross-check above did not stop: a Windows host whose + /// report prints `driver_status: amd_display_driver_detected` must not be + /// told, a few lines higher in the same output, to install the AMD display + /// driver. + /// + /// `NoGpuBackend` keeps the confirmed absence — the probe did look and did + /// find no Vulkan GPU device, and that is not in question here — but the + /// *advice* has to be one the host can act on, and "install the driver you + /// are already showing" is not. `vulkaninfo` is not offered as the next + /// step either, unlike under `VulkanUnconfirmed`: reaching `Absent` takes + /// `vulkaninfo`'s own answer, so that question is already settled. + /// + /// Both directions are asserted, against the two hosts that differ only in + /// `driver_status`, so an arm that dropped the cross-check again fails here + /// rather than passing on the Linux half alone. + #[test] + fn examine_render_does_not_tell_a_no_backend_windows_host_to_install_the_driver_it_has() { + let absent = || { + Some(VulkanSupport { + status: VulkanStatus::Absent, + detail: Some("no Vulkan driver found".to_owned()), + ..VulkanSupport::default() + }) + }; + // The shared fixture is already a Windows host carrying + // `AMD_DISPLAY_DRIVER_DETECTED` — the exact summary this arm shipped + // being rendered against, and the reason the omission was invisible. + let windows = gfx90c_summary_with_vulkan(absent()); + assert_eq!( + windows.driver.status, + super::super::AMD_DISPLAY_DRIVER_DETECTED, + "this test is only a discrimination while the fixture it reuses is the \ + display-driver host" + ); + let windows = windows.render_text(); + let support = windows + .lines() + .find(|line| line.contains("rocm_support:")) + .expect("every host gets a support line"); + + assert!( + windows.contains("driver_status: amd_display_driver_detected"), + "the contradiction this guards against is between two lines of one report: {windows}" + ); + assert!( + !support.contains("install one"), + "this host already has the driver that sentence names: {support}" + ); + assert!( + support.contains("repairing or updating it"), + "and it is owed the step that can actually open a GPU path: {support}" + ); + assert!( + support.contains("no Vulkan GPU driver was found"), + "the confirmed absence itself is unchanged; only the advice is: {support}" + ); + assert!( + !support.contains("run `vulkaninfo`"), + "`Absent` already has `vulkaninfo`'s answer behind it, so there is nothing \ + left to ask it: {support}" + ); + + // The Linux shape of the same status: `driver_status` there reports the + // `amdgpu` kernel module, which says nothing about a userspace ICD, so + // the install advice is true and must survive. + let mut linux = gfx90c_summary_with_vulkan(absent()); + linux.os = "linux".to_owned(); + linux.driver = DriverSummary { + policy: "amdgpu_dkms".to_owned(), + status: "amdgpu_module_loaded".to_owned(), + detail: None, + windows: None, + }; + let linux = linux.render_text(); + assert!( + linux.contains("install one (Mesa RADV on Linux, the AMD display driver on Windows)"), + "a host with no display-driver evidence still gets the install step: {linux}" + ); + assert!( + !linux.contains("repairing or updating it"), + "and must not be told about a Windows driver it cannot have: {linux}" + ); + } + /// An unmapped target must not be *claimed* unsupported, even though every /// host now gets a `rocm_support` line. /// From 1540bda9f8116b081ce0d2962605ceeffa09d783 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 17 Nov 2026 17:05:02 +0200 Subject: [PATCH 52/59] test(e2e): say that the cross-surface device check cannot fire, and let it read both renderers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `assert_services_list_reports_device`'s comparison against `rocm serve`'s summary was dead twice over. `world.serve_summary` comes from `run_rocm_bounded`, which redirects stdout to a file, so `serve_cmd.rs`'s `summary_mode` is false and the CLI prints `print_managed_launch_plain`, which has no device row in any state; and `device_row_value` looked for `"device:"` while `render_summary` writes a padded label with no colon, so it could not have matched a summary even if one carried the row. The parser now reads both renderers' shapes (and still refuses the `device_policy:` row the serve plan prints), and the step says in full that on today's lanes the comparison does not run and the backend assertion above it is what discriminates — rather than letting the step's name imply coverage it has not got. Making it live means driving the vLLM and Lemonade per-PR serve canaries through a `TuiSession`, which is a change to those canaries' own harness. Signed-off-by: Roman Sirokov --- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 48 +++++++++++++++++-- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 8a2baa6950..b9c8e469ea 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -3682,11 +3682,29 @@ fn parse_device_row(value: &str) -> Option<(&str, Option)> { /// this step flake on timing rather than on behavior. const DEVICE_ROW_DEADLINE: Duration = Duration::from_secs(90); -/// The value of the first `device:` row in `text`, trimmed. +/// The value of the first row labelled `device` in `text`, trimmed. +/// +/// Two renderers, two shapes, and this has to read both: `services_text.rs` +/// writes `device: `, while `serve_summary.rs`'s `render_summary` +/// writes a column of padded labels and no colon at all +/// (` device ` — pinned by its own +/// `device_row_shown_only_when_known` test, which asserts +/// `starts_with("device")`). An earlier revision looked only for `"device:"`, +/// so it could never have matched a deployment summary even when one carried +/// the row. +/// +/// The label has to end where it ends: `rocm serve`'s non-interactive plan +/// prints a `device_policy:` row, and matching that here would compare a +/// policy against a device. fn device_row_value(text: &str) -> Option<&str> { - text.lines() - .find_map(|line| line.trim_start().strip_prefix("device:")) - .map(str::trim) + text.lines().find_map(|line| { + let rest = line.trim_start().strip_prefix("device")?; + let rest = rest.strip_prefix(':').unwrap_or(rest); + if !rest.starts_with(char::is_whitespace) { + return None; + } + Some(rest.trim()).filter(|value| !value.is_empty()) + }) } /// ROCMAI-550: `rocm services list` must show the backend/device the engine @@ -3706,7 +3724,11 @@ fn device_row_value(text: &str) -> Option<&str> { /// 3. it equals what the `rocm serve` deployment summary printed for the same /// service — *when* that summary printed a device row at all, which is a /// documented maybe rather than a promise (see the comment at that -/// comparison, and `serve_summary::DeploymentSummary::device`). +/// comparison, and `serve_summary::DeploymentSummary::device`). Facts 1 and +/// 2 are what discriminate on today's lanes; fact 3 does not run, because +/// the summary these scenarios capture is pipe-redirected and therefore the +/// row-less plain form. The comment at that comparison says so in full +/// rather than letting the name of this step imply coverage it has not got. /// The row is polled rather than read once: see [`DEVICE_ROW_DEADLINE`] for /// the window in which a correct implementation legitimately has no row yet. /// Running out of that window is an assertion naming what was last seen, not a @@ -3795,6 +3817,22 @@ async fn assert_services_list_reports_device(world: &mut E2eWorld) { // that scenario in model_serving.feature). What remains asserted is the // half that is a real invariant: the two surfaces read one engine over one // RPC, so where both speak they must say the same thing. + // + // **State plainly what this cannot do today.** On every current lane the + // comparison below does not run at all, and not for the reason above: + // `world.serve_summary` is `rocm serve`'s stdout captured by + // `run_rocm_bounded`, which redirects it to a file, so + // `serve_cmd.rs`'s `summary_mode` (`background && stdout().is_terminal()`) + // is false and the CLI takes `print_managed_launch_plain`, which renders no + // device row in any state. The device row is an interactive-summary + // feature. So this is a cross-check that is *armed* — `device_row_value` + // reads both renderers' shapes, and it fires for any Given that captures a + // summary through a PTY — but today nothing drives these two scenarios + // that way, and the backend assertion above is what actually discriminates + // for serve-05 and serve-07. Making it live means driving the vLLM and + // Lemonade per-PR serve canaries through a `TuiSession` (as serve-28/41/60 + // do for their mock launches), which is a change to the canaries' own + // harness and is tracked separately rather than smuggled in here. // And only when a summary was recorded at all. The backend assertion above // is this step's real invariant and has already run; the cross-surface // comparison is an extra that needs a Given which captured `rocm serve`'s From 16ccdf90f44377c2b349fbe7afc13c6caf0bdd8b Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 17 Nov 2026 17:05:02 +0200 Subject: [PATCH 53/59] test(engines): pin the VulkanOnly banner apart from the three no-backend levels `gpu_execution_line` tells a `VulkanOnly` host which backend it has and tells the other three that the CPU is not an alternative. Neither sentence appeared anywhere outside that function, so merging the `VulkanOnly` arm into the shared one below it left every test green -- including the ROCm-parity sweep, which only reads "ROCm GPU execution is required" and is satisfied by either. Asserted as the discrimination rather than as either sentence: each level requires its own banner present and the other's absent, and the three no-usable-backend levels are swept rather than sampled. Signed-off-by: Roman Sirokov Signed-off-by: Roman Sirokov --- apps/rocm/src/examine_report_text.rs | 73 ++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/apps/rocm/src/examine_report_text.rs b/apps/rocm/src/examine_report_text.rs index d1b84ac73e..a4397e6dca 100644 --- a/apps/rocm/src/examine_report_text.rs +++ b/apps/rocm/src/examine_report_text.rs @@ -1681,4 +1681,77 @@ mod tests { let _ = fs::remove_dir_all(root); } + + /// The other half of the banner, and the half nothing asserted: + /// `gpu_execution_line` tells a `VulkanOnly` host which backend it has, + /// and tells the other three that the CPU is not an alternative. Neither + /// sentence appeared anywhere outside that function, so merging the + /// `VulkanOnly` arm into the shared one below it left every test green — + /// including the ROCm-parity sweep above, which only reads the word "ROCm + /// GPU execution is required" and is satisfied by either sentence. + /// + /// The discrimination is the finding, not either sentence on its own: + /// `VulkanOnly` is the single level where the classifier established a GPU + /// backend, and "CPU execution is not supported" there is true but silent + /// about the backend this host actually has. So each level asserts its own + /// banner present *and* the other's absent, and the three no-usable-backend + /// levels are swept rather than sampled, so a future arm cannot quietly + /// pick up the Vulkan claim. + #[test] + fn engines_banner_claims_a_vulkan_backend_only_where_one_was_established() { + let (root, paths) = test_paths("engines-banner-vulkan-only"); + let banner = |level| { + render_engine_inventory_text_for_host(Some(&paths), "lemonade", level) + .lines() + .find(|line| line.contains("GPU execution is required")) + .expect("every level prints the GPU-execution banner") + .to_owned() + }; + + let vulkan_only = banner(rocm_core::GpuSupportLevel::VulkanOnly); + assert!( + vulkan_only.contains("that is the Vulkan backend, not ROCm"), + "the one level with an established GPU backend must name it:\n{vulkan_only}" + ); + assert!( + !vulkan_only.contains("CPU execution is not supported"), + "and must not borrow the sentence of the levels that have none:\n{vulkan_only}" + ); + + for level in [ + rocm_core::GpuSupportLevel::VulkanUnconfirmed, + rocm_core::GpuSupportLevel::NoGpuBackend, + rocm_core::GpuSupportLevel::Unknown, + ] { + let line = banner(level); + assert!( + !line.contains("that is the Vulkan backend, not ROCm"), + "{level:?} established no Vulkan backend and must not claim one:\n{line}" + ); + assert!( + line.contains("CPU execution is not supported"), + "but the refusal is what is invariant across them, so it is still \ + said:\n{line}" + ); + assert_ne!( + line, vulkan_only, + "{level:?} must not render the `VulkanOnly` banner" + ); + } + + // `RocmHip` is the fourth answer, and the only one that may still + // require ROCm. Without it the sweep above would pass on a banner that + // had lost that arm too. + let rocm_hip = banner(rocm_core::GpuSupportLevel::RocmHip); + assert!( + rocm_hip.contains("ROCm GPU execution is required"), + "a ROCm-capable host keeps the original claim:\n{rocm_hip}" + ); + assert_ne!( + rocm_hip, vulkan_only, + "and it is not the Vulkan sentence either" + ); + + let _ = fs::remove_dir_all(root); + } } From 8b811d6b97ee324e00d1aa3eaf1582a753510b00 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 17 Nov 2026 17:05:24 +0200 Subject: [PATCH 54/59] docs: correct the device-row gating, the gfx_target entry and the Lemonade serve flow `device_worth_reporting` claimed the `"unknown"`-until-ready gate was uniform and cited vLLM and Lemonade as sharing it. They do not: vLLM and AIM answer as soon as a state file exists, while Lemonade keeps saying `"unknown"` until its own inference probe returns. That difference is the whole reason `SERVE_DEVICE_PROBE_BUDGET` exists, so the comment now names it. The probe bound alongside it also cites AIM's own `HEALTHCHECK_TIMEOUT_MS`, which it relied on without naming. `docs/architecture.md`'s `gfx_target.rs` entry still described pure lookups after that file gained the whole `GpuSupportLevel` classifier; the gate checks presence, not accuracy, so nothing caught it. `docs/manual-testing.md` dropped `--foreground` from the Lemonade serve example but still called the result "that foreground server" 34 lines later, with no way to stop it. The prose now matches a backgrounded service and gives the stop command, as the parallel flow below it already does. Also drops review-thread references from source comments and a note to reviewers from docs/testing.md; each says the reason instead. Signed-off-by: Roman Sirokov --- apps/rocm/src/cli.rs | 6 +++--- apps/rocm/src/engine_ipc.rs | 31 ++++++++++++++++++++++--------- apps/rocm/src/therock/family.rs | 2 +- docs/architecture.md | 2 +- docs/manual-testing.md | 13 ++++++++++--- docs/testing.md | 9 ++++----- 6 files changed, 41 insertions(+), 22 deletions(-) diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index c67be7ff3a..45ace9b237 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -2732,9 +2732,9 @@ mod tests { /// Sibling of `every_help_text_in_the_command_tree_is_ascii`, for the same /// reason one level down: clap renders a doc comment verbatim, with no /// markdown processing anywhere in its pipeline, so `**every**` written in - /// a doc comment reaches the user's terminal as literal asterisks. Review - /// on #678 caught exactly that on `Examine`, which the ASCII test could - /// not see because `*` and `_` are ASCII. + /// a doc comment reaches the user's terminal as literal asterisks. That had + /// already happened on `Examine`, where the ASCII test could not see it + /// because `*` and `_` are ASCII. /// /// Only `**` and `__` are rejected, and only those: they have no honest /// use in plain help text. A single `*` does (`gfx110*`, a glob in an diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index 4954c4e04d..8a7858b480 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -329,8 +329,8 @@ pub(crate) fn run_builtin_engine_stdio(engine: &str) -> Result<()> { /// How long one call can take, since `rocm services list` makes one per ready /// service: [`engine_request`] itself imposes no deadline, so the bound comes /// from the engine. The three built-in engines answer in-process and every -/// endpoint probe they make is timeout-bounded (`engines/vllm`'s -/// `HEALTHCHECK_TIMEOUT_MS`, `engines/lemonade`'s +/// endpoint probe they make is timeout-bounded (`engines/vllm`'s and +/// `engines/aim`'s own `HEALTHCHECK_TIMEOUT_MS`, `engines/lemonade`'s /// `rocm_core::INFERENCE_PROBE_TIMEOUT`), so a stalled server costs seconds, /// not forever. An *external* plugin is a spawned child that /// `engine_request` waits on unbounded, exactly as every other engine call @@ -365,11 +365,24 @@ pub(crate) fn fetch_service_device( /// The engine's reported device, or `None` when it said nothing usable. /// -/// `"unknown"` is the engines' own device value when no state is on disk for -/// this service id yet (see `engines/vllm/src/state.rs` and -/// `engines/lemonade/src/lib.rs`), and it is not more informative than showing -/// nothing — a `device: unknown` row claims the CLI asked and learned -/// something, which is the opposite of what happened. +/// `"unknown"` is the engines' own "I cannot say yet", and it is not more +/// informative than showing nothing — a `device: unknown` row claims the CLI +/// asked and learned something, which is the opposite of what happened. +/// +/// *When* an engine stops saying it is not uniform, and that difference is why +/// [`SERVE_DEVICE_PROBE_BUDGET`] exists at all: +/// +/// * `engines/vllm` and `engines/aim` answer as soon as a state file exists +/// for the service id (`Some(state) => reported_device(state)`, +/// `None => "unknown"`), so the row is available from the launch write +/// onwards, readiness or not; +/// * `engines/lemonade` keeps answering `"unknown"` while a state file plainly +/// exists — its `healthcheck_service_at` reports a device only once the +/// service is `ready`, which there means its own inference probe has come +/// back. +/// +/// So this filter is what keeps the slowest of the three from printing a row +/// that says nothing; it is not a uniform no-state-on-disk guard. /// /// A named function rather than an inline `then_some` because the only caller /// is an RPC to a live engine: in default CI that call never succeeds, so the @@ -481,10 +494,10 @@ where /// `device:` row, so a bridge client calling the tool makes no probe and waits /// for nothing — unlike the sibling `engines` MCP tool, which ROCMAI-550 did /// make host-aware. The shared name is the trap, so this says which surface it -/// means (review on #678); giving the tool the row is a change in its own +/// means; giving the tool the row is a change in its own /// right, not something this constant may be read as already covering. /// -/// # The decision, recorded (review on #678) +/// # The decision, recorded /// /// This is accepted rather than merely observed. `rocm services list` and /// `rocm chat`'s `services` command can both block ~18s on a host whose diff --git a/apps/rocm/src/therock/family.rs b/apps/rocm/src/therock/family.rs index 99f09f44dc..aa7c3e250a 100644 --- a/apps/rocm/src/therock/family.rs +++ b/apps/rocm/src/therock/family.rs @@ -178,7 +178,7 @@ pub(super) fn resolve_family( /// /// Asked of [`gpu_support_level`] rather than re-tested here. The install path /// used to call `gfx_target_has_no_rocm_build` directly, which was *exactly* -/// equivalent (review on #678 checked that it was) and still wrong to keep: it +/// equivalent, and still wrong to keep: it /// made this a second, independent definition of the tier whose claim these /// messages then go on to make, so a change to the classification would have /// left `rocm examine` and `rocm install sdk` disagreeing about the same GPU diff --git a/docs/architecture.md b/docs/architecture.md index 1d2ac90756..36cbd50215 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -310,7 +310,7 @@ The library every first-party app and engine crate depends on. It holds host exa | [`fix/render.rs`](../crates/rocm-core/src/fix/render.rs) | Rendering the catalog for people: the listing, a recipe's plan and verify lines, the flag summary `rocm diagnose` shares, and the refusal of a diagnosis position as a fix-id. | flat | | [`fix/runners.rs`](../crates/rocm-core/src/fix/runners.rs) | The host-mutating runners, one per auto-applicable fix, each printing the exact change, honouring `--dry-run` and asking consent first. | flat | | [`foreign_temp_probe.rs`](../crates/rocm-core/src/foreign_temp_probe.rs) | Test probe that a writer publishing through `atomic_write` never touches another writer's temp file; built only for tests and the `test-support` feature. | flat | -| [`gfx_target.rs`](../crates/rocm-core/src/gfx_target.rs) | Pure gfx-target lookups: extracting a gfx token, mapping a target to its TheRock family, the AMD PCI-id and marketing-name tables, and kernel gfx-version decoding. | domain | +| [`gfx_target.rs`](../crates/rocm-core/src/gfx_target.rs) | Pure gfx-target lookups: extracting a gfx token, mapping a target to its TheRock family, the AMD PCI-id and marketing-name tables, and kernel gfx-version decoding. Also the `GpuSupportLevel` classifier (`gpu_support_level`, `gpu_support_level_for_host_gpu`, `VULKAN_ONLY_SUPPORT_CLAIM`) that turns those two facts into the one ROCm/Vulkan support claim `examine`, `install` and the engine notes all render, so no surface invents its own. | domain | | [`gguf.rs`](../crates/rocm-core/src/gguf.rs) | Reads a GGUF model's shape (layers, heads, head width, parameter count, data offset) from its header, never the weights. | domain | | [`gguf_fit.rs`](../crates/rocm-core/src/gguf_fit.rs) | Pure arithmetic on whether a GGUF model fits in GPU memory at a context size: weights, KV cache, compute buffer and reserve against VRAM. | domain | | [`gguf_source.rs`](../crates/rocm-core/src/gguf_source.rs) | Where a GGUF header comes from: a local file, or a Hugging Face file read by HTTP range requests. | domain | diff --git a/docs/manual-testing.md b/docs/manual-testing.md index fba91c457a..bc0d1b50d8 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -439,9 +439,16 @@ in `MSVCP140.dll`), the Visual C++ runtime on the machine is probably older than fix-21-msvc-redist-outdated` prints the update plan and changes nothing. After the update, restart Lemonade and repeat the serve command above. -With that foreground server stopped, check that a catalogued recipe reaches -Lemonade with a build it can load, from a short name alone — no `--engine`, no -quantization variant typed by hand: +That server is in the background — `rocm serve` returns to the shell once it is +ready — so stop it before continuing, or it keeps the GPU and port 11435: + +```powershell +rocm services stop --yes +``` + +With it stopped, check that a catalogued recipe reaches Lemonade with a build +it can load, from a short name alone — no `--engine`, no quantization variant +typed by hand: ```powershell rocm serve qwen-small --managed diff --git a/docs/testing.md b/docs/testing.md index 6101476fac..0a83fefc46 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -4458,11 +4458,10 @@ same report prints, so every lane checks the level its own hardware implies: cargo xtask e2e -- -n '(examine-55)' ``` -Vulkan-only here is inferred from the gfx table, not probed: this change has -three levels and this section describes three. Probing the host for a Vulkan -driver, and the further levels that evidence makes knowable, is separate work. -Written this way deliberately — a forward reference to an unmerged change reads -as false the moment this merges on its own. +Vulkan-only here is inferred from the gfx table, not probed: there are three +support levels and this section describes three. Probing the host for a Vulkan +driver, and the further levels that evidence would make knowable, is separate +work. ## Windows Tool Notes From 0cce209903a591a161e190f1aeee5ff35d3f932a Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 17 Nov 2026 17:05:37 +0200 Subject: [PATCH 55/59] docs(vulkan): describe the evidence model the probe actually has `docs/host-probing.md` said Windows devices come from "the same registry read" -- they come from `vulkaninfo` there as on Linux, the registry is the ICD source only -- and both it and the `vulkan.rs` architecture row still described an `enumerated` flag that `vulkan_devices_from_capture` no longer returns; it returns a `VulkanEvidence` whose `did_answer()` carries that question. Both now state the invariant as implemented: `vulkaninfo` must have answered before anything may be reported as a confirmed absence, and a GPU manifest outranks a software-only enumeration. README's Vulkan-unconfirmed row named `vulkaninfo` as the source that could not be asked; `Inconclusive` is reachable when `vulkaninfo` answered and the ICD/registry listing is what failed, which is realistic on Windows. Generalized to either source, as the `VulkanUnconfirmed` doc comment now is too. Also splits the two comments that had merged above the inner `match` in `gpu_support_level`. Signed-off-by: Roman Sirokov Signed-off-by: Roman Sirokov --- README.md | 4 ++-- crates/rocm-core/src/gfx_target.rs | 7 +++++-- docs/architecture.md | 2 +- docs/host-probing.md | 8 ++++++-- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index fe40c14db8..4f0a927172 100644 --- a/README.md +++ b/README.md @@ -666,8 +666,8 @@ case, at one of five levels: | --- | --- | | ROCm/HIP-supported | A TheRock package family covers this GPU. | | Vulkan-only | No ROCm build, **and** either a Vulkan GPU driver was found here, or the platform cannot be probed for one at all (anything other than Linux and Windows, or a summary from an older `rocm remote doctor` that predates the probe). The `rocm_support` line says which of the two it is, and only names a driver when one was actually found. | -| Vulkan-unconfirmed | No ROCm build, and the probe could not reach an answer — no ICD, no enumerated device, and no `vulkaninfo` to ask. | -| no GPU backend | No ROCm build, **and** the probe looked and found no Vulkan GPU driver. | +| Vulkan-unconfirmed | No ROCm build, and the probe could not reach an answer: nothing was found, and at least one of the two sources — the ICD/registry listing or `vulkaninfo` — never answered. Either one can be the source that went unread (on Windows the registry read is a separate PowerShell call that can fail on its own), and the `vulkan:` line names which. | +| no GPU backend | No ROCm build, **and** the probe looked and found no Vulkan GPU driver. This takes `vulkaninfo`'s own answer: the ICD listing is not exhaustive on either platform, so it never denies a driver by itself. | | unknown | No GPU detected, or a target this build's catalog does not recognize. | The last row is a detection gap rather than a verdict about the hardware, which diff --git a/crates/rocm-core/src/gfx_target.rs b/crates/rocm-core/src/gfx_target.rs index e6e6a13767..8f3d4cab95 100644 --- a/crates/rocm-core/src/gfx_target.rs +++ b/crates/rocm-core/src/gfx_target.rs @@ -586,8 +586,10 @@ pub enum GpuSupportLevel { /// and serve a model on it. VulkanOnly, /// Recognized, confirmed to have no ROCm/HIP build, and the Vulkan probe - /// could not reach an answer — no ICD manifest, no enumerated device, and - /// no `vulkaninfo` to ask. + /// could not reach an answer — nothing was found, and at least one of its + /// two sources never answered. Either one can be the source that went + /// unread: `vulkaninfo` missing or failing, or (on Windows, where the + /// registry read is its own PowerShell call) the driver listing. /// /// Deliberately NOT folded into [`Self::NoGpuBackend`]. That level renders /// as a confirmed fact and is serialized for JSON consumers, and this state @@ -734,6 +736,7 @@ pub fn gpu_support_level( // is the common no-driver shape, since most such hosts have no // `vulkaninfo` either. The wording it gets is hedged rather than a // confirmed absence — see `render.rs`. + // // Every variant is spelled out, with no `_` and no guard. A guard // reads well but does not count toward exhaustiveness, so the arms // would have to end in a wildcard — and a wildcard here silently diff --git a/docs/architecture.md b/docs/architecture.md index ec4574421f..627cbfd6d3 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -323,7 +323,7 @@ The library every first-party app and engine crate depends on. It holds host exa | [`host_gpu/linux.rs`](../crates/rocm-core/src/host_gpu/linux.rs) | Linux sysfs, KFD and DRM probes: the KFD topology, the DRM ip-discovery gfx target, the amdgpu device and driver version, the primary GPU name. | domain | | [`host_gpu/os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) | Kernel version, distro name, installed RAM, and the CPU model and vendor for the host summary. One reader returns the CPU model and vendor together for both the summary and `Examination::probe`: Linux from `/proc/cpuinfo`, Windows from the CPU row of the system inventory. `detect_kernel_version` is now the only `uname -r` reader; [`examine/host_env.rs`](../crates/rocm-core/src/examine/host_env.rs)'s `probe_os` calls it for the examination's `kernel_release`. | domain | | [`host_gpu/render.rs`](../crates/rocm-core/src/host_gpu/render.rs) | The `rocm examine` text rendering of an `ExamineSummary`. | domain | -| [`host_gpu/vulkan.rs`](../crates/rocm-core/src/host_gpu/vulkan.rs) | Whether a model can run through Vulkan on this host: ICD manifests, the Windows Khronos registry and `vulkaninfo --summary`, classified into `VulkanStatus`. `Absent` is reserved for a probe that was able to look and ruled a GPU path out: a software-only ICD, an enumeration of only software devices, or a `vulkaninfo` that ran and reported nothing (what `examine-57` plants). It takes *both* sources having answered — `icds_listed` for the driver listing, `enumerated` for `vulkaninfo` — so when either could not be read (a Windows `powershell` that is missing, fails or times out; a `vulkaninfo` that is absent or exits non-zero) and nothing was found, the answer is `Inconclusive`, which reaches the user as the `VulkanUnconfirmed` support level rather than a confirmed absence, with a detail naming which source went unread. Neither source is exhaustive, and claiming absence would tell a host with a working driver to install one. A software rasteriser never counts as a GPU path. An enumeration that returned only non-AMD hardware is reported as such rather than as a missing driver, but is still routed to the `VulkanOnly` level — a known limitation. | domain | +| [`host_gpu/vulkan.rs`](../crates/rocm-core/src/host_gpu/vulkan.rs) | Whether a model can run through Vulkan on this host: ICD manifests, the Windows Khronos registry and `vulkaninfo --summary`, classified into `VulkanStatus`. Each source arrives as a `VulkanEvidence` carrying whether it *answered* (`did_answer()`) apart from what it found, so an empty list can never be read as an answer by accident. `Absent` is reserved for a probe that was able to look and ruled a GPU path out: a `vulkaninfo` that ran and reported nothing (what `examine-57` plants), a software-only ICD list that `vulkaninfo` did not contradict, or an enumeration of only software devices with no GPU manifest beside it. `vulkaninfo` must have answered on every one of those paths, since neither the Linux directory list nor the Windows Khronos key is exhaustive enough to deny a driver alone; where nothing at all was found the driver listing must have answered too, because an empty list is also what a Windows `powershell` that is missing, fails or times out produces. Anything short of that is `Inconclusive`, which reaches the user as the `VulkanUnconfirmed` support level rather than a confirmed absence, with a detail naming which source went unread — claiming absence there would tell a host with a working driver to install one. A software rasteriser never counts as a GPU path, but neither does it deny one that an ICD manifest asserts: a software-only enumeration alongside `radeon_icd.x86_64.json`, the ordinary shape of a missing `render`-group membership, is `DriverPresentDevicesUnverified`. An enumeration that returned only non-AMD hardware is reported as such rather than as a missing driver, but is still routed to the `VulkanOnly` level — a known limitation. | domain | | [`host_gpu/windows_inventory.rs`](../crates/rocm-core/src/host_gpu/windows_inventory.rs) | The Windows display-adapter inventory: the PowerShell queries, their parsers, and assembling one inventory from several sources, including the system query whose CPU row the CPU reader in [`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) takes. | domain | | [`host_gpu/wsl.rs`](../crates/rocm-core/src/host_gpu/wsl.rs) | WSL detection and the Windows host driver as seen from the guest: the WSL2 signals, the librocdxg chain and the host-driver probes. | domain | | [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs) | The generic append-only JSONL store engine: sibling `.lock` file, single-line appends, torn-tail repair, size rotation and tail-first reads. | domain | diff --git a/docs/host-probing.md b/docs/host-probing.md index 9b1ef9f929..dce7b1b920 100644 --- a/docs/host-probing.md +++ b/docs/host-probing.md @@ -197,12 +197,16 @@ Verdict: not a duplicate — only the summary stack reads it — but it is a spa | | Linux | Windows | |---|---|---| | ICDs | the manifest directories, read from the filesystem | `powershell` over the Khronos `Drivers` registry keys | -| Devices | `vulkaninfo --summary` | the same registry read; `vulkaninfo` is rarely on PATH | +| Devices | `vulkaninfo --summary` | `vulkaninfo --summary` as well — there is no registry source for devices — but it is rarely on PATH, so this source usually does not answer at all | | Timeout | 1.5 s (`OPTIONAL_COMMAND_TIMEOUT`), per candidate spelling | the same 1.5 s, which a cold PowerShell start can exceed | | Failure | `None` from `capture_optional_command`, which does not distinguish "not installed" from "timed out" | the same | | Lookup ran? | always true: reading directories that may not exist is still a lookup | `false` whenever `powershell` is missing, exits non-zero or times out | -The last row is `icds_listed`, the ICD-side counterpart of the `enumerated` flag that `vulkan_devices_from_capture` returns for `vulkaninfo`. Both say whether that source *answered*, and `classify_vulkan_support` requires both before it will report an empty result as a confirmed absence. On a platform that is neither Linux nor Windows the flag is `false` — there is nothing to read there — though the classifier never consults it, since `vulkan_is_probeable()` returns `Unknown` first. +The two sources do not return a bare list. Each arrives as a `VulkanEvidence` — `VulkanEvidence` for the ICD manifests, `VulkanEvidence` for the enumerated devices — whose `did_answer()` is whether that source *answered* at all, as opposed to answering with nothing. The last row of the table is the ICD side of that; `vulkan_devices_from_capture` is the `vulkaninfo` side, mapping a `None` capture to `VulkanEvidence::unanswered()`. The two monomorphizations are distinct types, so passing one source's evidence where the other belongs is a compile error rather than a silent status flip. `classify_vulkan_support` reads them as the local bindings `icds_listed` and `enumerated`. + +What it does with them: `vulkaninfo` must have answered before any empty-or-software result may be reported as a confirmed `VulkanStatus::Absent`, because neither the Linux directory list nor the Windows Khronos key is exhaustive and so neither can deny a driver on its own. Where nothing at all was found, the ICD listing must have answered too — an empty `icds` is also what a PowerShell read that failed or timed out produces. Anything short of that is `Inconclusive`. On a platform that is neither Linux nor Windows both flags are `false` — there is nothing to read there — though the classifier never consults them, since `vulkan_is_probeable()` returns `Unknown` first. + +A driver the ICD listing *did* find is also evidence in the other direction: a host carrying `radeon_icd.x86_64.json` whose `vulkaninfo` enumerates only `llvmpipe` (the ordinary shape of a missing `render`-group membership) is `DriverPresentDevicesUnverified`, not `Absent`. Denying the driver there would print "no Vulkan GPU driver was found - install one" beside the `icds` field naming the driver that is already installed. Who pays it. The probe runs once per full gather, so every caller of `ExamineSummary::gather` pays one: the text and JSON of `rocm examine`, the MCP and bridge snapshots, the free-form answer, `rocm install driver` and its `--reconcile` form, and `rocmd`'s host snapshot, MCP and sandbox paths. It is deliberately *not* run by the lean scope, `gather_without_driver_facts`, which the runtime-install finalizer uses to read the GPU target and nothing else; that scope leaves `vulkan` as `None` and `vulkan_status` reports `Unknown`. `the_summary_without_driver_facts_does_not_probe_vulkan` pins that, so restoring an unconditional probe fails a test rather than quietly adding a process to the install path. From ab87a9033989b3c8cd5d03540a1a0d65c93e6edf Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 17 Nov 2026 19:54:58 +0200 Subject: [PATCH 56/59] docs(vulkan): say which ICD evidence actually reaches Inconclusive `VulkanStatus::Inconclusive`'s doc gave "no ICD manifest" as a precondition, and the classifier reaches that status with a non-empty `icds`: a host whose only manifest is a software rasteriser, with no `vulkaninfo` answer behind it, lands here rather than in `Absent`, because the ICD search is not exhaustive and so lavapipe-is-all-I-can-see does not rule out a GPU driver the probe never reads. The behaviour is right and already pinned by `a_lavapipe_only_install_is_not_a_confirmed_absence_until_vulkaninfo_answers`. Only the doc was wrong, so only the doc changes, and it now names that test so the next reader can check the claim instead of taking it. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/host_gpu/vulkan.rs | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/crates/rocm-core/src/host_gpu/vulkan.rs b/crates/rocm-core/src/host_gpu/vulkan.rs index 69c1f1dd6a..6eeb95f8e4 100644 --- a/crates/rocm-core/src/host_gpu/vulkan.rs +++ b/crates/rocm-core/src/host_gpu/vulkan.rs @@ -84,10 +84,23 @@ pub enum VulkanStatus { /// covering this status plants the exit-0 shape rather than reproducing a /// driverless host. Absent, - /// The probe came away with nothing either way: no ICD manifest, no - /// enumerated device, and at least one of the two sources that never - /// answered — `vulkaninfo` missing or failing, or (on Windows) a registry - /// listing that could not be run. + /// The probe came away with nothing either way: no ICD manifest *that + /// names a GPU driver*, no enumerated device, and at least one of the two + /// sources that never answered — `vulkaninfo` missing or failing, or (on + /// Windows) a registry listing that could not be run. + /// + /// `icds` is therefore not necessarily empty here, and the first clause is + /// written that way deliberately. A host whose only manifest is a software + /// rasteriser — `lvp_icd.x86_64.json` and nothing else — has found no GPU + /// driver, but it has not *ruled one out* either: the ICD search is not + /// exhaustive (a fixed directory list on Linux that ignores + /// `VK_ICD_FILENAMES`/`VK_DRIVER_FILES`, the legacy Khronos key alone on + /// Windows), so "lavapipe is the only manifest I can see" leaves a GPU + /// driver this probe never reads perfectly possible. Without `vulkaninfo`'s + /// answer behind it that host lands here rather than in [`Self::Absent`], + /// which is what + /// `a_lavapipe_only_install_is_not_a_confirmed_absence_until_vulkaninfo_answers` + /// holds. /// /// Distinct from [`Self::Unknown`], which is a platform that cannot be /// probed at all. The difference decides what the host is told: an From f3a33ca3cb51186576fa1c0fe35a3b3ed8789ef8 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 17 Nov 2026 19:55:35 +0200 Subject: [PATCH 57/59] test(vulkan): drive the Windows registry lookup from every lane The file and docs/host-probing.md both claimed the Khronos registry read was "covered by CI's windows-build-and-test job". Nothing calls `detect_icds` on Windows, so that job compiles the call and never runs it: the program name, the flags and the script handed to PowerShell had no test at all. Dropping `-Command`, or losing `-NonInteractive` so a prompting profile turns the probe into a 1.5s timeout, would have shipped and shown up only on real hardware as a permanently inconclusive Vulkan line. `windows_icds_through` takes the subprocess as an argument, so everything the Windows path does except starting the process now runs on every lane, argv included, through to `classify_vulkan_support`. What is left uncovered is PowerShell's own behaviour on a real host, and both docs now say exactly that instead of naming a job that does not test it. That residue is bounded: every such failure arrives as a `None`, which the seam routes to `Inconclusive` rather than to a claim about the machine. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/host_gpu/vulkan.rs | 138 ++++++++++++++++++++++-- docs/host-probing.md | 4 +- 2 files changed, 132 insertions(+), 10 deletions(-) diff --git a/crates/rocm-core/src/host_gpu/vulkan.rs b/crates/rocm-core/src/host_gpu/vulkan.rs index 6eeb95f8e4..08c40dd5d1 100644 --- a/crates/rocm-core/src/host_gpu/vulkan.rs +++ b/crates/rocm-core/src/host_gpu/vulkan.rs @@ -467,9 +467,36 @@ fn detect_icds() -> VulkanEvidence { /// otherwise a flaky PowerShell on a host with a working AMD driver reads /// identically to a host with no driver at all — the conflation the device /// side's "did it answer" exists to prevent. +/// +/// Everything this does beyond starting the process lives in +/// [`windows_icds_through`], which is compiled and tested on every platform. #[cfg(windows)] fn detect_icds() -> VulkanEvidence { - windows_icds_from_capture(capture_optional_command( + windows_icds_through(capture_optional_command) +} + +/// The Windows ICD lookup with the subprocess as an argument: which program is +/// run, with which arguments, and what its capture is turned into. +/// +/// Split out from [`detect_icds`] so that the whole of the Windows registry +/// path except the `CreateProcess` call itself can be driven from a Linux lane. +/// The argv is the part that had no test at all: `windows_icds_from_capture` +/// was tested over hand-written captures, but nothing checked that the capture +/// handed to it came from PowerShell running +/// [`WINDOWS_VULKAN_ICD_SCRIPT`] — so dropping `-Command`, or pointing the +/// lookup at a program that is not on a stock Windows install, would have +/// compiled and shipped, failing only on real hardware as a silent +/// [`VulkanStatus::Inconclusive`]. `the_windows_registry_lookup_runs_the_khronos_script_through_powershell` +/// pins it. +/// +/// `-NonInteractive` and `-NoProfile` are not decoration: a profile that +/// prompts turns this probe into a 1.5 s timeout on every `examine`, and a +/// timeout is one of the `None`s that must not read as an empty registry key. +#[cfg(any(windows, test))] +fn windows_icds_through( + capture: impl FnOnce(&str, &[&str]) -> Option, +) -> VulkanEvidence { + windows_icds_from_capture(capture( "powershell", &[ "-NoLogo", @@ -493,11 +520,23 @@ const fn detect_icds() -> VulkanEvidence { /// The pure half of the Windows [`detect_icds`]: the manifests the capture /// named, and whether PowerShell actually **answered**. /// -/// Compiled on every platform, not just Windows, so the Linux lanes can test the -/// failure path at all — the registry read itself has no gate but CI's -/// `windows-build-and-test` job. `Some` means the script exited 0 and its stdout -/// is in hand, so an empty list from it is a real "the key holds no ICD"; `None` +/// Compiled on every platform, not just Windows, so the Linux lanes can test +/// the failure path at all. `Some` means the script exited 0 and its stdout is +/// in hand, so an empty list from it is a real "the key holds no ICD"; `None` /// is the un-run lookup that must not be reported as one. +/// +/// **What is and is not covered**, stated exactly, because an overstated claim +/// here is what leaves a gap nobody goes looking for. This function and the +/// argv around it ([`windows_icds_through`]) are unit-tested on every platform. +/// The one step no test executes is starting the PowerShell process: no test in +/// this repository runs `detect_icds` on Windows, so CI's +/// `windows-build-and-test` job *compiles* that call and never runs it. What +/// could still break there and nowhere else is the behaviour of PowerShell +/// itself on a real host — the script's output encoding, a policy that blocks +/// it, a cold start slower than `OPTIONAL_COMMAND_TIMEOUT`. Each of those +/// arrives as a `None`, which this function routes to the hedged +/// [`VulkanStatus::Inconclusive`] rather than to a claim about the machine, so +/// the untested step cannot produce a false statement — only a less useful one. #[must_use] pub fn windows_icds_from_capture(captured: Option) -> VulkanEvidence { captured.map_or_else(VulkanEvidence::unanswered, |output| { @@ -1571,10 +1610,91 @@ GPU0: ); } - /// The Windows seam itself, at the only level a Linux lane can reach it: the - /// pure half of `detect_icds`. The registry read behind it is exercised - /// nowhere but CI's `windows-build-and-test`, so the failure and timeout - /// paths — all of which arrive here as `None` — are covered through this. + /// The Windows registry lookup, driven from a Linux lane through the one + /// seam `detect_icds` leaves: the subprocess as an argument. + /// + /// This is the half that had no test. `windows_icds_from_capture` was + /// exercised over captures written by hand, which says nothing about where + /// a real capture comes from — the program name, the flags, and the script + /// were only ever read by a human. Asserting the argv *here*, against the + /// same `WINDOWS_VULKAN_ICD_SCRIPT` the parser's own test reads, is what + /// ties the two halves together: an argv that no longer runs the script, + /// or runs it interactively, fails here rather than on a user's machine as + /// a permanently inconclusive Vulkan line. + /// + /// What it still does not reach is `CreateProcess` itself, and no test in + /// this repository does — see `windows_icds_from_capture` for exactly what + /// that leaves uncovered and why the residue is bounded. + #[test] + fn the_windows_registry_lookup_runs_the_khronos_script_through_powershell() { + let mut argv = None; + let icds = windows_icds_through(|program, args| { + argv = Some(( + program.to_owned(), + args.iter().map(|arg| (*arg).to_owned()).collect::>(), + )); + // What a real AMD Adrenalin host prints: both the 64-bit and the + // 32-bit registration, one `ICD` row each. + Some( + "ICD\tC:\\Windows\\System32\\DriverStore\\FileRepository\\u0397342.inf_amd64_x\\amdvlk64.json\n\ + ICD\tC:\\Windows\\SysWOW64\\amdvlk32.json\n" + .to_owned(), + ) + }); + + let (program, args) = argv.expect("the lookup must actually run a command"); + assert_eq!( + program, "powershell", + "the program has to be the one a stock Windows install has" + ); + assert_eq!( + args.last().map(String::as_str), + Some(WINDOWS_VULKAN_ICD_SCRIPT), + "the script that is run must be the one the parser's test reads, or the two halves \ + can drift apart while each looks right: {args:?}" + ); + assert!( + args.iter().any(|arg| arg == "-Command"), + "without it PowerShell treats the script as a file name: {args:?}" + ); + assert!( + args.iter().any(|arg| arg == "-NonInteractive") + && args.iter().any(|arg| arg == "-NoProfile"), + "a profile that prompts turns this probe into a 1.5s timeout on every `examine`, and \ + a timeout reads as `Inconclusive`: {args:?}" + ); + + // End to end: the capture became evidence that answered, and the + // classifier made the right thing of it. + assert!(icds.did_answer()); + assert_eq!(icds.found().len(), 2, "{icds:?}"); + assert_eq!(amd_icd_driver_label(&icds.found()[0]), Some("AMDVLK")); + let support = classify_vulkan_support(icds, VulkanEvidence::unanswered(), true); + assert_eq!( + support.status, + VulkanStatus::DriverPresentDevicesUnverified, + "a registry that named an AMD driver has an installed driver behind it, even with no \ + `vulkaninfo` to enumerate a device: {support:?}" + ); + + // The other end of the same wiring: a PowerShell that never answered + // must come out of the whole path as "could not look", not as a host + // with an empty registry key. + let unread = windows_icds_through(|_program, _args| None); + assert!( + !unread.did_answer() && unread.found().is_empty(), + "{unread:?}" + ); + assert_eq!( + classify_vulkan_support(unread, VulkanEvidence::unanswered(), true).status, + VulkanStatus::Inconclusive + ); + } + + /// The capture-to-evidence half on its own: what each shape of PowerShell + /// output becomes. The failure and timeout paths all arrive here as one + /// `None`, so this is where they are covered; the test above covers the + /// argv that produces it. #[test] fn a_powershell_that_never_answered_is_not_an_empty_registry_key() { // Missing `powershell`, a non-zero exit, or a timeout: all one `None`. diff --git a/docs/host-probing.md b/docs/host-probing.md index 3adf6dfbf9..6981426164 100644 --- a/docs/host-probing.md +++ b/docs/host-probing.md @@ -210,7 +210,9 @@ A driver the ICD listing *did* find is also evidence in the other direction: a h Who pays it. The probe runs once per full gather, so every caller of `ExamineSummary::gather` pays one: the text and JSON of `rocm examine`, the MCP and bridge snapshots, the free-form answer, `rocm install driver` and its `--reconcile` form, and `rocmd`'s host snapshot, MCP and sandbox paths. It is deliberately *not* run by the lean scope, `gather_without_driver_facts`, which the runtime-install finalizer uses to read the GPU target and nothing else; that scope leaves `vulkan` as `None` and `vulkan_status` reports `Unknown`. `the_summary_without_driver_facts_does_not_probe_vulkan` pins that, so restoring an unconditional probe fails a test rather than quietly adding a process to the install path. -What a user can see: on a Windows host whose PowerShell start is slower than 1.5 s, the probe times out and the host reads Vulkan-unconfirmed rather than Vulkan-capable. It is no longer told the *same thing* as a host with a genuinely empty registry key, which is what `icds_listed` buys: the timed-out host is told the driver list could not be read, never that no driver was found, and it can no longer reach `Absent` at all. The collapse that remains is inside `capture_optional_command`, which still cannot say *which* failure it was ("not installed" and "timed out" are one `None`), so the sentence names the possibilities rather than the cause. `windows_icds_from_capture` is the pure seam where that `None` is turned into "did not answer", and it is unit-tested on every platform; the registry read itself still has no gate but CI's `windows-build-and-test`. +What a user can see: on a Windows host whose PowerShell start is slower than 1.5 s, the probe times out and the host reads Vulkan-unconfirmed rather than Vulkan-capable. It is no longer told the *same thing* as a host with a genuinely empty registry key, which is what `icds_listed` buys: the timed-out host is told the driver list could not be read, never that no driver was found, and it can no longer reach `Absent` at all. The collapse that remains is inside `capture_optional_command`, which still cannot say *which* failure it was ("not installed" and "timed out" are one `None`), so the sentence names the possibilities rather than the cause. `windows_icds_from_capture` is the pure seam where that `None` is turned into "did not answer", and it is unit-tested on every platform. + +What the Windows ICD path's tests do and do not reach, stated exactly, because an overstated coverage claim is what leaves a gap nobody goes looking for. `windows_icds_through` takes the subprocess as an argument, so the program name, the flags and the script handed to PowerShell — and what the resulting capture becomes, all the way through `classify_vulkan_support` — are driven from every lane by `the_windows_registry_lookup_runs_the_khronos_script_through_powershell`. The one step no test in this repository executes is starting the process: nothing calls `detect_icds` on Windows, so the `windows-build-and-test` job *compiles* that call without ever running it. What that leaves uncovered is PowerShell's own behaviour on a real host — the script's output encoding, an execution policy that blocks it, a cold start past the 1.5 s timeout. All three arrive as a `None`, which the seam routes to `Inconclusive`, so the untested step can make the answer less useful but not false. ## Marketing name to gfx target From bf773728ea4cc3fe5c23996b1de62ca7274e46a6 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 17 Nov 2026 19:56:09 +0200 Subject: [PATCH 58/59] fix(install): grade the detected-GPU refusal's Vulkan claim by the probe MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `unsupported_host_gpu_message` closed, unconditionally, with "the Vulkan backend can still be used - try `rocm serve --engine lemonade`", without ever consulting the probe this branch exists to add. On a host with no AMD Vulkan driver one session could print "no Vulkan driver was found, so no GPU backend can run here" from `rocm examine` and the opposite from `rocm install sdk`, and the command it named could not clear the condition it was printed for, which AGENTS.md §3 requires of remediation advice. The message now classifies with this host's real `VulkanStatus` through the same `gpu_support_level` the examine line uses, and takes its wording from the same shared constants: only `VulkanOnly` keeps the Lemonade recommendation, `VulkanUnconfirmed` makes it conditional on what `vulkaninfo` reports, and `NoGpuBackend` offers the driver install instead. The refusal itself is untouched. Its gate is `target_is_vulkan_only`, which still classifies with `Unknown`, because whether a ROCm build exists is a fact about the catalog and must not turn on an unrelated Vulkan probe; the probe is paid for inside the already-failing branch. The sibling `unsupported_requested_target_message` keeps the unconditional claim for the same reason it keeps its own closing sentence: it is about a target the user named, which may not be the GPU in this machine, so no probe of this host bears on it. Signed-off-by: Roman Sirokov --- apps/rocm/src/therock/family.rs | 251 ++++++++++++++++++++++++++--- crates/rocm-core/src/gfx_target.rs | 46 ++++-- 2 files changed, 256 insertions(+), 41 deletions(-) diff --git a/apps/rocm/src/therock/family.rs b/apps/rocm/src/therock/family.rs index 1448f92ad8..cd037626e4 100644 --- a/apps/rocm/src/therock/family.rs +++ b/apps/rocm/src/therock/family.rs @@ -12,9 +12,10 @@ use super::source::{FamilyResolution, TheRockChannel}; use anyhow::{Result, bail}; use rocm_core::{ - AppPaths, GpuSupportLevel, VULKAN_ONLY_SUPPORT_CLAIM, detect_host_gfx_target, - detect_host_gpu_diagnostics, detect_managed_therock_family, gpu_support_level, - known_therock_families, normalize_therock_family, runtime_is_windows, + AppPaths, GpuSupportLevel, VULKAN_ONLY_SUPPORT_CLAIM, VULKAN_UNCONFIRMED_CLAIM, + VULKAN_UNCONFIRMED_NEXT_STEP, VulkanStatus, detect_host_gfx_target, + detect_host_gpu_diagnostics, detect_managed_therock_family, detect_vulkan_support, + gpu_support_level, known_therock_families, normalize_therock_family, runtime_is_windows, }; use std::fmt::Write as _; @@ -158,7 +159,15 @@ pub(super) fn resolve_family( if let Some(raw_arch) = host_arch.as_deref() && target_is_vulkan_only(raw_arch) { - let message = unsupported_host_gpu_message(raw_arch); + // The probe is paid for *here*, inside the branch that is already + // failing, and not by `target_is_vulkan_only` above: the refusal is a + // fact about the target, and only the sentence that follows it is a + // claim about this host. Keeping the two apart is what lets the gate + // stay exactly what it was while the advice stops over-promising. No + // successful resolve runs this, so `install sdk`'s hot path is + // unchanged, and `detect_host_gpu_diagnostics` on the next line already + // costs more than one `vulkaninfo` attempt. + let message = unsupported_host_gpu_message(raw_arch, detect_vulkan_support().status); let diagnostics = detect_host_gpu_diagnostics(); bail!("{message}\n\n{diagnostics}"); } @@ -200,9 +209,17 @@ pub(super) fn resolve_family( /// fact that has no bearing on whether a ROCm SDK exists. `Unknown` keeps the /// `VulkanOnly` tier for every target the catalog confirms has no ROCm build, /// which is exactly the set this gate refused before the probe existed — the -/// refusals are unchanged. It is also why these two messages may still -/// interpolate the unconditional `VULKAN_ONLY_SUPPORT_CLAIM` while `rocm -/// examine`, which does probe, grades its wording instead. +/// refusals are unchanged. +/// +/// This is only the *gate*, not the sentence. Whether a target has a ROCm build +/// is a fact about the catalog, and probing Vulkan to answer it would make +/// `install sdk` refuse or allow on an unrelated fact. What the refusal may +/// then *say* about Vulkan is a different question, and the two messages below +/// answer it differently: [`unsupported_host_gpu_message`] is about the GPU in +/// this machine, so it grades its claim by this host's probe, while +/// [`unsupported_requested_target_message`] is about a GPU the user named, +/// which may not be here at all, so it keeps the unconditional +/// `VULKAN_ONLY_SUPPORT_CLAIM` — there is no host whose probe would bear on it. fn target_is_vulkan_only(value: &str) -> bool { matches!( gpu_support_level( @@ -218,6 +235,31 @@ fn target_is_vulkan_only(value: &str) -> bool { /// covers. It deliberately does not list `--family` values: none of them was /// built for this GPU. /// +/// **The Vulkan half of this message is graded by the probe, not asserted.** +/// It used to close, unconditionally, with "the Vulkan backend can still be +/// used - try `rocm serve --engine lemonade`", on a host nobody had +/// looked at. That is the same unearned claim the Vulkan probe exists to +/// remove, and because this branch runs only after *this machine's* GPU has +/// been detected, there is a probe result that bears on it directly: the same +/// session could have `rocm examine` saying "no Vulkan driver was found, so no +/// GPU backend can run here" while `rocm install sdk` sent the user to serve a +/// model on that backend. AGENTS.md §3 asks that a named command be able to +/// clear the condition it was printed for, and on a `NoGpuBackend` host that +/// `rocm serve` cannot. +/// +/// So `vulkan` is routed through [`gpu_support_level`] — the same classifier +/// `rocm examine`'s `rocm_support` line uses, with the same status — and the +/// closing sentence follows the level. The claims themselves come from +/// `rocm-core`'s shared constants, so this message and that line cannot drift +/// into describing one host two ways. Only `VulkanOnly` keeps the Lemonade +/// recommendation; `VulkanUnconfirmed` makes it conditional on what +/// `vulkaninfo` reports, and `NoGpuBackend` does not make it at all. +/// +/// The *refusal* is unchanged by any of this: the gate is +/// [`target_is_vulkan_only`], which does not probe. A host with no Vulkan +/// driver was refused before and is refused now; it is only told something true +/// afterwards. +/// /// The remediation pins the engine with `--engine lemonade` rather than /// suggesting a bare `rocm serve `. This branch knows the *host* has no /// ROCm build, which is not the same as knowing which engine `serve` would @@ -241,13 +283,60 @@ fn target_is_vulkan_only(value: &str) -> bool { /// `a_gfx90c_host_the_kernel_gives_no_compute_node_still_has_a_usable_gpu` /// drives both halves off one planted `/sys`, and serve-61 runs the real binary /// against that same host shape on a GPU-less lane. -fn unsupported_host_gpu_message(raw_arch: &str) -> String { +fn unsupported_host_gpu_message(raw_arch: &str, vulkan: VulkanStatus) -> String { + // The same three arguments `target_is_vulkan_only` passes, with this host's + // real status in place of `Unknown`. The caller has already established + // that the first two make the target Vulkan-only, so only the three Vulkan + // tiers are reachable here. + let level = gpu_support_level( + Some(raw_arch), + normalize_therock_family(raw_arch).as_deref(), + vulkan, + ); + let vulkan_sentence = match level { + // A device answered, a driver is installed, or the platform cannot be + // probed at all. The last of those keeps the benefit of the doubt it + // had before any probe existed — see `GpuSupportLevel::VulkanOnly`. + GpuSupportLevel::VulkanOnly => format!( + "The GPU is detected but is {VULKAN_ONLY_SUPPORT_CLAIM}. \ + Try `rocm serve --engine lemonade` - it runs on Lemonade's \ + Vulkan backend instead." + ), + // Looked at, nothing settled. The Lemonade suggestion survives, but as + // a consequence of what `vulkaninfo` reports rather than as a promise: + // this level is reached routinely by a working Adrenalin laptop, so + // withholding it entirely would be as wrong as asserting it. + GpuSupportLevel::VulkanUnconfirmed => format!( + "The GPU is detected but is not supported by ROCm, and \ + {VULKAN_UNCONFIRMED_CLAIM} - {VULKAN_UNCONFIRMED_NEXT_STEP}. \ + If it reports a device, `rocm serve --engine lemonade` runs \ + on Lemonade's Vulkan backend." + ), + // Both halves confirmed: no ROCm build, and the probe looked for a + // Vulkan driver and found none. Naming `rocm serve` here would name a + // command that cannot clear what printed it, so the only step left is + // the one that can. + GpuSupportLevel::NoGpuBackend => { + "The GPU is detected but is not supported by ROCm, and no Vulkan GPU driver was \ + found here either, so there is no GPU backend to serve a model on. Install one \ + (Mesa RADV on Linux, the AMD display driver on Windows) and re-run `rocm \ + examine` to confirm it." + .to_owned() + } + // Unreachable: the caller gated on `target_is_vulkan_only`, which asks + // `gpu_support_level` the same first two questions, so the catalog has + // already confirmed a recognized target with no ROCm build and no + // covering family. Spelled out rather than folded into a `_` so that a + // new `GpuSupportLevel` is a compile error here instead of silently + // inheriting a Vulkan claim, and worded so that if one ever did arrive + // it would assert nothing. + GpuSupportLevel::RocmHip | GpuSupportLevel::Unknown => { + "Run `rocm examine` to see what this host supports.".to_owned() + } + }; format!( "no ROCm SDK build exists for the detected GPU target `{raw_arch}`, so \ - `rocm install sdk` has nothing to install for it.\n\ - The GPU is detected but is {VULKAN_ONLY_SUPPORT_CLAIM}. \ - Try `rocm serve --engine lemonade` - it runs on Lemonade's \ - Vulkan backend instead." + `rocm install sdk` has nothing to install for it.\n{vulkan_sentence}" ) } @@ -425,14 +514,19 @@ mod tests { /// alone can be satisfied by quietly rewriting the other's expectation. /// /// `rocm examine`'s `rocm_support` line is deliberately not in this list. - /// It is the one surface that pays for the Vulkan probe, so it grades the - /// claim by what the probe found rather than making it unconditionally - - /// see `rocm_support_line` in `rocm-core`'s `host_gpu/render.rs`. These - /// two share the constant precisely because they do *not* probe. + /// It grades the claim finer than three tiers, naming the device the probe + /// enumerated - see `rocm_support_line` in `rocm-core`'s + /// `host_gpu/render.rs`. + /// + /// The host-side refusal is asked here at the status where it *does* make + /// the claim. Which statuses those are is + /// `the_detected_gpu_refusal_grades_its_vulkan_claim_by_the_probe`'s + /// question; this one is only that the wording comes from the shared + /// constant and not from a second copy. #[test] fn both_install_refusals_render_the_shared_vulkan_only_claim() { for message in [ - unsupported_host_gpu_message("gfx90c"), + unsupported_host_gpu_message("gfx90c", VulkanStatus::Available), unsupported_requested_target_message("gfx90c", "--family"), ] { assert!( @@ -443,9 +537,116 @@ mod tests { } } + /// The detected-GPU refusal must say about Vulkan only what this host's + /// probe established. + /// + /// The regression this pins is a contradiction visible inside one session: + /// on a host with no AMD Vulkan driver, `rocm examine` reported "no Vulkan + /// GPU driver was found" while `rocm install sdk` closed with "the Vulkan + /// backend can still be used; try `rocm serve --engine lemonade`". + /// Both read as facts about the same machine, and the command the second + /// one named could not clear the condition it was printed for, which + /// AGENTS.md §3 requires of remediation advice. + /// + /// Every `VulkanStatus` is covered, so a status that stops being routed + /// through `gpu_support_level` fails here. `Unknown` is in the + /// claim-keeping list deliberately: a platform the probe cannot read at + /// all keeps the benefit of the doubt it had before the probe existed, + /// which is the one exception `GpuSupportLevel::VulkanOnly` documents. + #[test] + fn the_detected_gpu_refusal_grades_its_vulkan_claim_by_the_probe() { + // A device answered, a driver is installed, or nobody could look: all + // three may offer Lemonade, and all three say so with the shared claim. + for status in [ + VulkanStatus::Available, + VulkanStatus::DriverPresentDevicesUnverified, + VulkanStatus::Unknown, + ] { + let message = unsupported_host_gpu_message("gfx90c", status); + assert!( + message.contains(VULKAN_ONLY_SUPPORT_CLAIM), + "{status:?} is the tier that has earned the claim: {message}" + ); + assert!( + message.contains("rocm serve --engine lemonade"), + "{status:?}: {message}" + ); + } + + // Looked at and nothing settled. The question is open, so the message + // must say so rather than answer it in either direction, and the + // Lemonade suggestion survives only as a consequence of what + // `vulkaninfo` reports. + let unconfirmed = unsupported_host_gpu_message("gfx90c", VulkanStatus::Inconclusive); + assert!( + unconfirmed.contains(VULKAN_UNCONFIRMED_CLAIM), + "a host that was probed and showed nothing is owed the hedged claim, the same one \ + `rocm examine` gives it: {unconfirmed}" + ); + assert!( + !unconfirmed.contains(VULKAN_ONLY_SUPPORT_CLAIM), + "nothing here established a Vulkan backend: {unconfirmed}" + ); + assert!( + unconfirmed.contains("vulkaninfo"), + "the one thing that would settle it has to be named: {unconfirmed}" + ); + assert!( + unconfirmed.contains("If it reports a device, `rocm serve --engine lemonade`"), + "the serve suggestion must be conditional on the probe, not a promise: {unconfirmed}" + ); + + // Both halves confirmed. `rocm serve` cannot clear this, so it must not + // be named at all. + let absent = unsupported_host_gpu_message("gfx90c", VulkanStatus::Absent); + assert!( + !absent.contains(VULKAN_ONLY_SUPPORT_CLAIM), + "`rocm examine` tells this host no GPU backend can run here; `install sdk` must not \ + tell it the opposite in the same session: {absent}" + ); + assert!( + !absent.contains("rocm serve"), + "the probe looked and found no Vulkan driver, so serving on one is a command that \ + cannot clear the condition it is printed for (AGENTS.md §3): {absent}" + ); + assert!( + !absent.contains("lemonade") && !absent.contains("Lemonade"), + "naming the engine makes the same promise as naming the command: {absent}" + ); + assert!( + absent.contains("no Vulkan GPU driver was found"), + "the host is still owed what was established about it: {absent}" + ); + assert!( + absent.contains("Mesa RADV") && absent.contains("AMD display driver"), + "and the one step that can clear it: {absent}" + ); + + // The target and the opening sentence are the same whatever Vulkan + // reported: the refusal itself is a fact about the catalog, and only + // the sentence after it is a claim about this host. + for status in [ + VulkanStatus::Available, + VulkanStatus::DriverPresentDevicesUnverified, + VulkanStatus::Inconclusive, + VulkanStatus::Absent, + VulkanStatus::Unknown, + ] { + let message = unsupported_host_gpu_message("gfx90c", status); + assert!( + message.contains("no ROCm SDK build exists for the detected GPU target `gfx90c`"), + "{status:?}: {message}" + ); + assert!( + target_is_vulkan_only("gfx90c"), + "and the gate that printed it does not consult the probe at all" + ); + } + } + #[test] fn unsupported_gpu_error_names_the_gpu_and_does_not_suggest_families() { - let message = unsupported_host_gpu_message("gfx90c"); + let message = unsupported_host_gpu_message("gfx90c", VulkanStatus::Available); assert!(message.contains("no ROCm SDK build"), "{message}"); assert!(message.contains("`gfx90c`"), "{message}"); assert!(message.contains("Vulkan"), "{message}"); @@ -471,7 +672,7 @@ mod tests { /// every config, because an explicit `--engine` out-ranks all of it. #[test] fn unsupported_gpu_error_pins_the_engine_a_configured_default_could_override() { - let message = unsupported_host_gpu_message("gfx90c"); + let message = unsupported_host_gpu_message("gfx90c", VulkanStatus::Available); assert!( message.contains("rocm serve --engine lemonade"), @@ -520,11 +721,13 @@ mod tests { ); } - // The sibling path did look at this host, so it keeps the claim: this is - // the asymmetry, not a blanket ban on the sentence. It pins the engine - // for a different reason — see - // `unsupported_gpu_error_pins_the_engine_a_configured_default_could_override`. - let detected = unsupported_host_gpu_message("gfx90c"); + // The sibling path did look at this host, so it may make the claim when + // the probe backs it: this is the asymmetry, not a blanket ban on the + // sentence. It pins the engine for a different reason — see + // `unsupported_gpu_error_pins_the_engine_a_configured_default_could_override` + // — and which probe results earn it is + // `the_detected_gpu_refusal_grades_its_vulkan_claim_by_the_probe`'s. + let detected = unsupported_host_gpu_message("gfx90c", VulkanStatus::Available); assert!( detected.contains("rocm serve --engine lemonade"), "{detected}" diff --git a/crates/rocm-core/src/gfx_target.rs b/crates/rocm-core/src/gfx_target.rs index 1ce1835315..033c8a3b51 100644 --- a/crates/rocm-core/src/gfx_target.rs +++ b/crates/rocm-core/src/gfx_target.rs @@ -630,20 +630,31 @@ pub enum GpuSupportLevel { /// classifier that decides when it is true rather than being retyped at each /// site. /// -/// `rocm examine`'s `rocm_support` line deliberately does **not** interpolate -/// this. It is the one surface that pays for the Vulkan probe, so it grades the -/// second half of the claim by what the probe established — an enumerated -/// device is named, an installed-but-unenumerated driver gets a weaker -/// sentence, an unprobed platform admits it — and the unconditional "can still -/// be used" is exactly the promise this PR exists to stop making. See -/// `rocm_support_line` in `host_gpu/render.rs`. The two install refusals keep -/// it because they are asked of a *target*, before and without any host probe -/// (they classify with [`VulkanStatus::Unknown`]), so the graded wording has no -/// evidence to grade on there. +/// **This constant is the claim, not the decision to make it.** Who may say it +/// is settled by whether the speaker has a host to grade it against, and the +/// three surfaces differ: +/// +/// - `unsupported_requested_target_message` says it unconditionally, and that +/// is correct: it is about a target the *user named*, which may not be the +/// GPU in this machine, so no probe of this host bears on it. It classifies +/// with [`VulkanStatus::Unknown`] for the same reason. +/// - `unsupported_host_gpu_message` is about the GPU that was *detected here*, +/// so it classifies with this host's real [`VulkanStatus`] and interpolates +/// this constant only at [`GpuSupportLevel::VulkanOnly`]. At +/// `VulkanUnconfirmed` it says [`VULKAN_UNCONFIRMED_CLAIM`] instead, and at +/// `NoGpuBackend` it says neither. Saying it unconditionally there put `rocm +/// install sdk` and `rocm examine` in flat contradiction about the same +/// machine in the same session. +/// - `rocm examine`'s `rocm_support` line does not interpolate it at all. It +/// grades the second half of the claim finer than three tiers — an +/// enumerated device is *named*, so the reader can check it rather than take +/// it — and the unconditional "can still be used" is exactly the promise this +/// work exists to stop making. See `rocm_support_line` in +/// `host_gpu/render.rs`. /// /// Deliberately just the claim, with no leading subject and no trailing -/// punctuation: a caller supplies both. Changing the wording here changes both -/// refusals together, which is the point. +/// punctuation: a caller supplies both. Changing the wording here changes every +/// site that makes it, which is the point. pub const VULKAN_ONLY_SUPPORT_CLAIM: &str = "not supported by ROCm; the Vulkan backend can still be used"; @@ -871,11 +882,12 @@ mod tests { /// refusals, so its wording is pinned here rather than only asserted as /// "whatever the constant says". /// - /// Both refusals interpolate [`VULKAN_ONLY_SUPPORT_CLAIM`], which is what - /// stops them drifting — but it also means a careless edit to the constant - /// silently rewords both at once, and their own tests would still pass - /// because they assert containment of the same constant. This test is the - /// one place that has to be changed deliberately. + /// Both refusals interpolate [`VULKAN_ONLY_SUPPORT_CLAIM`] — one + /// unconditionally, one only at [`GpuSupportLevel::VulkanOnly`] — which is + /// what stops them drifting. But it also means a careless edit to the + /// constant silently rewords both at once, and their own tests would still + /// pass because they assert containment of the same constant. This test is + /// the one place that has to be changed deliberately. #[test] fn the_vulkan_only_claim_is_pinned_word_for_word() { assert_eq!( From 8f37693d653715e761b9b2019ebfc0b94cdbc83f Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 17 Nov 2026 20:36:31 +0200 Subject: [PATCH 59/59] docs(examine): point the vulkaninfo-exits-1 note at the scenario that records it The manual-testing note cited examine-53, which is main's unrelated dangling librocdxg scenario. The gap it describes -- a real `vulkaninfo` with no reachable driver printing "Found no drivers!" and exiting 1, which the probe reads as a source that did not answer -- is recorded on examine-57, whose feature-file comment says it plants the exit-0 shape rather than a driverless host. Signed-off-by: Roman Sirokov --- docs/manual-testing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/manual-testing.md b/docs/manual-testing.md index ee0fabfad9..c7bd98082e 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -345,7 +345,7 @@ checks, which cannot be made anywhere else: - With the driver removed but `vulkaninfo` still installed, expect the *could-not-confirm* wording on Linux rather than a confirmed absence, and it must **not** still name a driver. This is the known gap recorded on - examine-53, measured rather than assumed: a real `vulkaninfo` with no + examine-57, measured rather than assumed: a real `vulkaninfo` with no reachable driver prints "Found no drivers!" and exits **1**, and the probe counts only an exit-0 run as having answered, so the host is classed inconclusive. Telling "no drivers" apart from "the tool is broken" needs the