From e12ed3953927945cde3ecb115f04c296344176f0 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 14 Oct 2026 14:02:18 +0300 Subject: [PATCH 01/37] feat(examine,serve): report real GPU/backend support instead of a static ROCm claim ROCMAI-550: on a GPU with no ROCm/HIP build (e.g. gfx90c), `rocm examine` and `rocm engines` kept claiming the embedded Lemonade engine runs "ROCm llama.cpp backend" even though it actually falls back to Vulkan. `rocm install sdk`'s refusal for such a GPU didn't name the actual next step, and neither `rocm serve`'s deployment summary nor `rocm services list` showed which backend/ device an engine actually selected. - add `GpuSupportLevel` (RocmHip/VulkanOnly/Unknown) in rocm-core, derived from the existing `normalize_therock_family`/`gfx_target_has_no_rocm_build` primitives - make the Lemonade engine note in `rocm examine` and `rocm engines` GPU-aware instead of a hardcoded ROCm claim - `rocm install sdk`'s unsupported-GPU errors now suggest `rocm serve ` - surface the engine's actual backend/device (read live via the existing `Healthcheck` RPC, never guessed) in `rocm serve`'s deployment summary and `rocm services list` Most of the ticket's `rocm examine`/`install sdk` complaints for gfx90c were already fixed by #648 (ROCMAI-542); this covers what was left. Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 189 +++++++++++++++++- apps/rocm/src/providers.rs | 1 + apps/rocm/src/serve_cmd.rs | 14 +- apps/rocm/src/serve_summary.rs | 28 +++ apps/rocm/src/therock.rs | 8 +- crates/rocm-core/src/host_gpu.rs | 59 ++++++ crates/rocm-core/src/lib.rs | 4 +- .../features/model_serving.feature | 2 + tests/e2e-cucumber/tests/e2e/serving_steps.rs | 21 ++ 9 files changed, 311 insertions(+), 15 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 297f8154e8..218a7a54fa 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -109,7 +109,8 @@ use rocm_core::{ use rocm_engine_protocol::{ DEFAULT_LOG_TAIL_LINES, DetectRequest, DetectResponse, DevicePolicy, ENGINE_RECIPE_CONTRACT_VERSION, EngineMethod, EnginePluginDescriptor, EngineRecipeHint, - EngineRequestEnvelope, EngineResponseEnvelope, GpuSelection, StopRequest, StopResponse, + EngineRequestEnvelope, EngineResponseEnvelope, GpuSelection, HealthcheckRequest, + HealthcheckResponse, StopRequest, StopResponse, }; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; @@ -2740,6 +2741,21 @@ fn builtin_codex_bridge_engine_inventory() -> Vec { .collect() } +/// The lemonade engine note shown in `rocm examine` and `rocm engines`, adjusted +/// for whether this host's detected GPU can actually use the ROCm llama.cpp +/// backend. Both call sites have a live `GpuSupportLevel` in scope; other +/// places that print this note statically (the dashboard TUI, `rocmd`'s own +/// catalog) have no host probe in front of them and keep the unconditional +/// ROCm wording, since they describe the engine in the abstract. +const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str { + match level { + rocm_core::GpuSupportLevel::VulkanOnly => { + "default embedded Lemonade server; this GPU has no ROCm/HIP support, so it runs on the Vulkan llama.cpp backend" + } + _ => "default embedded Lemonade server with ROCm llama.cpp backend", + } +} + const fn builtin_engine_inventory() -> &'static [(&'static str, &'static str)] { &[ ( @@ -9066,7 +9082,17 @@ fn examine_human_report( let mut output = render_examine_plain_header(&summary); output.push_str(&summary.render_text()); append_examine_runtime_state(&mut output, paths, config)?; - append_examine_engine_inventory(&mut output, paths, config, &summary.default_engine); + let gpu_support_level = rocm_core::gpu_support_level( + summary.detected_gfx_target.as_deref(), + summary.compatible_therock_family.as_deref(), + ); + append_examine_engine_inventory( + &mut output, + paths, + config, + &summary.default_engine, + gpu_support_level, + ); Ok((output, summary)) } @@ -9124,6 +9150,10 @@ fn render_engine_inventory_text_with_paths(paths: Option<&AppPaths>) -> String { // constant put the `*` on Lemonade even on Instinct, where `serve` picks vLLM. let host_gpu = rocm_core::detect_host_gpu_summary(paths); let host_default_engine = rocm_core::default_engine_for_host(&host_gpu); + let gpu_support_level = rocm_core::gpu_support_level( + host_gpu.gfx_target.as_deref(), + host_gpu.therock_family.as_deref(), + ); // A configured `default_engine` still wins over the host preference, mirroring // `select_serve_engine` (and `append_examine_engine_inventory`). Without this, // the legend could mark an engine `serve` will not actually default to. @@ -9170,6 +9200,11 @@ fn render_engine_inventory_text_with_paths(paths: Option<&AppPaths>) -> String { } else { " " }; + let note = if *name == "lemonade" { + lemonade_engine_note(gpu_support_level) + } else { + *note + }; let _ = writeln!(output, "{marker} {name:10} {note}"); append_engine_detect_summary(&mut output, name, paths); } @@ -9294,6 +9329,7 @@ fn append_examine_engine_inventory( paths: &AppPaths, config: &RocmCliConfig, host_default_engine: &str, + gpu_support_level: rocm_core::GpuSupportLevel, ) { // Mirrors `select_serve_engine`'s guard: a blank configured value must not // out-rank the host preference, or `effective_default` below would become @@ -9380,6 +9416,11 @@ fn append_examine_engine_inventory( .or(entry.last_installed_env_id.as_deref()) }) .unwrap_or(""); + let note = if *engine == "lemonade" { + lemonade_engine_note(gpu_support_level) + } else { + *note + }; let _ = writeln!( output, " {marker} {engine} adapter={adapter} runtime_pref={runtime_pref} note={note}" @@ -10708,6 +10749,16 @@ pub(crate) fn render_services_text(paths: &AppPaths, all: bool) -> Result Option { + let response: HealthcheckResponse = engine_request( + Some(paths), + engine, + EngineMethod::Healthcheck, + &HealthcheckRequest { + service_id: service_id.to_owned(), + }, + ) + .ok()?; + // `"unknown"` is the engines' own device value when no state is on disk for + // this service id yet (see `engines/vllm/src/state.rs` and + // `engines/lemonade/src/lib.rs`), which is not more informative than + // showing nothing. + (!response.device.is_empty() && response.device != "unknown").then_some(response.device) +} + fn decode_engine_response(envelope: EngineResponseEnvelope) -> Result where R: DeserializeOwned, @@ -21605,6 +21681,10 @@ install therock"; ); assert!(rendered.contains("- svc-ready")); assert!(rendered.contains(" stop: rocm services stop svc-ready --yes")); + // No real engine state file backs this record, so the engine's own + // Healthcheck reports `device: "unknown"` — not more informative than + // omitting the line, so it must not appear. + assert!(!rendered.contains("device:"), "{rendered}"); assert!(!rendered.contains("- svc-failed")); assert!(all.contains("- svc-failed")); assert!(all.contains(" restart: rocm services restart svc-failed --yes")); @@ -25829,7 +25909,13 @@ install therock"; }; let mut output = String::new(); - append_examine_engine_inventory(&mut output, &paths, &config, "vllm"); + append_examine_engine_inventory( + &mut output, + &paths, + &config, + "vllm", + rocm_core::GpuSupportLevel::Unknown, + ); assert!( output.contains("configured_default_engine: "), @@ -26559,7 +26645,13 @@ install therock"; // Host default deliberately disagrees with the configured value: an // engine the user chose must outrank whatever the GPU would prefer. - append_examine_engine_inventory(&mut output, &paths, &config, "lemonade"); + append_examine_engine_inventory( + &mut output, + &paths, + &config, + "lemonade", + rocm_core::GpuSupportLevel::Unknown, + ); assert!(output.contains("engine_inventory:")); assert!(output.contains("configured_default_engine: vllm")); @@ -26576,7 +26668,13 @@ install therock"; let config = RocmCliConfig::default(); let mut output = String::new(); - append_examine_engine_inventory(&mut output, &paths, &config, "vllm"); + append_examine_engine_inventory( + &mut output, + &paths, + &config, + "vllm", + rocm_core::GpuSupportLevel::Unknown, + ); let _ = fs::remove_dir_all(root); let legend = format!("legend: {DEFAULT_ENGINE_MARKER} = default engine"); @@ -26602,7 +26700,13 @@ install therock"; }; let mut output = String::new(); - append_examine_engine_inventory(&mut output, &paths, &config, "vllm"); + append_examine_engine_inventory( + &mut output, + &paths, + &config, + "vllm", + rocm_core::GpuSupportLevel::Unknown, + ); let _ = fs::remove_dir_all(root); assert!( @@ -26622,7 +26726,13 @@ install therock"; let config = RocmCliConfig::default(); let mut output = String::new(); - append_examine_engine_inventory(&mut output, &paths, &config, "vllm"); + append_examine_engine_inventory( + &mut output, + &paths, + &config, + "vllm", + rocm_core::GpuSupportLevel::Unknown, + ); // The configured line still reads as unset — the host default must not be // mistaken for something the user chose. @@ -26654,7 +26764,13 @@ install therock"; }; let mut output = String::new(); - append_examine_engine_inventory(&mut output, &paths, &config, "vllm"); + append_examine_engine_inventory( + &mut output, + &paths, + &config, + "vllm", + rocm_core::GpuSupportLevel::Unknown, + ); assert!( output.contains("effective_default_engine: lemonade"), @@ -26679,6 +26795,55 @@ install therock"; let _ = fs::remove_dir_all(root); } + #[test] + fn examine_engine_inventory_names_vulkan_for_a_gpu_with_no_rocm_support() { + // A gfx90c host (or any confirmed-no-ROCm-build GPU) actually runs + // Lemonade's Vulkan backend, not the ROCm one the static catalog text + // always claimed. + let (root, paths) = test_paths("examine-engine-inventory-vulkan-only"); + let config = RocmCliConfig::default(); + let mut output = String::new(); + + append_examine_engine_inventory( + &mut output, + &paths, + &config, + "lemonade", + rocm_core::GpuSupportLevel::VulkanOnly, + ); + + assert!( + output.contains("no ROCm/HIP support") && output.contains("Vulkan llama.cpp backend"), + "{output}" + ); + assert!( + !output.contains("note=default embedded Lemonade server with ROCm"), + "{output}" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn examine_engine_inventory_keeps_rocm_note_for_a_rocm_capable_gpu() { + let (root, paths) = test_paths("examine-engine-inventory-rocm-capable"); + let config = RocmCliConfig::default(); + let mut output = String::new(); + + append_examine_engine_inventory( + &mut output, + &paths, + &config, + "lemonade", + rocm_core::GpuSupportLevel::RocmHip, + ); + + assert!( + output.contains("note=default embedded Lemonade server with ROCm llama.cpp backend"), + "{output}" + ); + let _ = fs::remove_dir_all(root); + } + #[test] fn examine_is_silent_when_the_configured_engine_agrees_with_the_host() { let (root, paths) = test_paths("examine-engine-inventory-agrees"); @@ -26688,7 +26853,13 @@ install therock"; }; let mut output = String::new(); - append_examine_engine_inventory(&mut output, &paths, &config, "vllm"); + append_examine_engine_inventory( + &mut output, + &paths, + &config, + "vllm", + rocm_core::GpuSupportLevel::Unknown, + ); assert!( !output.contains("configured_default_note"), diff --git a/apps/rocm/src/providers.rs b/apps/rocm/src/providers.rs index 968cf69f9b..ecbdc5669c 100644 --- a/apps/rocm/src/providers.rs +++ b/apps/rocm/src/providers.rs @@ -2915,6 +2915,7 @@ mod tests { chat_endpoint: "http://127.0.0.1:8000/v1/chat/completions".to_owned(), service_id: "svc-ready".to_owned(), status: "ready".to_owned(), + device: None, already_running: false, metrics, api_key: None, diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 275115f2ac..5b1f38d8bd 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -46,8 +46,8 @@ use crate::service_launch::{ }; use crate::{ cli_progress, collect_serve_notes, detect_gpu_count, device_policy_name, engine_request, - gpu_vram_usage, parse_device_policy, parse_gpu_selection, resolve_engine_selection, - select_gpu_indices_under_launch_lock, serve_gpu_low_memory_warning, + fetch_service_device, gpu_vram_usage, parse_device_policy, parse_gpu_selection, + resolve_engine_selection, select_gpu_indices_under_launch_lock, serve_gpu_low_memory_warning, validate_engine_selection_runtime, validate_pinned_gpu_index, }; @@ -910,6 +910,15 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { }; spinner.clear(); + // Read what the engine actually selected rather than guessing from + // the pre-launch GPU plan: `gpu_indices` above can be empty for a + // Vulkan-only GPU that ROCm/amd-smi never enumerates, even though + // the server is running fine on it. Only meaningful once the + // server is ready to answer, same gate as the smoke test. + let device = (report.status == "ready") + .then(|| fetch_service_device(&paths, &selected_engine, &report.service_id)) + .flatten(); + let notes = collect_serve_notes( cpu_only, &gpu_selection, @@ -927,6 +936,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { chat_endpoint: format!("{}/chat/completions", report.endpoint_url), service_id: report.service_id.clone(), status: report.status.clone(), + device, already_running: report.already_running, metrics, api_key: launched_key, diff --git a/apps/rocm/src/serve_summary.rs b/apps/rocm/src/serve_summary.rs index 5fea572403..45a17cc898 100644 --- a/apps/rocm/src/serve_summary.rs +++ b/apps/rocm/src/serve_summary.rs @@ -55,6 +55,10 @@ pub(crate) struct DeploymentSummary { pub service_id: String, /// `"ready"`, `"starting"`, or an existing service's status. pub status: String, + /// The backend/device the engine actually selected (e.g. `"vulkan gpu 0"`, + /// `"rocm gpu 0"`), read from the engine's own `Healthcheck` response. + /// `None` when the query failed or the service was not ready to ask. + pub device: Option, /// True when an equivalent server was already running and nothing was spawned. pub already_running: bool, pub metrics: SmokeMetrics, @@ -121,11 +125,17 @@ pub(crate) fn render_summary(summary: &DeploymentSummary) -> String { ] }; + let device_row = summary + .device + .as_deref() + .map(|device| ("device", device.to_owned())); + let rows: Vec<(&str, String)> = [ vec![ ("status", summary.status.clone()), ("engine", summary.engine.clone()), ], + device_row.into_iter().collect(), model_rows, vec![ ("endpoint", summary.chat_endpoint.clone()), @@ -265,6 +275,7 @@ mod tests { chat_endpoint: "http://127.0.0.1:1337/v1/chat/completions".to_owned(), service_id: "vllm-qwen-1720000000".to_owned(), status: "ready".to_owned(), + device: None, already_running: false, metrics: SmokeMetrics { ttft: Some(Duration::from_millis(180)), @@ -435,6 +446,23 @@ mod tests { assert!(render_summary(&summary).contains("note: selected GPU 0 has low free VRAM")); } + #[test] + fn device_row_shown_only_when_known() { + let without_device = render_summary(&base_summary()); + assert!(!without_device.contains("device"), "{without_device}"); + + let mut summary = base_summary(); + summary.device = Some("vulkan gpu 0".to_owned()); + let rendered = render_summary(&summary); + assert!( + rendered + .lines() + .any(|line| line.trim_start().starts_with("device") + && line.trim_end().ends_with("vulkan gpu 0")), + "{rendered}" + ); + } + #[test] fn api_key_client_config_shown_only_when_present() { // Loopback / no key: the summary must not mention an api key at all. diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index 9a692c32cb..0efd9fbc3d 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -3459,7 +3459,8 @@ fn unsupported_host_gpu_message(raw_arch: &str) -> String { "no ROCm SDK build exists for the detected GPU target `{raw_arch}`, so \ `rocm install sdk` has nothing to install for it.\n\ The GPU is detected but is not supported by ROCm; \ - the Vulkan backend can still be used." + the Vulkan backend can still be used. \ + Try `rocm serve ` — it runs on Lemonade's Vulkan backend instead." ) } @@ -3475,7 +3476,8 @@ fn unsupported_requested_target_message(value: &str, source: &str) -> String { "no ROCm SDK build exists for the GPU target `{value}` requested by \ {source}, so `rocm install sdk` has nothing to install for it.\n\ That GPU is not supported by ROCm; the Vulkan backend can still be \ - used. Nothing was installed and no other family was substituted." + used. Nothing was installed and no other family was substituted. \ + Try `rocm serve ` — it runs on Lemonade's Vulkan backend instead." ) } @@ -7665,6 +7667,7 @@ mod tests { assert!(message.contains("no ROCm SDK build"), "{message}"); assert!(message.contains("`gfx90c`"), "{message}"); assert!(message.contains("Vulkan"), "{message}"); + assert!(message.contains("rocm serve "), "{message}"); assert!(!message.contains("--family"), "{message}"); for family in known_therock_families() { assert!( @@ -7746,6 +7749,7 @@ mod tests { let message = unsupported_requested_target_message("gfx90c", "--family"); assert!(message.contains("`gfx90c`"), "{message}"); assert!(message.contains("--family"), "{message}"); + assert!(message.contains("rocm serve "), "{message}"); // The user named it; nothing was detected, so the error must not say so. assert!(!message.contains("detected"), "{message}"); diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index c985ed5358..5256e09188 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -128,6 +128,40 @@ pub fn gfx_target_has_no_rocm_build(gfx: &str) -> bool { && normalize_therock_family(gfx).is_none() } +/// How well this host's GPU is supported, for messages that tell the user what +/// they can actually do rather than just reporting raw detection fields. +/// +/// `Unknown` deliberately covers both "nothing detected" and "a gfx target we +/// don't recognize" rather than being called `Unsupported` — see the doc +/// comment on [`gfx_target_has_no_rocm_build`] and +/// `GFX_TARGETS_WITHOUT_ROCM_SUPPORT` in `diagnose.rs` for why conflating +/// "confirmed no ROCm build" with "unrecognized" would turn a detection gap +/// into a false claim about the hardware. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuSupportLevel { + /// A TheRock package family covers this GPU: full ROCm/HIP support. + RocmHip, + /// Recognized, confirmed to have no ROCm/HIP build, but Vulkan-based + /// engines (the default Lemonade backend) still work on it. + VulkanOnly, + /// No GPU detected yet, or a gfx target this catalog doesn't recognize. + Unknown, +} + +/// Classify the host's GPU support level from the same two facts +/// `ExamineSummary` and `resolve_family` already compute: the detected gfx +/// target and whichever TheRock family (if any) covers it. +pub fn gpu_support_level( + detected_gfx_target: Option<&str>, + compatible_therock_family: Option<&str>, +) -> GpuSupportLevel { + match (detected_gfx_target, compatible_therock_family) { + (_, Some(_)) => GpuSupportLevel::RocmHip, + (Some(gfx), None) if gfx_target_has_no_rocm_build(gfx) => GpuSupportLevel::VulkanOnly, + _ => GpuSupportLevel::Unknown, + } +} + pub fn normalize_therock_family(value: &str) -> Option { let normalized = value.trim().to_ascii_lowercase(); if normalized.is_empty() { @@ -3457,6 +3491,31 @@ mod tests { assert!(!gfx_target_has_no_rocm_build("")); } + #[test] + fn gpu_support_level_is_rocm_hip_when_a_family_covers_it() { + assert_eq!( + gpu_support_level(Some("gfx90a"), Some("gfx90a")), + GpuSupportLevel::RocmHip + ); + } + + #[test] + fn gpu_support_level_is_vulkan_only_for_gfx90c() { + assert_eq!( + gpu_support_level(Some("gfx90c"), None), + GpuSupportLevel::VulkanOnly + ); + } + + #[test] + fn gpu_support_level_is_unknown_without_detection_or_for_an_unrecognized_target() { + assert_eq!(gpu_support_level(None, None), GpuSupportLevel::Unknown); + assert_eq!( + gpu_support_level(Some("gfx_totally_unrecognized"), None), + GpuSupportLevel::Unknown + ); + } + /// The predicate must agree with `normalize_therock_family`, not merely /// happen to. /// diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index a23896c3fe..1004a33767 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -65,11 +65,11 @@ pub use fix::{ }; pub use hardware_root::host_path; pub use host_gpu::{ - DriverSummary, ExamineSummary, HostGpuSummary, WslHostDriverProbe, WslSummary, + DriverSummary, ExamineSummary, GpuSupportLevel, HostGpuSummary, WslHostDriverProbe, WslSummary, default_engine_for_host, default_engine_for_platform, detect_gpu_driver_version, detect_host_gfx_target, detect_host_gpu_diagnostics, detect_host_gpu_summary, detect_system_ram_gib, extract_first_gfx_token, gfx_target_has_no_rocm_build, - has_usable_amd_gpu, interactive_terminal, is_amdgpu_device, is_wsl_host, + gpu_support_level, has_usable_amd_gpu, interactive_terminal, is_amdgpu_device, is_wsl_host, known_therock_families, normalize_therock_family, preferred_serve_engine_for_host_gpu_summary, usable_amd_gpu_indices, }; diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 93c3c874e3..a67149b05c 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -43,6 +43,7 @@ Feature: Model serving When the user sends a chat completion request Then the response contains a model reply And the response identifies the correct model + And the services list reports the engine's actual backend device # Large-model coverage (dogfooding W9): serve a representative large model for # each GPU platform end-to-end at least once. MI300X uses Qwen/Qwen3.6-27B through @@ -68,6 +69,7 @@ Feature: Model serving When the user sends a chat completion request Then the response contains a model reply And the response identifies the correct model + And the services list reports the engine's actual backend device # HF-checkpoint direct-serve canary (EAI-8026). An `owner/repo:variant` ref # bypasses Lemonade's model router and runs a packaged llama-server directly diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 4a382eb833..6840d5e61f 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1346,6 +1346,27 @@ async fn assert_service_in_list(world: &mut E2eWorld) { ); } +/// ROCMAI-550: `rocm services list` must show the backend/device the engine +/// actually selected (read live from its `Healthcheck` RPC), not just that a +/// service exists. Reuses `assert_service_in_list`'s model-name match so this +/// only checks the row for the service this scenario just started. +#[then("the services list reports the engine's actual backend device")] +async fn assert_services_list_reports_device(world: &mut E2eWorld) { + let (stdout, _, _) = crate::run_rocm(world, &["services", "list"]); + let model = world.model_name.as_deref().unwrap_or(""); + let block = stdout + .split("\n- ") + .find(|block| block.to_lowercase().contains(&model.to_lowercase())) + .unwrap_or_else(|| panic!("no services-list entry for model {model}:\n{stdout}")); + assert!( + block + .lines() + .any(|line| line.trim_start().starts_with("device:")), + "services list entry for {model} is missing a `device:` line — the engine's \ + Healthcheck RPC should report the backend it actually selected:\n{block}" + ); +} + #[then("the list reports the attempt and how to look at it")] async fn assert_past_attempts_reported(world: &mut E2eWorld) { let stdout = world From c63fa43cae664a719b12641a7273fac6a6049164 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 14 Oct 2026 15:30:01 +0300 Subject: [PATCH 02/37] test(serve): keep the api key out of summary assertion messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeQL flagged two assertions as cleartext logging of `api_key`. Those two use `base_summary()`, whose key is `None`, so the literal instances are false positives — but the pattern is not. `render_summary` prints the key by design, and `api_key_client_config_shown_only_when_present` sets a real one, so a failure there copied `endpoint-secret` into CI logs, which are retained and widely readable. Assert on the real text, print a redacted copy. Covers all twelve sites that interpolated a rendered summary, not just the two CodeQL picked. Production output is unchanged: the key is still shown once, and the test pinning that still asserts the exact string. Signed-off-by: Roman Sirokov --- apps/rocm/src/serve_summary.rs | 65 +++++++++++++++++++++++++++------- 1 file changed, 53 insertions(+), 12 deletions(-) diff --git a/apps/rocm/src/serve_summary.rs b/apps/rocm/src/serve_summary.rs index 45a17cc898..b153db4753 100644 --- a/apps/rocm/src/serve_summary.rs +++ b/apps/rocm/src/serve_summary.rs @@ -267,6 +267,22 @@ fn compute_metrics( mod tests { use super::*; + /// A rendered summary with the api key masked, for panic messages. + /// + /// `render_summary` prints the key by design, and a failing assertion that + /// interpolated the whole summary would copy it into CI logs, which are + /// retained and widely readable. Assert on the real text; print this. + fn redacted(rendered: &str) -> String { + rendered + .lines() + .map(|line| match line.split_once("api key: ") { + Some((prefix, _)) => format!("{prefix}api key: "), + None => line.to_owned(), + }) + .collect::>() + .join("\n") + } + fn base_summary() -> DeploymentSummary { DeploymentSummary { engine: "vllm".to_owned(), @@ -378,17 +394,19 @@ mod tests { line.trim_start().starts_with(label) && line.trim_end().ends_with(value) }) }; - assert!(row("requested model", "qwen"), "{rendered}"); + assert!(row("requested model", "qwen"), "{}", redacted(&rendered)); assert!( row("resolved model", "Qwen3-4B-Instruct-2507-GGUF"), - "{rendered}" + "{}", + redacted(&rendered) ); // The bare `model` row is not used when the two differ. assert!( !rendered .lines() .any(|line| line.trim_start().starts_with("model ")), - "{rendered}" + "{}", + redacted(&rendered) ); } @@ -407,7 +425,11 @@ mod tests { let mut summary = base_summary(); summary.status = "starting".to_owned(); let rendered = render_summary(&summary); - assert!(rendered.contains("not ready yet"), "heading: {rendered}"); + assert!( + rendered.contains("not ready yet"), + "heading: {}", + redacted(&rendered) + ); assert!(rendered.contains("did not report ready")); assert!(rendered.contains("rocm logs --service")); } @@ -422,14 +444,20 @@ mod tests { summary.status = "running".to_owned(); summary.metrics = SmokeMetrics::default(); let rendered = render_summary(&summary); - assert!(rendered.contains("not ready yet"), "heading: {rendered}"); + assert!( + rendered.contains("not ready yet"), + "heading: {}", + redacted(&rendered) + ); assert!( rendered.contains("no smoke test was run"), - "the skipped smoke test must be explained: {rendered}" + "the skipped smoke test must be explained: {}", + redacted(&rendered) ); assert!( rendered.contains("still loading"), - "the user needs to know this resolves on its own: {rendered}" + "the user needs to know this resolves on its own: {}", + redacted(&rendered) ); assert!(rendered.contains("rocm services list")); } @@ -449,7 +477,11 @@ mod tests { #[test] fn device_row_shown_only_when_known() { let without_device = render_summary(&base_summary()); - assert!(!without_device.contains("device"), "{without_device}"); + assert!( + !without_device.contains("device"), + "{}", + redacted(&without_device) + ); let mut summary = base_summary(); summary.device = Some("vulkan gpu 0".to_owned()); @@ -459,7 +491,8 @@ mod tests { .lines() .any(|line| line.trim_start().starts_with("device") && line.trim_end().ends_with("vulkan gpu 0")), - "{rendered}" + "{}", + redacted(&rendered) ); } @@ -467,14 +500,22 @@ mod tests { fn api_key_client_config_shown_only_when_present() { // Loopback / no key: the summary must not mention an api key at all. let plain = render_summary(&base_summary()); - assert!(!plain.contains("api key"), "{plain}"); + assert!(!plain.contains("api key"), "{}", redacted(&plain)); // Public bind: the key is shown once as secure client config. let mut summary = base_summary(); summary.api_key = Some("endpoint-secret".to_owned()); let rendered = render_summary(&summary); - assert!(rendered.contains("api key: endpoint-secret"), "{rendered}"); - assert!(rendered.contains("shown only now"), "{rendered}"); + assert!( + rendered.contains("api key: endpoint-secret"), + "{}", + redacted(&rendered) + ); + assert!( + rendered.contains("shown only now"), + "{}", + redacted(&rendered) + ); } #[test] From 67cd4823f099b7ff78d29de4965731fa510c3af5 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 14 Oct 2026 16:43:52 +0300 Subject: [PATCH 03/37] test(runtime): assert the Vulkan-serving remediation where the refusal is produced MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The gfx90c refusals in `therock.rs` gained a `rocm serve ` remediation sentence, a second user-observable output change in this PR with no scenario behind it. AGENTS.md §3 wants remediation advice asserted where the message is really printed, not only against the formatter. runtime-19 already opens the one door of the two a lane can reach -- naming the target with `--family` -- so the assertion goes there. The detected-host door needs a Vega APU no runner has and stays unit-covered, as its existing comment already records. Also names the codex-bridge engine inventory at both the static table and the bridge helper as a deliberate scope boundary, instead of only in `lemonade_engine_note`'s doc comment where a reader of the table would miss it. Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 31 ++++++++++++++++--- .../features/runtime_setup.feature | 8 +++++ tests/e2e-cucumber/tests/e2e/runtime_steps.rs | 26 ++++++++++++++++ 3 files changed, 61 insertions(+), 4 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 218a7a54fa..63d0177db8 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -2724,6 +2724,19 @@ fn build_codex_bridge_gpu_snapshot(config: &RocmCliConfig) -> CodexBridgeGpuSnap } } +/// The engine catalogue in the codex-bridge snapshot, taken verbatim from +/// [`builtin_engine_inventory`] — including lemonade's unconditional "with ROCm +/// llama.cpp backend", which ROCMAI-550 replaced in `rocm examine` and +/// `rocm engines` but deliberately left alone here. +/// +/// It is left alone because this list is the static catalogue, in the same +/// shape as the unconditional `installed_binary: true` beside it: every field +/// but `default_for_platform` describes the engine, not this host. A consumer +/// that wants the host's real support level reads it off the `examine` block +/// in the same snapshot, which carries `detected_gfx_target` and +/// `compatible_therock_family` — the two inputs `rocm_core::gpu_support_level` +/// takes. Making this row host-aware would change the bridge's JSON contract +/// and is worth its own change, not a rider on this one. fn builtin_codex_bridge_engine_inventory() -> Vec { let default_engine = default_engine_for_platform(); let binary_path = daemon_binary_path() @@ -2743,10 +2756,11 @@ fn builtin_codex_bridge_engine_inventory() -> Vec { /// The lemonade engine note shown in `rocm examine` and `rocm engines`, adjusted /// for whether this host's detected GPU can actually use the ROCm llama.cpp -/// backend. Both call sites have a live `GpuSupportLevel` in scope; other -/// places that print this note statically (the dashboard TUI, `rocmd`'s own -/// catalog) have no host probe in front of them and keep the unconditional -/// ROCm wording, since they describe the engine in the abstract. +/// backend. Both call sites have a live `GpuSupportLevel` in scope; the places +/// that print this note statically — the dashboard TUI, `rocmd`'s own catalog, +/// and [`builtin_codex_bridge_engine_inventory`]'s copy of +/// [`builtin_engine_inventory`] — keep the unconditional ROCm wording, since +/// they describe the engine in the abstract rather than this host. const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str { match level { rocm_core::GpuSupportLevel::VulkanOnly => { @@ -2756,6 +2770,15 @@ const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str } } +/// The static engine catalogue: what each engine is, independent of any host. +/// +/// Lemonade's entry says "with ROCm llama.cpp backend" unconditionally, which +/// is only true of a host whose GPU has a ROCm build. Callers with a live +/// [`rocm_core::GpuSupportLevel`] must not print this row for lemonade — they +/// substitute [`lemonade_engine_note`], as `rocm examine` and `rocm engines` +/// do. This row is what the remaining host-blind consumers get; see +/// [`builtin_codex_bridge_engine_inventory`] for why that is the right answer +/// for them. const fn builtin_engine_inventory() -> &'static [(&'static str, &'static str)] { &[ ( diff --git a/tests/e2e-cucumber/features/runtime_setup.feature b/tests/e2e-cucumber/features/runtime_setup.feature index 0e2047c6fb..a2164d1f92 100644 --- a/tests/e2e-cucumber/features/runtime_setup.feature +++ b/tests/e2e-cucumber/features/runtime_setup.feature @@ -410,8 +410,16 @@ Feature: Runtime configuration # build" is absent however the run then ends. The second guards the weaker # failure where a refusal is printed but still dangles a family name in front # of the user. + # + # ROCMAI-550 added a third Then. Having taken every package family away, the + # refusal now has to leave the user somewhere to go, and it says `rocm serve + # ` on the Vulkan backend. That is a remediation line naming a command, + # so AGENTS.md §3 wants it asserted where the refusal is actually produced + # rather than only against the formatter -- this is the one door of the two + # that a lane can open, so it is asserted here. @id:runtime-install-sdk-refuses-unsupported-gpu-target Scenario: runtime-19 - Asking for a GPU target ROCm does not build is refused outright When the user dry-runs an SDK install for the gfx90c Vega iGPU Then the install is refused as having no ROCm build for that GPU And the refusal offers no package family to install instead + And the refusal points at serving on the Vulkan backend instead diff --git a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs index c7e6197d42..8c7378394d 100644 --- a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs @@ -1457,3 +1457,29 @@ async fn assert_unsupported_family_offers_nothing(world: &mut E2eWorld) { ); } } + +/// Having removed every `--family` the user could try, the refusal has to say +/// what *will* work. ROCMAI-550 added that sentence, and this pins it where the +/// refusal is really produced rather than against the formatter alone. +/// +/// Only the wording is checked here, because this lane cannot check the rest of +/// it: proving that `rocm serve` then reaches a GPU with no ROCm build needs a +/// Vega APU, and no runner has one (same gap as the detected-host door, above). +/// The two halves the advice rests on are proven elsewhere — +/// `rocm_core::gfx_target_has_no_rocm_build` is what put the user here, and +/// `selects_vulkan_when_rocm_unsupported` in `engines/lemonade`'s +/// `backend_alignment` is what makes llama.cpp pick `vulkan` once `rocm` comes +/// back unsupported, which is the claim the sentence makes. +#[then("the refusal points at serving on the Vulkan backend instead")] +async fn assert_unsupported_family_points_at_vulkan_serving(world: &mut E2eWorld) { + let output = unsupported_family_attempt(world); + assert!( + output.contains("rocm serve "), + "the refusal took every package family away, so it must name what does \ + work instead:\n{output}" + ); + assert!( + output.contains("Vulkan"), + "the refusal should say the fallback is the Vulkan backend:\n{output}" + ); +} From 61bb3c3d1f7742c841f5199c7910a69482c4a6fc Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 14 Oct 2026 20:31:35 +0300 Subject: [PATCH 04/37] fix(install): stop the requested-target refusal predicting a backend `unsupported_requested_target_message` is raised from the `--family` and `ROCM_CLI_THEROCK_FAMILY` branches at the top of `resolve_family`, before any host probe runs, so the rejected target says nothing about the GPU in this machine. It had borrowed its sibling's closing sentence promising that `rocm serve ` runs on Lemonade's Vulkan backend -- true in `unsupported_host_gpu_message`, which is only reached once host detection itself returns a target with no ROCm build, but false here: a stale env value naming gfx90c refuses just the same on a host whose own GPU is fully ROCm-capable, and there serving would run on ROCm/HIP. Point at `rocm examine` instead. It is true on every host, it is still a next step, and it is the command this PR just taught to report the real support level. runtime-19 was asserting the wrong half of this: it drives `--family gfx90c`, so it was pinning the Vulkan-serving claim green on exactly the ROCm-capable runners where it was false. It now asserts the examine pointer and the negative. Also route `ExamineSummary::rocm_support_line` through `gpu_support_level` rather than re-deriving the same classification, and document the serve summary's `device:` row in README and the manual-testing GPU check. Signed-off-by: Roman Sirokov --- README.md | 7 ++- apps/rocm/src/therock.rs | 55 ++++++++++++++++++- crates/rocm-core/src/host_gpu.rs | 25 ++++++--- docs/manual-testing.md | 8 ++- .../features/model_serving.feature | 8 +++ .../features/runtime_setup.feature | 14 +++-- tests/e2e-cucumber/tests/e2e/runtime_steps.rs | 36 ++++++------ 7 files changed, 115 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 861591ae81..6b0b92f7e2 100644 --- a/README.md +++ b/README.md @@ -656,8 +656,11 @@ services stop --yes`) or match the original flags. By default the server runs in the background under rocm-cli's supervision and prints a deployment summary — a progress indicator while it starts, then a table -with the status, the full inference endpoint, the API-qualified model name, and a -quick smoke test (time to first token and approximate tokens/sec). Control +with the status, the engine and the backend device it actually selected (for +example `rocm gpu 0` or `vulkan gpu 0`, read back from the running engine), the +full inference endpoint, the API-qualified model name, and a quick smoke test +(time to first token and approximate tokens/sec). `rocm services` reports the +same `device` for each running service. Control returns to your shell with the server still running; manage it later with `rocm services` (below). diff --git a/apps/rocm/src/therock.rs b/apps/rocm/src/therock.rs index 0efd9fbc3d..2395762b68 100644 --- a/apps/rocm/src/therock.rs +++ b/apps/rocm/src/therock.rs @@ -3471,13 +3471,24 @@ fn unsupported_host_gpu_message(raw_arch: &str) -> String { /// here: the user named the target, so the error names where it came from and /// does not claim a GPU is present. Like that message it offers no `--family` /// values, since none of them was built for this GPU. +/// +/// That same distinction is why this one must not borrow its sibling's closing +/// sentence. `unsupported_host_gpu_message` can promise that `rocm serve` runs +/// on Lemonade's Vulkan backend because it is only reached once host detection +/// has itself come back with a target that has no ROCm build. This message is +/// raised from the `--family` / `ROCM_CLI_THEROCK_FAMILY` branches at the top of +/// [`resolve_family`], before any host probe runs, so the rejected target says +/// nothing about the GPU in this machine: a stale env value naming gfx90c +/// refuses just the same on a host whose own GPU is fully ROCm-capable, and +/// there `rocm serve` would run on ROCm/HIP. So it points at `rocm examine`, +/// which reports this host's real support level, instead of asserting one. fn unsupported_requested_target_message(value: &str, source: &str) -> String { format!( "no ROCm SDK build exists for the GPU target `{value}` requested by \ {source}, so `rocm install sdk` has nothing to install for it.\n\ That GPU is not supported by ROCm; the Vulkan backend can still be \ used. Nothing was installed and no other family was substituted. \ - Try `rocm serve ` — it runs on Lemonade's Vulkan backend instead." + Run `rocm examine` to see what this host itself supports." ) } @@ -7749,7 +7760,7 @@ mod tests { let message = unsupported_requested_target_message("gfx90c", "--family"); assert!(message.contains("`gfx90c`"), "{message}"); assert!(message.contains("--family"), "{message}"); - assert!(message.contains("rocm serve "), "{message}"); + assert!(message.contains("rocm examine"), "{message}"); // The user named it; nothing was detected, so the error must not say so. assert!(!message.contains("detected"), "{message}"); @@ -7757,6 +7768,46 @@ mod tests { assert!(message.contains("ROCM_CLI_THEROCK_FAMILY"), "{message}"); } + /// The requested-target refusal must not promise which backend `rocm serve` + /// will use, because it never looked at this host. + /// + /// Both refusals used to close with the same "`rocm serve ` runs on + /// Lemonade's Vulkan backend" sentence. In `unsupported_host_gpu_message` + /// that is a fact: it is only reached after `detect_host_gfx_target` returns + /// a target `gfx_target_has_no_rocm_build` confirms. This one fires from the + /// `--family` / `ROCM_CLI_THEROCK_FAMILY` branches before any probe runs — + /// see `cli_family_overrides_a_stale_env_value_naming_an_unsupported_gpu` + /// for an override that deliberately disagrees with its host — so on a + /// ROCm-capable machine it told the user Vulkan when serving would really + /// run on ROCm/HIP. It now points at `rocm examine` instead, which is true + /// on every host and still leaves somewhere to go. + #[test] + fn requested_target_error_does_not_claim_which_backend_serving_will_use() { + for source in ["--family", "ROCM_CLI_THEROCK_FAMILY"] { + let message = unsupported_requested_target_message("gfx90c", source); + assert!( + !message.contains("rocm serve"), + "the requested target says nothing about this host's GPU, so the \ + refusal must not predict what `rocm serve` would run on: {message}" + ); + assert!( + !message.contains("Lemonade"), + "naming the engine makes the same unchecked promise about this \ + host as naming the command: {message}" + ); + assert!( + message.contains("rocm examine"), + "dropping the claim must still leave the user a next step: {message}" + ); + } + + // The sibling path did look at this host, so it keeps the claim: this is + // the asymmetry, not a blanket ban on the sentence. + let detected = unsupported_host_gpu_message("gfx90c"); + assert!(detected.contains("rocm serve "), "{detected}"); + assert!(detected.contains("Vulkan"), "{detected}"); + } + #[test] fn release_channel_prefers_stable_versions() { let versions = vec![ diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index 5256e09188..b96ef8681b 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -1821,16 +1821,25 @@ impl ExamineSummary { /// it (for example the gfx90c Vega iGPU in Renoir / Cezanne APUs). Without /// it `compatible_therock_family: ` reads like a probe failure, when /// the GPU was in fact recognized and simply has no ROCm build. + /// + /// Classified by [`gpu_support_level`] rather than re-deriving the same + /// `(detected, covering family)` test here: this line and that function are + /// two renderings of one judgement, and the whole point of this change is + /// that support claims come from a single place. Every variant is spelled + /// out so that adding one is a compile error here rather than a silently + /// blank line. fn rocm_support_line(&self) -> String { - match ( - self.detected_gfx_target.as_deref(), - self.compatible_therock_family.as_deref(), - ) { - (Some(gfx), None) if gfx_target_has_no_rocm_build(gfx) => format!( - " rocm_support: GPU detected ({gfx}) but not supported by ROCm; \ - the Vulkan backend can still be used\n" + let detected = self.detected_gfx_target.as_deref(); + match gpu_support_level(detected, self.compatible_therock_family.as_deref()) { + // `VulkanOnly` is only ever returned for a detected target, so the + // fallback below is unreachable; it keeps the arm total without a + // panic path. + GpuSupportLevel::VulkanOnly => format!( + " rocm_support: GPU detected ({}) but not supported by ROCm; \ + the Vulkan backend can still be used\n", + detected.unwrap_or_default() ), - _ => String::new(), + GpuSupportLevel::RocmHip | GpuSupportLevel::Unknown => String::new(), } } diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 157c01ad7e..ac0650eff6 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -242,8 +242,12 @@ rocm serve qwen --engine lemonade --device gpu_required --foreground --port 1143 Expected result: -- The engine uses ROCm GPU execution. -- The run does not fall back to CPU or Vulkan. +- The engine uses ROCm GPU execution. The deployment summary's `device:` row + reports what the engine actually selected, so check it by reading that row + rather than inferring it from the absence of a fallback warning: expect + `rocm gpu `. `rocm services` shows the same `device` while it runs. +- The run does not fall back to CPU or Vulkan — a `device:` of `vulkan gpu ` + fails this check even though the serve itself succeeds. - If the GPU cannot be used, the command fails with a clear error. - The OpenAI-compatible endpoint answers a simple chat request. diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index a67149b05c..fdb6b9bff1 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -62,6 +62,14 @@ Feature: Model serving # Lemonade serve + inference (GGUF model). Engine coverage: Lemonade. The # lemonade per-PR canary: one real lemonade serve runs on every PR (the # counterpart to the vLLM canary above). + # + # ROCMAI-550's device-reporting Then is listed here for the lanes that run it, + # but it is NOT coverage on lemonade+linux: the whole scenario is xfail'd there + # (EAI-7423 -- the managed serve reaches ready then shuts down ~0.08s later), + # so no step of it is a live assertion on that lane. Moving the Then earlier + # would not change that: the service is gone before a healthcheck could report + # a device. The vLLM canary (serve-05) is what actually exercises device + # reporting on every PR today. @id:serve-lemonade-inference @requires-gpu @requires-engine:lemonade Scenario: serve-07 - A model served on lemonade responds to inference requests Given a managed runtime is active diff --git a/tests/e2e-cucumber/features/runtime_setup.feature b/tests/e2e-cucumber/features/runtime_setup.feature index a2164d1f92..b7c8fa1686 100644 --- a/tests/e2e-cucumber/features/runtime_setup.feature +++ b/tests/e2e-cucumber/features/runtime_setup.feature @@ -412,14 +412,16 @@ Feature: Runtime configuration # of the user. # # ROCMAI-550 added a third Then. Having taken every package family away, the - # refusal now has to leave the user somewhere to go, and it says `rocm serve - # ` on the Vulkan backend. That is a remediation line naming a command, - # so AGENTS.md §3 wants it asserted where the refusal is actually produced - # rather than only against the formatter -- this is the one door of the two - # that a lane can open, so it is asserted here. + # refusal now has to leave the user somewhere to go, and it points at `rocm + # examine`. That is a remediation line naming a command, so AGENTS.md §3 wants + # it asserted where the refusal is actually produced rather than only against + # the formatter -- this is the one door of the two that a lane can open, so it + # is asserted here. It also pins the negative: this branch refuses a requested + # target before probing the host at all, so it must not predict which backend + # serving would use on a runner whose own GPU is ROCm-capable. @id:runtime-install-sdk-refuses-unsupported-gpu-target Scenario: runtime-19 - Asking for a GPU target ROCm does not build is refused outright When the user dry-runs an SDK install for the gfx90c Vega iGPU Then the install is refused as having no ROCm build for that GPU And the refusal offers no package family to install instead - And the refusal points at serving on the Vulkan backend instead + And the refusal points at examining what this host supports diff --git a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs index 8c7378394d..438686cfb0 100644 --- a/tests/e2e-cucumber/tests/e2e/runtime_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/runtime_steps.rs @@ -1458,28 +1458,28 @@ async fn assert_unsupported_family_offers_nothing(world: &mut E2eWorld) { } } -/// Having removed every `--family` the user could try, the refusal has to say -/// what *will* work. ROCMAI-550 added that sentence, and this pins it where the -/// refusal is really produced rather than against the formatter alone. +/// Having removed every `--family` the user could try, the refusal has to leave +/// somewhere to go — and that somewhere must be true on whatever host runs it. /// -/// Only the wording is checked here, because this lane cannot check the rest of -/// it: proving that `rocm serve` then reaches a GPU with no ROCm build needs a -/// Vega APU, and no runner has one (same gap as the detected-host door, above). -/// The two halves the advice rests on are proven elsewhere — -/// `rocm_core::gfx_target_has_no_rocm_build` is what put the user here, and -/// `selects_vulkan_when_rocm_unsupported` in `engines/lemonade`'s -/// `backend_alignment` is what makes llama.cpp pick `vulkan` once `rocm` comes -/// back unsupported, which is the claim the sentence makes. -#[then("the refusal points at serving on the Vulkan backend instead")] -async fn assert_unsupported_family_points_at_vulkan_serving(world: &mut E2eWorld) { +/// This scenario asks for `--family gfx90c`, which is refused by +/// `unsupported_requested_target_message` before any host probe runs, so the +/// runner's own GPU has no bearing on the refusal. That makes this lane the +/// proof that the refusal does not predict a backend: these runners are +/// ROCm-capable, so an earlier draft's "`rocm serve ` runs on Lemonade's +/// Vulkan backend" was being asserted green here on exactly the kind of host +/// where it is false. `rocm examine` holds either way, and is the command that +/// reports the real support level this PR added. +#[then("the refusal points at examining what this host supports")] +async fn assert_unsupported_family_points_at_examine(world: &mut E2eWorld) { let output = unsupported_family_attempt(world); assert!( - output.contains("rocm serve "), - "the refusal took every package family away, so it must name what does \ - work instead:\n{output}" + output.contains("rocm examine"), + "the refusal took every package family away, so it must leave the user a \ + next step:\n{output}" ); assert!( - output.contains("Vulkan"), - "the refusal should say the fallback is the Vulkan backend:\n{output}" + !output.contains("rocm serve"), + "this refusal never looked at the host's GPU, so it must not predict \ + which backend serving would use here:\n{output}" ); } From 9f78b66e48f6a1abc44c82a3694df49638e7cbc5 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 16 Oct 2026 17:44:12 +0300 Subject: [PATCH 05/37] fix(bridge): report lemonade's real backend in the codex-bridge inventory The codex-bridge engine inventory kept the unconditional "with ROCm llama.cpp backend" wording that ROCMAI-550 removed from `rocm examine` and `rocm engines`, so a bridge client still rendered a ROCm claim to the operator on a host that cannot use it. Both call sites can answer the question: the snapshot already gathers the ExamineSummary carrying the two inputs, and the `engines` tool takes the same cheap host probe `rocm engines` uses. Only the `summary` string changes; the field set and the bridge's JSON shape are untouched. The dashboard TUI pane and rocmd's own catalog keep the static wording deliberately -- the pane has no probe behind it, and rocmd answers for whichever host a client asks about, not the one it runs on. Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 87 +++++++++++++++++++++++++++++++------------ 1 file changed, 63 insertions(+), 24 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 63d0177db8..28a064aa52 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -2693,19 +2693,42 @@ fn refresh_startup_update_check_quietly() { fn build_codex_bridge_snapshot(paths: &AppPaths) -> Result { let config = RocmCliConfig::load(paths).unwrap_or_default(); + // Hoisted out of the struct literal so the engine rows can be told what this + // host actually supports. The probe is already paid for: `gpu_support_level` + // takes exactly the two fields this summary already carries, so the engine + // list costs no detection the snapshot was not doing anyway. + let examine = ExamineSummary::gather()?; + let gpu_support_level = rocm_core::gpu_support_level( + examine.detected_gfx_target.as_deref(), + examine.compatible_therock_family.as_deref(), + ); Ok(CodexBridgeSnapshot { protocol: "rocmd-codex-bridge-v0".to_owned(), generated_at_unix_ms: rocm_core::unix_time_millis(), - examine: ExamineSummary::gather()?, + examine, gpu: build_codex_bridge_gpu_snapshot(&config), config, automation_runtime: AutomationRuntimeState::load(paths)?, recent_automation_events: load_recent_automation_events(paths, 32)?, - engines: builtin_codex_bridge_engine_inventory(), + engines: builtin_codex_bridge_engine_inventory(gpu_support_level), services: load_managed_services(paths)?, }) } +/// This host's GPU support level from the cheap host probe, for the call sites +/// that need it without gathering a whole [`ExamineSummary`]. +/// +/// The same two inputs `rocm examine` derives it from, read through +/// `detect_host_gpu_summary` rather than the full examine gather — exactly what +/// `render_engine_inventory_text_with_paths` (i.e. `rocm engines`) already does. +fn host_gpu_support_level(paths: Option<&AppPaths>) -> rocm_core::GpuSupportLevel { + let host_gpu = rocm_core::detect_host_gpu_summary(paths); + rocm_core::gpu_support_level( + host_gpu.gfx_target.as_deref(), + host_gpu.therock_family.as_deref(), + ) +} + fn build_codex_bridge_gpu_snapshot(config: &RocmCliConfig) -> CodexBridgeGpuSnapshot { if !config.telemetry.local_inspection_enabled() { return CodexBridgeGpuSnapshot { @@ -2724,20 +2747,26 @@ fn build_codex_bridge_gpu_snapshot(config: &RocmCliConfig) -> CodexBridgeGpuSnap } } -/// The engine catalogue in the codex-bridge snapshot, taken verbatim from -/// [`builtin_engine_inventory`] — including lemonade's unconditional "with ROCm -/// llama.cpp backend", which ROCMAI-550 replaced in `rocm examine` and -/// `rocm engines` but deliberately left alone here. +/// The engine catalogue in the codex-bridge snapshot and the bridge's `engines` +/// tool, built from [`builtin_engine_inventory`] with lemonade's note replaced +/// by [`lemonade_engine_note`] for the given support level. /// -/// It is left alone because this list is the static catalogue, in the same -/// shape as the unconditional `installed_binary: true` beside it: every field -/// but `default_for_platform` describes the engine, not this host. A consumer -/// that wants the host's real support level reads it off the `examine` block -/// in the same snapshot, which carries `detected_gfx_target` and -/// `compatible_therock_family` — the two inputs `rocm_core::gpu_support_level` -/// takes. Making this row host-aware would change the bridge's JSON contract -/// and is worth its own change, not a rider on this one. -fn builtin_codex_bridge_engine_inventory() -> Vec { +/// It takes the level rather than reading the static row because the stale +/// "with ROCm llama.cpp backend" claim is exactly what ROCMAI-550 exists to +/// remove, and this is a user-facing surface like the other two: a bridge +/// client renders `summary` to the operator. Both call sites can answer the +/// question — `build_codex_bridge_snapshot` already gathers the +/// [`ExamineSummary`] that carries the two inputs, and the `engines` tool takes +/// the same cheap host probe `rocm engines` does — so there was no call site +/// here that genuinely could not see the host. +/// +/// Only `summary` moves. The field set, its names and `installed_binary: true` +/// beside it are untouched, so the bridge's JSON shape is the same; what +/// changes is that one string stops asserting ROCm support on a host that has +/// none, which is the same correction the two CLI surfaces got. +fn builtin_codex_bridge_engine_inventory( + gpu_support_level: rocm_core::GpuSupportLevel, +) -> Vec { let default_engine = default_engine_for_platform(); let binary_path = daemon_binary_path() .ok() @@ -2746,7 +2775,11 @@ fn builtin_codex_bridge_engine_inventory() -> Vec { .iter() .map(|(id, summary)| CodexBridgeEngine { id: (*id).to_owned(), - summary: (*summary).to_owned(), + summary: if *id == "lemonade" { + lemonade_engine_note(gpu_support_level).to_owned() + } else { + (*summary).to_owned() + }, default_for_platform: *id == default_engine, installed_binary: true, binary_path: binary_path.clone(), @@ -2754,13 +2787,18 @@ fn builtin_codex_bridge_engine_inventory() -> Vec { .collect() } -/// The lemonade engine note shown in `rocm examine` and `rocm engines`, adjusted -/// for whether this host's detected GPU can actually use the ROCm llama.cpp -/// backend. Both call sites have a live `GpuSupportLevel` in scope; the places -/// that print this note statically — the dashboard TUI, `rocmd`'s own catalog, -/// and [`builtin_codex_bridge_engine_inventory`]'s copy of -/// [`builtin_engine_inventory`] — keep the unconditional ROCm wording, since -/// they describe the engine in the abstract rather than this host. +/// The lemonade engine note shown in `rocm examine`, `rocm engines` and the +/// codex-bridge engine inventory, adjusted for whether this host's detected GPU +/// can actually use the ROCm llama.cpp backend. +/// +/// All three of those have a live `GpuSupportLevel` in scope. The two places +/// that still print the note statically — the dashboard TUI's engine-manager +/// pane and `rocmd`'s own catalog in `apps/rocmd/src/common.rs` — keep the +/// unconditional ROCm wording, and that is deliberate rather than an oversight: +/// the TUI pane renders a hard-coded descriptive list with no probe behind it, +/// and `rocmd` is the daemon answering for whichever host a client asks about, +/// not necessarily the one it runs on. Both are their own change if they ever +/// need to be host-aware. const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str { match level { rocm_core::GpuSupportLevel::VulkanOnly => { @@ -8082,7 +8120,8 @@ pub(crate) fn run_internal_mcp_call( )) } "engines" => { - let engines = builtin_codex_bridge_engine_inventory(); + let engines = + builtin_codex_bridge_engine_inventory(host_gpu_support_level(Some(paths))); Ok(internal_mcp_tool_success( format!("Found {} engine entries.", engines.len()), serde_json::json!({ "engines": engines }), From 18666d469a02e05b18058c9308779661cbefdba4 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 16 Oct 2026 18:33:29 +0300 Subject: [PATCH 06/37] test(engines): cover the GPU-aware engine inventory on `rocm engines` and the bridge `append_examine_engine_inventory` got tests for both support levels when ROCMAI-550 landed; the other two surfaces carrying the same correction did not. `rocm engines` and the codex-bridge inventory (including its `engines` MCP tool) could have gone back to asserting ROCm support on a Vulkan-only host with nothing red. Both were untestable for the same reason: the branch's input is a live sysfs probe, and no CI runner has a GPU that is detected *and* confirmed to have no ROCm build. So each surface now takes the level the way `append_examine_engine_inventory` already did -- the caller probes once, the renderer is given the answer. `render_engine_inventory_text_for_host` and `internal_mcp_engines_tool_result` are those renderers; neither changes what is printed. The three copies of the lemonade substitution collapse into `engine_inventory_note`, so the surfaces cannot drift. The new tests drive the renderers, not that helper: asserting on it would restate the condition rather than prove a surface prints it. `gpu_support_level_for_host_gpu` reads the two `HostGpuSummary` fields once instead of at each call site. They are same-typed positional arguments, so a transposed call site would have classified a Vulkan-only host as `RocmHip` and still compiled; it now has a test. Also drops the dangling `GFX_TARGETS_WITHOUT_ROCM_SUPPORT` reference from the `GpuSupportLevel` doc comment -- no such constant exists in `diagnose.rs` or anywhere else, so the doc pointed readers at nothing. Signed-off-by: Roman Sirokov --- apps/rocm/src/chat.rs | 96 ++++++++++++-- apps/rocm/src/main.rs | 208 +++++++++++++++++++++++++++---- crates/rocm-core/src/host_gpu.rs | 63 +++++++++- crates/rocm-core/src/lib.rs | 6 +- 4 files changed, 333 insertions(+), 40 deletions(-) diff --git a/apps/rocm/src/chat.rs b/apps/rocm/src/chat.rs index 7d68a46655..4cb251e093 100644 --- a/apps/rocm/src/chat.rs +++ b/apps/rocm/src/chat.rs @@ -3330,14 +3330,9 @@ pub(crate) fn run_internal_mcp_call( serde_json::json!(gpu), )) } - "engines" => { - let engines = - builtin_codex_bridge_engine_inventory(host_gpu_support_level(Some(paths))); - Ok(internal_mcp_tool_success( - format!("Found {} engine entries.", engines.len()), - serde_json::json!({ "engines": engines }), - )) - } + "engines" => Ok(internal_mcp_engines_tool_result(host_gpu_support_level( + Some(paths), + ))), "services" => { let services = load_managed_services(paths)?; Ok(internal_mcp_tool_success( @@ -3530,6 +3525,24 @@ pub(crate) fn run_internal_mcp_call( } } +/// The `engines` MCP tool result for a host at `gpu_support_level`. +/// +/// Split from the match arm above only so the GPU-aware branch is reachable +/// from a test: the arm's own argument is a live host probe, and no CI runner +/// has a GPU that is detected *and* confirmed to have no ROCm build, so the +/// Vulkan-only wording a bridge client would render could not otherwise be +/// asserted on. Nothing else moved — the summary text, the `engines` key and +/// the entries themselves are what the arm built inline. +fn internal_mcp_engines_tool_result( + gpu_support_level: rocm_core::GpuSupportLevel, +) -> serde_json::Value { + let engines = builtin_codex_bridge_engine_inventory(gpu_support_level); + internal_mcp_tool_success( + format!("Found {} engine entries.", engines.len()), + serde_json::json!({ "engines": engines }), + ) +} + fn internal_mcp_arguments(value: serde_json::Value) -> serde_json::Map { if let Some(arguments) = value .get("arguments") @@ -5982,6 +5995,73 @@ model recipes assert!(mcp_tool_result_text(&result).contains(env!("CARGO_PKG_VERSION"))); } + /// The bridge's `engines` tool carries the same claim `rocm engines` and + /// `rocm examine` carry, so it gets the same pair of tests. These assert on + /// the serialized payload because `structuredContent.engines[].summary` is + /// the exact string an MCP client renders to the operator. + #[test] + fn engines_mcp_tool_names_vulkan_for_a_gpu_with_no_rocm_support() { + let result = internal_mcp_engines_tool_result(rocm_core::GpuSupportLevel::VulkanOnly); + + let summary = engines_tool_lemonade_summary(&result); + assert!( + summary.contains("no ROCm/HIP support") && summary.contains("Vulkan llama.cpp backend"), + "the MCP client must be handed the real backend: {summary}" + ); + assert!( + !summary.contains("with ROCm llama.cpp backend"), + "the stale ROCm claim must not reach an MCP client: {summary}" + ); + assert!(mcp_tool_result_text(&result).contains("engine entries")); + } + + #[test] + fn engines_mcp_tool_keeps_rocm_note_for_a_rocm_capable_gpu() { + let result = internal_mcp_engines_tool_result(rocm_core::GpuSupportLevel::RocmHip); + + assert_eq!( + engines_tool_lemonade_summary(&result), + "default embedded Lemonade server with ROCm llama.cpp backend" + ); + } + + /// Pins the wiring the two tests above cannot see: that the tool reaches + /// the host-aware inventory through the live probe rather than a level + /// nailed to a constant. A hard-coded level that disagrees with this host + /// fails here. + #[test] + fn engines_mcp_tool_answers_for_the_host_it_runs_on() { + let (_root, paths) = test_paths("mcp-engines-host-probe"); + + let result = run_internal_mcp_call(&paths, "engines", serde_json::json!({}), false) + .expect("the engines mcp tool is read-only and should run"); + + assert_eq!( + result.get("isError").and_then(serde_json::Value::as_bool), + Some(false) + ); + assert_eq!( + result.get("structuredContent"), + internal_mcp_engines_tool_result(host_gpu_support_level(Some(&paths))) + .get("structuredContent"), + "the `engines` tool must report this host's support level" + ); + } + + /// The lemonade `summary` as an MCP client reads it out of the tool result. + fn engines_tool_lemonade_summary(result: &serde_json::Value) -> String { + result + .pointer("/structuredContent/engines") + .and_then(serde_json::Value::as_array) + .expect("the engines tool must return an `engines` array") + .iter() + .find(|engine| engine.get("id").and_then(serde_json::Value::as_str) == Some("lemonade")) + .and_then(|engine| engine.get("summary")) + .and_then(serde_json::Value::as_str) + .expect("the engines tool must carry a lemonade summary") + .to_owned() + } + #[test] fn natural_language_plan_returns_structured_mutating_action() { let (_root, paths) = test_paths("nl-plan-structured-mutating"); diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 6fc7f24986..6ca53d6376 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -2736,11 +2736,7 @@ fn build_codex_bridge_snapshot(paths: &AppPaths) -> Result /// `detect_host_gpu_summary` rather than the full examine gather — exactly what /// `render_engine_inventory_text_with_paths` (i.e. `rocm engines`) already does. fn host_gpu_support_level(paths: Option<&AppPaths>) -> rocm_core::GpuSupportLevel { - let host_gpu = rocm_core::detect_host_gpu_summary(paths); - rocm_core::gpu_support_level( - host_gpu.gfx_target.as_deref(), - host_gpu.therock_family.as_deref(), - ) + rocm_core::gpu_support_level_for_host_gpu(&rocm_core::detect_host_gpu_summary(paths)) } fn build_codex_bridge_gpu_snapshot(config: &RocmCliConfig) -> CodexBridgeGpuSnapshot { @@ -2789,11 +2785,7 @@ fn builtin_codex_bridge_engine_inventory( .iter() .map(|(id, summary)| CodexBridgeEngine { id: (*id).to_owned(), - summary: if *id == "lemonade" { - lemonade_engine_note(gpu_support_level).to_owned() - } else { - (*summary).to_owned() - }, + summary: engine_inventory_note(id, summary, gpu_support_level).to_owned(), default_for_platform: *id == default_engine, installed_binary: true, binary_path: binary_path.clone(), @@ -2801,6 +2793,29 @@ fn builtin_codex_bridge_engine_inventory( .collect() } +/// The inventory note to print for `engine`, given the row `catalog_note` the +/// static catalogue carries for it and what this host actually supports. +/// +/// The one place the lemonade substitution lives. All three host-aware +/// inventory renderers — [`render_engine_inventory_text_for_host`] +/// (`rocm engines`), [`append_examine_engine_inventory`] (`rocm examine`) and +/// [`builtin_codex_bridge_engine_inventory`] (the codex bridge and its +/// `engines` MCP tool) — route their note through this, so the three cannot +/// drift into disagreeing about the same host. Tests exercise the renderers +/// rather than this function: asserting on it directly would only restate the +/// condition below instead of proving a surface prints it. +fn engine_inventory_note( + engine: &str, + catalog_note: &'static str, + gpu_support_level: rocm_core::GpuSupportLevel, +) -> &'static str { + if engine == "lemonade" { + lemonade_engine_note(gpu_support_level) + } else { + catalog_note + } +} + /// The lemonade engine note shown in `rocm examine`, `rocm engines` and the /// codex-bridge engine inventory, adjusted for whether this host's detected GPU /// can actually use the ROCm llama.cpp backend. @@ -5146,11 +5161,27 @@ fn render_engine_inventory_text_with_paths(paths: Option<&AppPaths>) -> String { // Mark the engine this GPU actually serves on as primary. Using the platform // constant put the `*` on Lemonade even on Instinct, where `serve` picks vLLM. let host_gpu = rocm_core::detect_host_gpu_summary(paths); - let host_default_engine = rocm_core::default_engine_for_host(&host_gpu); - let gpu_support_level = rocm_core::gpu_support_level( - host_gpu.gfx_target.as_deref(), - host_gpu.therock_family.as_deref(), - ); + render_engine_inventory_text_for_host( + paths, + rocm_core::default_engine_for_host(&host_gpu), + rocm_core::gpu_support_level_for_host_gpu(&host_gpu), + ) +} + +/// `rocm engines` for a host whose preferred engine and GPU support level the +/// caller has already probed. +/// +/// Split from [`render_engine_inventory_text_with_paths`] for the same reason +/// [`append_examine_engine_inventory`] takes both as parameters: the host probe +/// reads real sysfs, so a test cannot otherwise reach the Vulkan-only branch — +/// no CI runner has a GPU that is detected *and* confirmed to have no ROCm +/// build. Both values come from one `detect_host_gpu_summary` call above, so +/// the marker and the note still cannot disagree about the same host. +fn render_engine_inventory_text_for_host( + paths: Option<&AppPaths>, + host_default_engine: &str, + gpu_support_level: rocm_core::GpuSupportLevel, +) -> String { // A configured `default_engine` still wins over the host preference, mirroring // `select_serve_engine` (and `append_examine_engine_inventory`). Without this, // the legend could mark an engine `serve` will not actually default to. @@ -5197,11 +5228,7 @@ fn render_engine_inventory_text_with_paths(paths: Option<&AppPaths>) -> String { } else { " " }; - let note = if *name == "lemonade" { - lemonade_engine_note(gpu_support_level) - } else { - *note - }; + let note = engine_inventory_note(name, note, gpu_support_level); let _ = writeln!(output, "{marker} {name:10} {note}"); append_engine_detect_summary(&mut output, name, paths); } @@ -5413,11 +5440,7 @@ fn append_examine_engine_inventory( .or(entry.last_installed_env_id.as_deref()) }) .unwrap_or(""); - let note = if *engine == "lemonade" { - lemonade_engine_note(gpu_support_level) - } else { - *note - }; + let note = engine_inventory_note(engine, note, gpu_support_level); let _ = writeln!( output, " {marker} {engine} adapter={adapter} runtime_pref={runtime_pref} note={note}" @@ -20455,6 +20478,141 @@ pub(crate) mod tests { let _ = fs::remove_dir_all(root); } + /// `rocm engines` makes the same claim `rocm examine` does, through a + /// different renderer, so it needs the same pair of tests: + /// `examine_engine_inventory_names_vulkan_for_a_gpu_with_no_rocm_support` + /// is this test's counterpart on the examine surface. + #[test] + fn engines_inventory_names_vulkan_for_a_gpu_with_no_rocm_support() { + let (root, paths) = test_paths("engines-inventory-vulkan-only"); + + let output = render_engine_inventory_text_for_host( + Some(&paths), + "lemonade", + rocm_core::GpuSupportLevel::VulkanOnly, + ); + + assert!( + output.contains("no ROCm/HIP support") && output.contains("Vulkan llama.cpp backend"), + "`rocm engines` must name the backend this GPU really runs on:\n{output}" + ); + assert!( + !output.contains("Lemonade server with ROCm llama.cpp backend"), + "`rocm engines` must not keep asserting ROCm support on a host without it:\n{output}" + ); + // The substitution is lemonade's alone; vLLM's row is not host-aware. + assert!( + output.contains("Linux/WSL ROCm GPU serving engine through external vLLM"), + "only lemonade's note is support-level aware:\n{output}" + ); + let _ = fs::remove_dir_all(root); + } + + #[test] + fn engines_inventory_keeps_rocm_note_for_a_rocm_capable_gpu() { + let (root, paths) = test_paths("engines-inventory-rocm-capable"); + + let output = render_engine_inventory_text_for_host( + Some(&paths), + "vllm", + rocm_core::GpuSupportLevel::RocmHip, + ); + + assert!( + output.contains("default embedded Lemonade server with ROCm llama.cpp backend"), + "a ROCm-capable host must keep the ROCm wording:\n{output}" + ); + assert!( + !output.contains("Vulkan llama.cpp backend"), + "a ROCm-capable host must not be told it is on Vulkan:\n{output}" + ); + let _ = fs::remove_dir_all(root); + } + + /// An unrecognized or undetected GPU is a gap in what the catalog knows, + /// not a confirmed absence of ROCm — printing the Vulkan wording for it + /// would turn that gap into a false claim about the hardware, which is the + /// distinction `GpuSupportLevel::Unknown` exists to preserve. + #[test] + fn engines_inventory_does_not_claim_vulkan_for_an_unrecognized_gpu() { + let (root, paths) = test_paths("engines-inventory-unknown"); + + let output = render_engine_inventory_text_for_host( + Some(&paths), + "lemonade", + rocm_core::GpuSupportLevel::Unknown, + ); + + assert!( + !output.contains("no ROCm/HIP support"), + "an unrecognized GPU must not be reported as confirmed ROCm-less:\n{output}" + ); + let _ = fs::remove_dir_all(root); + } + + /// The codex bridge renders `summary` to the operator just as the two CLI + /// surfaces render their note, so the same correction has to reach it. + #[test] + fn codex_bridge_engine_inventory_names_vulkan_for_a_gpu_with_no_rocm_support() { + let engines = builtin_codex_bridge_engine_inventory(rocm_core::GpuSupportLevel::VulkanOnly); + + let lemonade = engines + .iter() + .find(|engine| engine.id == "lemonade") + .expect("the bridge inventory must still carry lemonade"); + assert!( + lemonade.summary.contains("no ROCm/HIP support") + && lemonade.summary.contains("Vulkan llama.cpp backend"), + "bridge clients must be told the real backend: {}", + lemonade.summary + ); + assert!( + !lemonade.summary.contains("with ROCm llama.cpp backend"), + "the stale ROCm claim must not survive in the bridge: {}", + lemonade.summary + ); + // Only `summary` is support-level aware; the bridge's JSON shape and + // every other engine's row are untouched by the substitution. + assert!(lemonade.installed_binary); + let vllm = engines + .iter() + .find(|engine| engine.id == "vllm") + .expect("the bridge inventory must still carry vllm"); + assert_eq!( + vllm.summary, + "Linux/WSL ROCm GPU serving engine through external vLLM" + ); + } + + #[test] + fn codex_bridge_engine_inventory_keeps_rocm_note_for_a_rocm_capable_gpu() { + let engines = builtin_codex_bridge_engine_inventory(rocm_core::GpuSupportLevel::RocmHip); + + let lemonade = engines + .iter() + .find(|engine| engine.id == "lemonade") + .expect("the bridge inventory must still carry lemonade"); + assert_eq!( + lemonade.summary, + "default embedded Lemonade server with ROCm llama.cpp backend" + ); + } + + #[test] + fn codex_bridge_engine_inventory_does_not_claim_vulkan_for_an_unrecognized_gpu() { + let engines = builtin_codex_bridge_engine_inventory(rocm_core::GpuSupportLevel::Unknown); + + let lemonade = engines + .iter() + .find(|engine| engine.id == "lemonade") + .expect("the bridge inventory must still carry lemonade"); + assert!( + !lemonade.summary.contains("no ROCm/HIP support"), + "an unrecognized GPU must not be reported as confirmed ROCm-less: {}", + lemonade.summary + ); + } + #[test] fn examine_is_silent_when_the_configured_engine_agrees_with_the_host() { let (root, paths) = test_paths("examine-engine-inventory-agrees"); diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index e46e442774..bd9e17a388 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -133,10 +133,12 @@ pub fn gfx_target_has_no_rocm_build(gfx: &str) -> bool { /// /// `Unknown` deliberately covers both "nothing detected" and "a gfx target we /// don't recognize" rather than being called `Unsupported` — see the doc -/// comment on [`gfx_target_has_no_rocm_build`] and -/// `GFX_TARGETS_WITHOUT_ROCM_SUPPORT` in `diagnose.rs` for why conflating -/// "confirmed no ROCm build" with "unrecognized" would turn a detection gap -/// into a false claim about the hardware. +/// comment on [`gfx_target_has_no_rocm_build`] for why conflating "confirmed +/// no ROCm build" with "unrecognized" would turn a detection gap into a false +/// claim about the hardware. Only that predicate's conjunction with +/// [`normalize_therock_family`] establishes the "confirmed" half; an +/// unrecognized target satisfies neither conjunct, so it is a gap in what this +/// catalog knows rather than a fact about the card. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum GpuSupportLevel { /// A TheRock package family covers this GPU: full ROCm/HIP support. @@ -162,6 +164,21 @@ pub fn gpu_support_level( } } +/// [`gpu_support_level`] for a [`HostGpuSummary`] the caller already probed. +/// +/// The two fields are read here, once, rather than at each call site: they are +/// positional arguments of the same type on [`gpu_support_level`], so a call +/// site that transposed them would classify a Vulkan-only host as `RocmHip` +/// and still compile. `rocm engines` and the codex-bridge `engines` tool both +/// derive the level from this struct, so both go through this. +#[must_use] +pub fn gpu_support_level_for_host_gpu(host_gpu: &HostGpuSummary) -> GpuSupportLevel { + gpu_support_level( + host_gpu.gfx_target.as_deref(), + host_gpu.therock_family.as_deref(), + ) +} + pub fn normalize_therock_family(value: &str) -> Option { let normalized = value.trim().to_ascii_lowercase(); if normalized.is_empty() { @@ -3548,6 +3565,44 @@ mod tests { ); } + /// The two `HostGpuSummary` fields must reach `gpu_support_level` in the + /// right positions. Both are `Option`, so transposing them still + /// compiles — and it would answer `RocmHip` for exactly the Vulkan-only + /// host this classification exists to stop lying to. + #[test] + fn gpu_support_level_for_host_gpu_reads_a_vulkan_only_summary_as_vulkan_only() { + let host_gpu = HostGpuSummary { + name: Some("AMD Radeon Graphics".to_owned()), + gfx_target: Some("gfx90c".to_owned()), + therock_family: None, + }; + assert_eq!( + gpu_support_level_for_host_gpu(&host_gpu), + GpuSupportLevel::VulkanOnly + ); + } + + #[test] + fn gpu_support_level_for_host_gpu_reads_a_covered_summary_as_rocm_hip() { + let host_gpu = HostGpuSummary { + name: Some("AMD Instinct MI300X".to_owned()), + gfx_target: Some("gfx942".to_owned()), + therock_family: Some("gfx94X-dcgpu".to_owned()), + }; + assert_eq!( + gpu_support_level_for_host_gpu(&host_gpu), + GpuSupportLevel::RocmHip + ); + } + + #[test] + fn gpu_support_level_for_host_gpu_reads_an_undetected_summary_as_unknown() { + assert_eq!( + gpu_support_level_for_host_gpu(&HostGpuSummary::default()), + GpuSupportLevel::Unknown + ); + } + #[test] fn gpu_support_level_is_unknown_without_detection_or_for_an_unrecognized_target() { assert_eq!(gpu_support_level(None, None), GpuSupportLevel::Unknown); diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 64255f6c1c..569b09fc95 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -88,9 +88,9 @@ pub use host_gpu::{ default_engine_for_host, default_engine_for_platform, detect_gpu_driver_version, detect_host_gfx_target, detect_host_gpu_diagnostics, detect_host_gpu_summary, detect_system_ram_gib, extract_first_gfx_token, gfx_target_has_no_rocm_build, - gpu_support_level, has_usable_amd_gpu, interactive_terminal, is_amdgpu_device, is_wsl_host, - known_therock_families, normalize_therock_family, preferred_serve_engine_for_host_gpu_summary, - usable_amd_gpu_indices, + gpu_support_level, gpu_support_level_for_host_gpu, has_usable_amd_gpu, interactive_terminal, + is_amdgpu_device, is_wsl_host, known_therock_families, normalize_therock_family, + preferred_serve_engine_for_host_gpu_summary, usable_amd_gpu_indices, }; #[cfg(any(target_os = "linux", test))] use host_gpu::{KfdGpuNode, kfd_gpu_nodes_in}; From a9ed8fa4e49b3923d6bc847d7aa8277560ed26bc Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Mon, 2 Nov 2026 15:51:15 +0200 Subject: [PATCH 07/37] ROCMAI-550: make the device row name a device and examine answer every tier Review round 3 on #678. - `rocm examine` now prints a `rocm_support` line for RocmHip and Unknown as well as VulkanOnly. The Unknown-tier lemonade note sends the user to `rocm examine`, and examine printed nothing for exactly those hosts, so the remediation named a command that could not clear the condition. The note is also reworded to promise the support level, which is what examine delivers, rather than the backend, which only a launch can settle. - The `device:` row carries the GPU name. ROCMAI-550 asks for the backend "and the device name"; on the ticket's own Windows gfx90c host no ordinal is recorded, so the row rendered a bare `vulkan gpu` - a backend and a noun naming nothing. All three engines now record `gpu_name` at launch and format through the shared `rocm_engine_protocol::format_reported_device`, which drops the `gpu` noun when there is no ordinal behind it. - `fetch_service_devices` splits its budget loop into `collect_service_devices` with an injected clock and probe, so the budget-exhausted exit is tested - including that later services are not probed at all. The ~18s worst case (budget plus one in-flight probe) is documented on `SERVICE_DEVICE_PROBE_BUDGET`, on `render_services_text` and in README. - The e2e `device:` step polls for the row instead of panicking on the first look: lemonade reports `unknown` until its own inference probe returns, so a correct implementation legitimately has no row for a short window. - AIM's `rocm gpu` change is pinned by a test that drives the real write-state/read-back/report path. Signed-off-by: Roman Sirokov --- README.md | 10 +- apps/rocm/src/engine_ipc.rs | 146 +++++++++++++++++- apps/rocm/src/main.rs | 11 +- apps/rocm/src/services_text.rs | 17 ++ crates/rocm-core/src/host_gpu.rs | 121 +++++++++++++-- crates/rocm-engine-protocol/src/lib.rs | 94 +++++++++++ docs/manual-testing.md | 17 +- engines/aim/src/state.rs | 105 +++++++++++-- engines/lemonade/src/state.rs | 66 ++++++-- engines/vllm/src/state.rs | 49 ++++-- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 92 +++++++---- 11 files changed, 640 insertions(+), 88 deletions(-) diff --git a/README.md b/README.md index d40c138971..8689fdfcc3 100644 --- a/README.md +++ b/README.md @@ -865,7 +865,8 @@ services stop --yes`) or match the original flags. By default the server runs in the background under ROCm CLI's supervision and prints a deployment summary: a progress indicator while it starts, then a table with the status, the engine and the backend device it actually selected (for -example `rocm gpu 0` or `vulkan gpu 0`, read back from the running engine), the +example `rocm gpu 0 (AMD Instinct MI300X)`, or `vulkan (AMD Radeon Graphics)` +where the engine pinned no GPU ordinal — read back from the running engine), the full inference endpoint, the API-qualified model name, and a quick smoke test (time to first token and approximate tokens per second). `rocm services` reports the same `device` for each running service. Control @@ -921,6 +922,13 @@ rocm services remove --yes rocm services prune [--older-than-hours | --any-age] [--dry-run] [--yes] ``` +`list` asks each running server which backend and GPU it is actually on, so it +can take a few seconds rather than returning instantly. The query is capped in +total — about 18 seconds worst case, however many servers are listed — and a +server that does not answer in time simply has no `device` line rather than +holding up the rest of the list. The same applies to asking about servers in +`rocm chat`, which prints the same list. + `restart` starts a server again only once its stop has confirmed that every recorded process is gone; if the stop cannot confirm that, it says so, starts nothing, and leaves the stop request recorded. diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index efa1d70d4b..94ce6c719c 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -342,6 +342,23 @@ pub(crate) fn fetch_service_device( /// couple of healthy engines answer (`engines/lemonade`'s endpoint probe is /// the slowest at `rocm_core::INFERENCE_PROBE_TIMEOUT`, 8s worst case, and /// only on a miss) while still bounding the whole render. +/// +/// **Worst-case wall time for `rocm services list` is this budget plus one +/// in-flight probe, i.e. ~18s** with the built-in engines, not 10s. The budget +/// is a *start* gate: it is tested before each probe, never inside one, so a +/// probe begun at 9.99s still runs to its own timeout. The arithmetic is 10s +/// of elapsed budget plus Lemonade's 8s `INFERENCE_PROBE_TIMEOUT`. That bound +/// applies equally to the chat `services` tool, which renders through the same +/// [`crate::services_text::render_services_text`]. +/// +/// Why not a per-probe deadline that would hold the total at 10s: the probe is +/// `engine_request`, which for a built-in engine is a synchronous in-process +/// call and for an external plugin is a `wait()` on a spawned child. Neither +/// can be cancelled from here without either a thread per probe (ruled out by +/// the process-wide env/mutex scoping `fetch_service_devices` documents) or a +/// kill path for a child mid-RPC. Both are larger changes than the row is +/// worth; the start gate is what keeps the cost flat in the number of +/// services, which is the growth that actually matters. pub(crate) const SERVICE_DEVICE_PROBE_BUDGET: Duration = Duration::from_secs(10); /// Devices for several services at once, keyed by service id, under one shared @@ -361,7 +378,9 @@ pub(crate) const SERVICE_DEVICE_PROBE_BUDGET: Duration = Duration::from_secs(10) /// The budget is checked between probes, not inside one: a single external /// plugin that never exits still blocks, because `engine_request` waits on its /// child unbounded. That is the pre-existing behavior of every engine call -/// site in the CLI and is not made worse here. +/// site in the CLI and is not made worse here. See +/// [`SERVICE_DEVICE_PROBE_BUDGET`] for what that makes the worst-case wall +/// time of a `rocm services list`. pub(crate) fn fetch_service_devices<'a, I>( paths: &AppPaths, services: I, @@ -370,12 +389,40 @@ where I: IntoIterator, { let started = Instant::now(); + collect_service_devices( + services, + || started.elapsed(), + |service_id, engine| fetch_service_device(paths, engine, service_id), + ) +} + +/// The budget loop of [`fetch_service_devices`], with the clock and the probe +/// injected. +/// +/// Split out so the budget-exhausted exit has a test. In the real function the +/// clock is `Instant::elapsed` and the probe is a live engine RPC, so covering +/// that branch would otherwise mean making a test wait out +/// [`SERVICE_DEVICE_PROBE_BUDGET`] against real engines — which is both slow +/// and exactly the timing-dependent shape that flakes. What the branch has to +/// guarantee is not just that later services lack a `device:` row but that +/// they were never *probed*, since the whole point is to stop paying; only an +/// injected probe can observe that. +fn collect_service_devices<'a, I, C, P>( + services: I, + mut elapsed: C, + mut probe: P, +) -> std::collections::HashMap +where + I: IntoIterator, + C: FnMut() -> Duration, + P: FnMut(&str, &str) -> Option, +{ let mut devices = std::collections::HashMap::new(); for (service_id, engine) in services { - if started.elapsed() >= SERVICE_DEVICE_PROBE_BUDGET { + if elapsed() >= SERVICE_DEVICE_PROBE_BUDGET { break; } - if let Some(device) = fetch_service_device(paths, engine, service_id) { + if let Some(device) = probe(service_id, engine) { devices.insert(service_id.to_owned(), device); } } @@ -601,6 +648,99 @@ mod tests { use super::*; use crate::tests::test_paths; + /// Every service inside the budget is probed and reported. + #[test] + fn service_device_probes_run_while_the_budget_holds() { + let mut probed = Vec::new(); + let devices = collect_service_devices( + [("svc-a", "lemonade"), ("svc-b", "vllm")], + || Duration::ZERO, + |service_id, engine| { + probed.push(service_id.to_owned()); + Some(format!("{engine}-device")) + }, + ); + + assert_eq!(probed, ["svc-a", "svc-b"]); + assert_eq!( + devices.get("svc-a").map(String::as_str), + Some("lemonade-device") + ); + assert_eq!( + devices.get("svc-b").map(String::as_str), + Some("vllm-device") + ); + } + + /// Once [`SERVICE_DEVICE_PROBE_BUDGET`] is spent the remaining services are + /// *not probed at all*, and what was already learned is still rendered. + /// + /// This is the branch that keeps `rocm services list` — and the chat + /// `services` tool, which renders through the same + /// `render_services_text` — from scaling its wall time with the number of + /// services. Dropping the later rows is only half of it; the point is to + /// stop paying for them, so the probe counter below is the real assertion. + /// Asserting it against the live clock and live engines would mean a test + /// that waits out ten real seconds, which is why the loop takes both + /// injected. + #[test] + fn service_device_probes_stop_once_the_budget_is_exhausted() { + let mut probed = Vec::new(); + // Inside the budget for the first service, over it from then on. + let mut tick = 0; + let devices = collect_service_devices( + [ + ("svc-fast", "lemonade"), + ("svc-hung", "vllm"), + ("svc-never-reached", "aim"), + ], + || { + tick += 1; + if tick <= 1 { + Duration::ZERO + } else { + SERVICE_DEVICE_PROBE_BUDGET + } + }, + |service_id, _engine| { + probed.push(service_id.to_owned()); + Some("rocm gpu 0".to_owned()) + }, + ); + + assert_eq!( + probed, + ["svc-fast"], + "the budget must stop further probing, not merely hide its results" + ); + assert_eq!( + devices.get("svc-fast").map(String::as_str), + Some("rocm gpu 0"), + "what was learned before the budget ran out is still reported" + ); + assert!( + !devices.contains_key("svc-hung") && !devices.contains_key("svc-never-reached"), + "an unprobed service must get no `device:` row: {devices:?}" + ); + } + + /// A service whose engine cannot answer costs its own probe and nothing + /// more: it is skipped, and the services after it are still probed. + #[test] + fn a_service_whose_probe_fails_does_not_suppress_the_rest() { + let devices = collect_service_devices( + [("svc-quiet", "lemonade"), ("svc-ok", "vllm")], + || Duration::ZERO, + |service_id, _engine| (service_id == "svc-ok").then(|| "rocm gpu 1".to_owned()), + ); + + assert!(!devices.contains_key("svc-quiet"), "{devices:?}"); + assert_eq!( + devices.get("svc-ok").map(String::as_str), + Some("rocm gpu 1") + ); + } + #[test] fn engine_plugin_discovery_finds_runtime_binary() -> Result<()> { let (root, paths) = test_paths("engine-plugin"); diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 30d136d325..d5d2e64ed5 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -2745,6 +2745,15 @@ fn engine_inventory_note( /// claims no backend and names the command that can answer. A new /// `GpuSupportLevel` variant now has to be classified here rather than /// silently inheriting the ROCm claim. +/// +/// What the `Unknown` arm promises is bounded by what the named command +/// delivers. It used to say `rocm examine` would show "which llama.cpp backend +/// this GPU uses", and examine showed no such thing — it printed a +/// `rocm_support` line only for `VulkanOnly`, so on exactly the hosts that got +/// this note it printed nothing at all. `ExamineSummary::rocm_support_line` +/// now answers for every level, and this says what that answer is: the support +/// level, not the backend. The backend is settled by launching one, which that +/// line in turn points at. const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str { match level { rocm_core::GpuSupportLevel::VulkanOnly => { @@ -2754,7 +2763,7 @@ const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str "default embedded Lemonade server with ROCm llama.cpp backend" } rocm_core::GpuSupportLevel::Unknown => { - "default embedded Lemonade server; run `rocm examine` to see which llama.cpp backend this GPU uses" + "default embedded Lemonade server; run `rocm examine` for this host's GPU support level, which says how to find the backend" } } } diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index 97bf5f1493..8424c0a045 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -382,6 +382,23 @@ fn write_past_attempts_hint(output: &mut String, past_attempts: usize, newest_id let _ = writeln!(output, " Reclaim the space: rocm services prune"); } +/// `rocm services list`, and the chat `services` tool, which renders the same +/// text. +/// +/// **This is no longer a pure local read.** Since ROCMAI-550 it makes one live +/// `Healthcheck` RPC per *ready* service to fill the `device:` row, capped in +/// aggregate by `engine_ipc::SERVICE_DEVICE_PROBE_BUDGET`. Worst case is that +/// budget plus one in-flight probe — about 18s with the built-in engines, not +/// 10s, because the budget gates the *start* of each probe and never +/// interrupts one. That doc comment carries the arithmetic and why a +/// per-probe deadline was not taken. +/// +/// The chat tool pays the same bound deliberately rather than rendering a +/// device-less view: the two surfaces would then disagree about one service +/// while claiming to show the same list, and "which GPU is this running on" is +/// a question asked in chat at least as often as at the prompt. The cost is +/// bounded and flat in the number of services, which is what makes it +/// acceptable there. pub(crate) fn render_services_text(paths: &AppPaths, all: bool) -> Result { // Counts describe the WHOLE registry, not the rows this view keeps. The // default view hides records that are not live, and counting the filtered diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index ce5accd162..1a2eab0f4a 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -1939,12 +1939,22 @@ impl ExamineSummary { }) } - /// The `rocm_support` line of the examine text, or an empty string. + /// The `rocm_support` line of the examine text. /// - /// Present only when a GPU was detected but no ROCm package family covers - /// it (for example the gfx90c Vega iGPU in Renoir / Cezanne APUs). Without - /// it `compatible_therock_family: ` reads like a probe failure, when - /// the GPU was in fact recognized and simply has no ROCm build. + /// ROCMAI-550 asks `rocm examine` to "report the support level with one + /// line of what the user can do", so every level gets a line — including + /// the ones where the answer is "this build cannot tell". The earlier draft + /// printed only the `VulkanOnly` line, which made this report unable to + /// answer the very question the other surfaces send people here to ask: + /// `lemonade_engine_note` in `apps/rocm/src/main.rs` tells an `Unknown` + /// host to run `rocm examine`, and silence there is not an answer. + /// + /// Without the `VulkanOnly` line specifically, + /// `compatible_therock_family: ` reads like a probe failure when + /// the GPU was in fact recognized and simply has no ROCm build (the gfx90c + /// Vega iGPU in Renoir / Cezanne APUs). The `Unknown` line exists for the + /// mirror-image misreading: the same `` value, where the CLI + /// genuinely does not know, must not be read as either claim. /// /// Classified by [`gpu_support_level`] rather than re-deriving the same /// `(detected, covering family)` test here: this line and that function are @@ -1960,10 +1970,33 @@ impl ExamineSummary { // panic path. GpuSupportLevel::VulkanOnly => format!( " rocm_support: GPU detected ({}) but not supported by ROCm; \ - the Vulkan backend can still be used\n", + the Vulkan backend can still be used - serve on it with \ + `rocm serve --engine lemonade`\n", detected.unwrap_or_default() ), - GpuSupportLevel::RocmHip | GpuSupportLevel::Unknown => String::new(), + // The covering family is the fact; it is already printed on its own + // line above, so this one says what it means rather than repeating + // it. + GpuSupportLevel::RocmHip => format!( + " rocm_support: GPU detected ({}) with full ROCm/HIP support; \ + install the SDK for it with `rocm install sdk`\n", + detected.unwrap_or("") + ), + // Not folded in with `VulkanOnly`: a detection gap is not a + // confirmed absence of ROCm support, and saying "Vulkan" here would + // be the same false claim in the other direction. What the user can + // do is let an engine decide and then read back what it chose - + // which is the `device:` row this same ticket added, and the only + // thing on this host that can actually settle the question. + GpuSupportLevel::Unknown => format!( + " rocm_support: unknown - {}, so this build can confirm neither \ + ROCm support nor its absence; `rocm serve --engine \ + lemonade` reports the backend it picked on its `device:` row\n", + match detected { + Some(gfx) => format!("GPU target {gfx} is not in this build's catalog"), + None => "no AMD GPU was detected".to_owned(), + } + ), } } @@ -5175,22 +5208,88 @@ Class Name: Display ); } + /// Every support level reports one, because the other surfaces send the + /// user here to find out: `lemonade_engine_note` tells an `Unknown` host to + /// run `rocm examine`, and an examine that prints nothing for `Unknown` + /// makes that advice name a command which cannot clear the condition. + /// + /// Each line is checked against what it is allowed to claim, not merely for + /// being non-empty — the failure mode this guards is a report that answers + /// confidently on a host where the CLI knows nothing. #[test] - fn examine_render_omits_the_support_note_for_supported_or_missing_gpus() { + fn examine_render_reports_a_support_level_for_every_host() { let family = normalize_therock_family("gfx1103"); let supported = examine_summary_with_gpu(Some("gfx1103"), family.as_deref()); let rendered = supported.render_text(); - assert!(!rendered.contains("rocm_support"), "{rendered}"); + assert!( + rendered.contains("rocm_support: GPU detected (gfx1103) with full ROCm/HIP support"), + "{rendered}" + ); + assert!(rendered.contains("rocm install sdk"), "{rendered}"); + // No GPU found: the level is reported as unknown, and nothing about the + // hardware is asserted in either direction. let missing = examine_summary_with_gpu(None, None); let rendered = missing.render_text(); - assert!(!rendered.contains("rocm_support"), "{rendered}"); + assert!( + rendered.contains("rocm_support: unknown - no AMD GPU was detected"), + "{rendered}" + ); + assert!( + !rendered.contains("Vulkan backend can still be used"), + "nothing was detected, so no backend may be promised: {rendered}" + ); // An unmapped target the CLI knows nothing about is not claimed to be // unsupported; only targets known to have no ROCm build are. let unknown = examine_summary_with_gpu(Some("gfx803"), None); let rendered = unknown.render_text(); - assert!(!rendered.contains("rocm_support"), "{rendered}"); + assert!( + rendered.contains( + "rocm_support: unknown - GPU target gfx803 is not in this build's catalog" + ), + "{rendered}" + ); + assert!( + !rendered.contains("not supported by ROCm"), + "an unrecognized target must not be reported as confirmed \ + ROCm-less: {rendered}" + ); + assert!( + !rendered.contains("full ROCm/HIP support"), + "nor as supported: {rendered}" + ); + } + + /// The remediation in `lemonade_engine_note` (`apps/rocm/src/main.rs`) for + /// an `Unknown` host names `rocm examine`. AGENTS.md requires a named + /// command to actually clear the condition that printed it, so the line + /// examine prints there has to be an answer about the support level — the + /// thing the note said it would show. + #[test] + fn examine_answers_the_support_question_the_unknown_engine_note_sends_users_here_for() { + for detected in [None, Some("gfx803")] { + let rendered = examine_summary_with_gpu(detected, None).render_text(); + let line = rendered + .lines() + .find(|line| line.trim_start().starts_with("rocm_support:")) + .unwrap_or_else(|| { + panic!( + "`rocm examine` is where the Unknown-tier engine note sends \ + the user; it must print a support level:\n{rendered}" + ) + }); + // Says which level it is, and leaves a command that can settle it. + assert!(line.contains("unknown"), "{line}"); + assert!( + line.contains("rocm serve --engine lemonade"), + "an `unknown` answer still has to leave somewhere to go: {line}" + ); + assert!( + line.contains("device:"), + "and name where the real backend shows up: {line}" + ); + } } #[test] diff --git a/crates/rocm-engine-protocol/src/lib.rs b/crates/rocm-engine-protocol/src/lib.rs index 6ca773ccd6..e11703ffb9 100644 --- a/crates/rocm-engine-protocol/src/lib.rs +++ b/crates/rocm-engine-protocol/src/lib.rs @@ -566,6 +566,51 @@ pub struct LaunchResponse { pub state_path: String, } +/// The [`HealthcheckResponse::device`] string, in the one shape every engine +/// reports and the `device:` row of `rocm serve` / `rocm services list` prints +/// verbatim. +/// +/// ROCMAI-550 asks those two surfaces for "the llama.cpp backend actually +/// chosen (`vulkan`/`rocm`) **and the device name**", so both halves are named +/// arguments here rather than text each engine formats for itself: +/// +/// | backend | ordinal | name | rendered | +/// | --- | --- | --- | --- | +/// | `vulkan` | `Some(0)` | `Some("AMD Radeon Graphics")` | `vulkan gpu 0 (AMD Radeon Graphics)` | +/// | `vulkan` | `Some(0)` | `None` | `vulkan gpu 0` | +/// | `vulkan` | `None` | `Some("AMD Radeon Graphics")` | `vulkan (AMD Radeon Graphics)` | +/// | `vulkan` | `None` | `None` | `vulkan` | +/// +/// The last row deliberately drops the `gpu` noun rather than printing a +/// dangling `vulkan gpu`. That bare form is not hypothetical: it is what the +/// ticket's own host produces. `gpu_indices` comes from +/// `engines/lemonade`'s `resolve_serve_gpu_indices`, whose probe +/// (`rocm_core::usable_amd_gpu_indices`) returns `None` off Linux, so a default +/// `rocm serve` on Windows pins no ordinal and records an empty list — and +/// Windows on a gfx90c Vega iGPU is exactly the machine ROCMAI-550 was filed +/// from. `device: vulkan gpu` would answer "which GPU?" with the word "gpu". +/// +/// The name is whatever the engine recorded for the host at launch; it is not +/// per-ordinal. On a multi-GPU host the ordinal is the precise half and the +/// name is context, which is why the ordinal is never dropped in favour of it. +#[must_use] +pub fn format_reported_device( + backend: &str, + gpu_index: Option, + gpu_name: Option<&str>, +) -> String { + use std::fmt::Write as _; + + let mut device = backend.to_owned(); + if let Some(index) = gpu_index { + let _ = write!(device, " gpu {index}"); + } + if let Some(name) = gpu_name.map(str::trim).filter(|name| !name.is_empty()) { + let _ = write!(device, " ({name})"); + } + device +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct HealthcheckResponse { /// Engine-reported service state. `ready` is reserved for a service that has @@ -646,6 +691,55 @@ pub struct LogsResponse { #[cfg(test)] mod tests { + /// The `device:` row has to carry a device, not the word "gpu". + /// + /// The ordinal-less case is the one ROCMAI-550 was filed about: on Windows + /// `rocm_core::usable_amd_gpu_indices` cannot probe, so a default + /// `rocm serve` records an empty `gpu_indices` and the old formatting + /// rendered a bare `vulkan gpu`. The name is what makes that row answer the + /// question; with neither, the noun goes too rather than dangling. + #[test] + fn reported_device_never_renders_a_dangling_gpu_noun() { + assert_eq!( + super::format_reported_device("vulkan", Some(0), Some("AMD Radeon Graphics")), + "vulkan gpu 0 (AMD Radeon Graphics)" + ); + assert_eq!( + super::format_reported_device("vulkan", None, Some("AMD Radeon Graphics")), + "vulkan (AMD Radeon Graphics)" + ); + assert_eq!( + super::format_reported_device("rocm", Some(1), None), + "rocm gpu 1" + ); + // The regression this exists to stop: nothing known beyond the backend + // must not be padded out with a noun that names no device. + assert_eq!( + super::format_reported_device("vulkan", None, None), + "vulkan" + ); + } + + /// A name the host probe could not fill in arrives as `Some("")` or + /// `Some(" ")` just as easily as `None` (the Windows display-adapter probe + /// and the Linux DRM read both build it from scraped text), and + /// `vulkan ()` is worse than no parenthetical at all. + #[test] + fn reported_device_treats_a_blank_gpu_name_as_absent() { + assert_eq!( + super::format_reported_device("vulkan", Some(0), Some("")), + "vulkan gpu 0" + ); + assert_eq!( + super::format_reported_device("vulkan", None, Some(" ")), + "vulkan" + ); + assert_eq!( + super::format_reported_device("rocm", Some(2), Some(" MI300X ")), + "rocm gpu 2 (MI300X)" + ); + } + fn resolve_response(served: Option<&str>) -> ResolveModelResponse { ResolveModelResponse { canonical_model_id: "image:tag".to_owned(), diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 05d6fb5f82..f568c2d098 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -285,14 +285,15 @@ Expected result: - The engine uses ROCm GPU execution. The deployment summary's `device:` row reports what the engine actually selected, so check it by reading that row - rather than inferring it from the absence of a fallback warning: expect - `rocm gpu `, or a bare `rocm gpu` when the launch pinned no ordinal (no - `--gpu`, nothing recorded in the engine's `gpu_indices`). `rocm services` - shows the same `device` while it runs. Every engine reports this row in the - same ` gpu []` shape, so the first word is the half that decides - this check. -- The run does not fall back to CPU or Vulkan — a `device:` of `vulkan gpu ` - fails this check even though the serve itself succeeds. + rather than inferring it from the absence of a fallback warning. Every engine + reports it in the same ` [gpu ] [()]` shape — so expect + something like `rocm gpu 0 (AMD Instinct MI300X)`, with the ordinal omitted + when the launch pinned none (no `--gpu`, nothing recorded in the engine's + `gpu_indices`) and the name omitted when the host probe could not read one. + `rocm services` shows the same `device` while it runs. The first word is the + half that decides this check. +- The run does not fall back to CPU or Vulkan — a `device:` whose first word is + `vulkan` fails this check even though the serve itself succeeds. - If the GPU cannot be used, the command fails with a clear error. - The OpenAI-compatible endpoint answers a simple chat request. diff --git a/engines/aim/src/state.rs b/engines/aim/src/state.rs index bcb783e9de..878af75960 100644 --- a/engines/aim/src/state.rs +++ b/engines/aim/src/state.rs @@ -33,16 +33,9 @@ pub(crate) fn healthcheck_service(request: HealthcheckRequest) -> Result reported_device(state), + None => "unknown".to_owned(), }; let listed = !endpoint_url.is_empty() @@ -56,10 +49,30 @@ pub(crate) fn healthcheck_service(request: HealthcheckRequest) -> Result String { + rocm_engine_protocol::format_reported_device( + "rocm", + None, + value_string(state, "gpu_name").as_deref(), + ) +} + pub(crate) fn endpoint_response(request: EndpointRequest) -> Result { require_nonempty(&request.service_id, "service_id")?; let state = read_service_state(&state_path(&request.service_id)) @@ -139,6 +152,13 @@ pub(crate) fn write_running_state( "port": port, "endpoint_url": endpoint, "device_policy": device_policy, + // The host's GPU name, so the `device:` row names a device and not + // only a backend (ROCMAI-550). This engine records no GPU ordinal, + // so without it its row would be the bare word `rocm` on every + // host. Captured once here rather than probed per healthcheck: + // `rocm services list` healthchecks every ready service on its + // render path. + "gpu_name": rocm_core::detect_host_gpu_summary(None).name, "engine_recipe": engine_recipe, "started_at_unix_ms": current_unix_millis(), }), @@ -361,6 +381,69 @@ mod tests { assert!(on_disk.contains("HF_TOKEN")); } + /// ROCMAI-550 changed the device this engine reports for an existing + /// service from the literal `"rocm_gpu"` to the shared + /// ` [gpu ] [()]` shape, and nothing pinned it. + /// + /// This drives the real persist-then-report path: + /// [`write_running_state`] writes the state file, [`read_service_state`] + /// reads it back and [`reported_device`] renders it — exactly what + /// [`healthcheck_service`] does for a tracked service, minus the + /// `state_path` lookup, which resolves through the caller's real + /// `AppPaths` and so cannot be driven from a unit test (the no-state half + /// is covered by + /// `healthcheck_reports_unknown_for_a_service_with_no_state_file`). + /// + /// Reverting the string to `"rocm_gpu"`, or dropping the `gpu_name` field + /// from the state write, fails this. + #[test] + fn reported_device_for_existing_state_matches_the_shared_engine_shape() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("svc-device.json"); + write_running_state( + &path, + "svc-device", + "some/model", + Some("Org/Served"), + "127.0.0.1", + 8000, + DevicePolicy::GpuRequired, + None, + ) + .unwrap(); + + // The probed name is whatever the test host has, so assert the shape + // rather than a value: `rocm`, optionally followed by `()`, and + // never the old `rocm_gpu` or a `gpu` noun with no ordinal behind it. + let state = read_service_state(&path).unwrap(); + let device = reported_device(&state); + assert!( + device == "rocm" || (device.starts_with("rocm (") && device.ends_with(')')), + "AIM must report the shared ` [()]` shape: {device}" + ); + assert!( + !device.contains("rocm_gpu"), + "the pre-ROCMAI-550 literal must not come back: {device}" + ); + assert!( + !device.split_whitespace().any(|word| word == "gpu"), + "this engine pins no ordinal, so a bare `gpu` noun would name \ + nothing: {device}" + ); + // The state file must actually carry the key the row is built from, + // even when the host probe found no name. + assert!( + state.get("gpu_name").is_some(), + "write_running_state must record gpu_name: {state}" + ); + + // And the name, when the host has one, reaches the row. + assert_eq!( + reported_device(&json!({ "gpu_name": "AMD Instinct MI300X" })), + "rocm (AMD Instinct MI300X)" + ); + } + #[test] fn status_helpers_round_trip_and_keep_other_fields() { let dir = tempfile::tempdir().unwrap(); diff --git a/engines/lemonade/src/state.rs b/engines/lemonade/src/state.rs index 6df1ded86d..08c3d8f852 100644 --- a/engines/lemonade/src/state.rs +++ b/engines/lemonade/src/state.rs @@ -55,13 +55,29 @@ pub(crate) fn inference_verified( } /// The device string reported once a model is loaded. Reflects the backend that -/// actually ran (`rocm`, `vulkan`, …) and the pinned GPU ordinal when known, -/// rather than a hardcoded value, so the report matches observed execution. +/// actually ran (`rocm`, `vulkan`, …), the pinned GPU ordinal when known, and +/// the GPU name recorded at launch, rather than a hardcoded value, so the +/// report matches observed execution. +/// +/// Formatted by [`rocm_engine_protocol::format_reported_device`] so all three +/// engines render one shape; see that function for why the ordinal-less row +/// drops the `gpu` noun rather than printing a bare `vulkan gpu`. pub(crate) fn reported_device(state: Option<&Value>, backend: &str) -> String { - match state.and_then(first_gpu_index_from_state) { - Some(index) => format!("{backend} gpu {index}"), - None => format!("{backend} gpu"), - } + rocm_engine_protocol::format_reported_device( + backend, + state.and_then(first_gpu_index_from_state), + state.and_then(gpu_name_from_state).as_deref(), + ) +} + +/// The GPU name recorded in service state at launch, if any. +/// +/// Read back rather than re-probed: `healthcheck_service` runs on the +/// `rocm services list` render path (see `fetch_service_devices`), where an +/// extra host probe per service is latency the row is not worth, and the launch +/// host is the one whose GPU actually ran the server. +fn gpu_name_from_state(state: &Value) -> Option { + value_string(state, "gpu_name") } /// The first pinned GPU ordinal recorded in service state, if any. @@ -155,6 +171,13 @@ pub(crate) fn write_running_state( "endpoint_url": endpoint_url(&request.host, request.port), "device_policy": device_policy_name(&request.device_policy), "gpu_indices": request.gpu_indices, + // The host's GPU name, so `healthcheck_service` can name a device + // and not just a backend (ROCMAI-550). Captured here, once, rather + // than probed per healthcheck: `rocm services list` healthchecks + // every ready service on its render path. Recorded even when it is + // `null`, so the key's absence means an older state file rather + // than a probe that came back empty. + "gpu_name": rocm_core::detect_host_gpu_summary(None).name, "runtime_id": request.runtime_id.as_deref().unwrap_or(runtime.manifest.env_id.as_str()), "env_id": request.env_id.as_deref().unwrap_or(runtime.manifest.env_id.as_str()), "runtime_kind": "lemonade_embeddable", @@ -700,11 +723,34 @@ mod tests { } #[test] - fn reported_device_reflects_backend_and_pinned_ordinal() { - let state = json!({ "gpu_indices": [1] }); - assert_eq!(reported_device(Some(&state), "vulkan"), "vulkan gpu 1"); + fn reported_device_reflects_backend_pinned_ordinal_and_gpu_name() { + let state = json!({ "gpu_indices": [1], "gpu_name": "AMD Instinct MI300X" }); + assert_eq!( + reported_device(Some(&state), "vulkan"), + "vulkan gpu 1 (AMD Instinct MI300X)" + ); // Without a recorded ordinal the backend is still reported truthfully. - assert_eq!(reported_device(None, "rocm"), "rocm gpu"); + assert_eq!(reported_device(None, "rocm"), "rocm"); + } + + /// ROCMAI-550's own host: Windows cannot probe GPU ordinals + /// (`rocm_core::usable_amd_gpu_indices` is `None` off Linux), so a default + /// `rocm serve` records an empty `gpu_indices` there and the only + /// device-identifying fact in state is the name. Before this the row read + /// `vulkan gpu` — the backend, then a noun naming nothing — which is the + /// shape the ticket filed as "doesn't show the GPU". + #[test] + fn reported_device_names_the_gpu_when_no_ordinal_was_pinned() { + let state = json!({ "gpu_indices": [], "gpu_name": "AMD Radeon(TM) Graphics" }); + assert_eq!( + reported_device(Some(&state), "vulkan"), + "vulkan (AMD Radeon(TM) Graphics)" + ); + + // An older state file, written before `gpu_name` was recorded, still + // reports its backend rather than a dangling noun. + let legacy = json!({ "gpu_indices": [] }); + assert_eq!(reported_device(Some(&legacy), "vulkan"), "vulkan"); } #[test] diff --git a/engines/vllm/src/state.rs b/engines/vllm/src/state.rs index cfd6607c1c..e56768106f 100644 --- a/engines/vllm/src/state.rs +++ b/engines/vllm/src/state.rs @@ -81,14 +81,16 @@ pub(crate) fn healthcheck_service(request: HealthcheckRequest) -> Result String { - match first_gpu_index_from_state(state) { - Some(index) => format!("rocm gpu {index}"), - None => "rocm gpu".to_owned(), - } + rocm_engine_protocol::format_reported_device( + "rocm", + first_gpu_index_from_state(state), + value_string(state, "gpu_name").as_deref(), + ) } /// The first pinned GPU ordinal recorded in service state, if any. @@ -178,10 +180,15 @@ pub(crate) fn write_running_state( "endpoint_url": endpoint_url(&request.host, request.port), "device_policy": "gpu_required", // Recorded so `healthcheck_service` can report the GPU this service - // was actually pinned to, in the same `"rocm gpu "` shape - // `engines/lemonade` reports. Introspection only, like the rest of - // this file — never read back to relaunch. + // was actually pinned to, in the same shape `engines/lemonade` + // reports. Introspection only, like the rest of this file — never + // read back to relaunch. "gpu_indices": request.gpu_indices, + // The host's GPU name, so the `device:` row names a device and not + // only a backend (ROCMAI-550). Captured once here rather than + // probed per healthcheck, because `rocm services list` + // healthchecks every ready service on its render path. + "gpu_name": rocm_core::detect_host_gpu_summary(None).name, "runtime_id": runtime.runtime_id, "requested_runtime_id": request.runtime_id, "env_id": request.env_id.as_deref().unwrap_or(runtime.env_id.as_str()), @@ -751,19 +758,27 @@ mod tests { /// The `device:` row `rocm serve` and `rocm services list` print comes /// straight from this string, so vLLM has to speak the same shape - /// `engines/lemonade` does (`"{backend} gpu {index}"`). It used to be the - /// literal `"rocm_gpu"`, which is the format README.md and - /// `docs/manual-testing.md` never described and which made one row - /// disagree with itself across engines. + /// `engines/lemonade` does. It used to be the literal `"rocm_gpu"`, which + /// is the format README.md and `docs/manual-testing.md` never described and + /// which made one row disagree with itself across engines. #[test] fn reported_device_matches_the_shape_lemonade_reports() { + assert_eq!( + reported_device(&json!({ "gpu_indices": [1], "gpu_name": "AMD Instinct MI300X" })), + "rocm gpu 1 (AMD Instinct MI300X)" + ); assert_eq!( reported_device(&json!({ "gpu_indices": [1] })), "rocm gpu 1" ); // No pinned ordinal recorded: the backend is still reported truthfully, - // exactly as `engines/lemonade`'s `reported_device(None, "rocm")` does. - assert_eq!(reported_device(&json!({ "status": "running" })), "rocm gpu"); - assert_eq!(reported_device(&json!({ "gpu_indices": [] })), "rocm gpu"); + // exactly as `engines/lemonade`'s `reported_device(None, "rocm")` does, + // and without the dangling `gpu` noun that names no device. + assert_eq!(reported_device(&json!({ "status": "running" })), "rocm"); + assert_eq!(reported_device(&json!({ "gpu_indices": [] })), "rocm"); + assert_eq!( + reported_device(&json!({ "gpu_name": "AMD Instinct MI300X" })), + "rocm (AMD Instinct MI300X)" + ); } } diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index c2a8cd2044..010a918869 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1446,23 +1446,42 @@ async fn assert_service_in_list(world: &mut E2eWorld) { /// The `device:` value as `rocm serve` / `rocm services list` print it, split /// into the backend it names and the GPU ordinal it pinned, if any. /// -/// Every engine reports `" gpu"` or `" gpu "` -/// (`engines/lemonade`'s and `engines/vllm`'s `reported_device`, -/// `engines/aim`'s literal). `None` for anything else, which is what makes a -/// hardcoded or malformed value fail rather than pass. +/// Every engine renders it through +/// `rocm_engine_protocol::format_reported_device`, so the grammar is +/// ``, then an optional ` gpu `, then an optional ` ()`. +/// `None` for anything else, which is what makes a hardcoded or malformed value +/// fail rather than pass. fn parse_device_row(value: &str) -> Option<(&str, Option)> { - let mut parts = value.split_whitespace(); + // The name is the only part that may contain spaces, and it is always last + // and parenthesised. Split on the FIRST " (" so a name with its own + // parentheses ("AMD Radeon (TM) Graphics") still leaves the head intact. + let head = match value.split_once(" (") { + Some((head, name)) if name.ends_with(')') => head, + Some(_) => return None, + None => value, + }; + let mut parts = head.split_whitespace(); let backend = parts.next()?; - if parts.next()? != "gpu" { - return None; - } let ordinal = match parts.next() { - Some(index) => Some(index.parse().ok()?), + Some("gpu") => Some(parts.next()?.parse().ok()?), + // A bare `gpu` with no ordinal is the shape ROCMAI-550 removed; so is + // any other trailing word. + Some(_) => return None, None => None, }; parts.next().is_none().then_some((backend, ordinal)) } +/// How long the `device:` row may take to show up after the serve returned. +/// +/// It is not available the instant a service is recorded `ready`: +/// `engines/lemonade` reports `"unknown"` until its own `inference_verified` +/// probe has come back, and `fetch_service_device` drops `"unknown"` rather +/// than printing it. So there is a real window in which a correct +/// implementation has no row yet, and failing on the first look would make +/// this step flake on timing rather than on behavior. +const DEVICE_ROW_DEADLINE: Duration = Duration::from_secs(90); + /// The value of the first `device:` row in `text`, trimmed. fn device_row_value(text: &str) -> Option<&str> { text.lines() @@ -1480,32 +1499,53 @@ fn device_row_value(text: &str) -> Option<&str> { /// `device:`" while reporting nothing real, so this parses it and holds it to /// three independent facts: /// -/// 1. it has the ` gpu []` shape every engine reports; +/// 1. it has the ` [gpu ] [()]` shape every engine reports; /// 2. the backend is one this engine can actually have selected - only /// Lemonade has a Vulkan path, so `vulkan` from `vllm` or `aim` is wrong, /// and `unknown` is wrong from any of them; /// 3. it equals what the `rocm serve` deployment summary printed for the same /// service, which is the row the preceding Given produced and which nothing /// else in these scenarios looks at. +/// The row is polled rather than read once: see [`DEVICE_ROW_DEADLINE`] for +/// the window in which a correct implementation legitimately has no row yet. +/// Running out of that window is an assertion naming what was last seen, not a +/// panic on the first look. #[then("the services list reports the engine's actual backend device")] async fn assert_services_list_reports_device(world: &mut E2eWorld) { - let (stdout, _, _) = crate::run_rocm(world, &["services", "list"]); - let model = world.model_name.as_deref().unwrap_or(""); - let block = stdout - .split("\n- ") - .find(|block| block.to_lowercase().contains(&model.to_lowercase())) - .unwrap_or_else(|| panic!("no services-list entry for model {model}:\n{stdout}")); - let device = device_row_value(block).unwrap_or_else(|| { - panic!( - "services list entry for {model} is missing a `device:` line - the \ - engine's Healthcheck RPC should report the backend it actually \ - selected:\n{block}" - ) - }); - let (backend, _ordinal) = parse_device_row(device).unwrap_or_else(|| { + let model = world.model_name.clone().unwrap_or_default(); + let deadline = Instant::now() + DEVICE_ROW_DEADLINE; + // The last thing looked at, so a timeout can show it rather than just + // saying it timed out: the service's own block when there was one, else the + // whole listing (which is the evidence for the service having gone missing). + let mut last_seen = String::new(); + let device = loop { + let (stdout, _, _) = crate::run_rocm(world, &["services", "list"]); + match stdout + .split("\n- ") + .find(|block| block.to_lowercase().contains(&model.to_lowercase())) + { + Some(block) => { + last_seen = block.to_owned(); + if let Some(device) = device_row_value(&last_seen) { + break device.to_owned(); + } + } + None => last_seen = stdout, + } + assert!( + Instant::now() < deadline, + "after {}s `rocm services list` still reports no `device:` line for \ + {model} - the engine's Healthcheck RPC should report the backend it \ + actually selected once the service is serving:\n{last_seen}", + DEVICE_ROW_DEADLINE.as_secs() + ); + tokio::time::sleep(Duration::from_secs(3)).await; + }; + let block = last_seen.as_str(); + let (backend, _ordinal) = parse_device_row(&device).unwrap_or_else(|| { panic!( - "`device: {device}` is not the ` gpu []` shape every \ - engine reports, so it cannot be the engine's own answer:\n{block}" + "`device: {device}` is not the ` [gpu ] [()]` shape \ + every engine reports, so it cannot be the engine's own answer:\n{block}" ) }); From 7fba9aa6b55a506f9d888c5b7a9a7edaeb23f406 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Mon, 2 Nov 2026 16:00:35 +0200 Subject: [PATCH 08/37] ROCMAI-550: resync the advice-scan exclusions with the new device shape The `device:` row now renders ` [gpu ] [()]`, so the strings the advised-command scan has to ignore moved with it: vLLM no longer builds `rocm gpu {index}`, AIM no longer carries a bare `rocm gpu`, and the two docs quote the named-device example instead. Also drops a redundant initialiser the e2e `device:` poll never reads. Signed-off-by: Roman Sirokov --- apps/rocm/src/advised_commands.rs | 30 +++++-------------- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 2 +- 2 files changed, 9 insertions(+), 23 deletions(-) diff --git a/apps/rocm/src/advised_commands.rs b/apps/rocm/src/advised_commands.rs index 39e58fa4c9..c33adbf72a 100644 --- a/apps/rocm/src/advised_commands.rs +++ b/apps/rocm/src/advised_commands.rs @@ -1279,40 +1279,26 @@ const NOT_INVOCATIONS: &[(&str, &str, &str)] = &[ "the doc's expected result is that the parser rejects this combination", ), // The `device:` row's value (ROCMAI-550). Every engine reports the backend - // it selected as ` gpu []` -- `vulkan gpu 0`, `rocm gpu 1` -- - // so the ROCm half of that value reads as a `rocm` invocation to this scan - // while naming no command at all. These are the engine side of it and the - // two docs that quote the format back. + // it selected as ` [gpu ] [()]` -- `vulkan (AMD Radeon + // Graphics)`, `rocm gpu 1 (AMD Instinct MI300X)` -- so the ROCm half of + // that value reads as a `rocm` invocation to this scan while naming no + // command at all. These are the engine side of it and the two docs that + // quote the format back. ( "engines/vllm/src/state.rs", - "rocm gpu {index}", - "the `device:` value vLLM reports, ` gpu `, not a command", - ), - ( - "engines/vllm/src/state.rs", - "rocm gpu", - "the `device:` value vLLM reports when no GPU ordinal was pinned", - ), - ( - "engines/aim/src/state.rs", "rocm gpu", - "the `device:` value AIM reports; its state records no GPU ordinal", + "the shape note for the `device:` value vLLM reports, not a command", ), ( "README.md", - "rocm gpu 0", + "rocm gpu 0 (AMD Instinct MI300X)", "an example `device:` value in the deployment-summary description, not a command", ), ( "docs/manual-testing.md", - "rocm gpu ", + "rocm gpu 0 (AMD Instinct MI300X)", "the `device:` value the tester is told to expect, not a command to run", ), - ( - "docs/manual-testing.md", - "rocm gpu", - "the ordinal-less `device:` value the tester is told to expect", - ), ]; /// Natural-language examples must reach the planner with a multi-word request: diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 010a918869..92bd5779b4 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1517,7 +1517,7 @@ async fn assert_services_list_reports_device(world: &mut E2eWorld) { // The last thing looked at, so a timeout can show it rather than just // saying it timed out: the service's own block when there was one, else the // whole listing (which is the evidence for the service having gone missing). - let mut last_seen = String::new(); + let mut last_seen; let device = loop { let (stdout, _, _) = crate::run_rocm(world, &["services", "list"]); match stdout From e1daa50ddf3e9d9a02c9b57fd1e9fec384f0cd40 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Mon, 2 Nov 2026 16:09:06 +0200 Subject: [PATCH 09/37] ROCMAI-550: drop the engine advice-scan exclusions the shared formatter removed Neither vLLM nor AIM holds a `"rocm gpu ..."` string literal any more - both build the `device:` value through `rocm_engine_protocol::format_reported_device` - so the entries that excused those literals no longer match anything and `exclusions_still_match_something` rejects them. Also drops a stale `"{backend} gpu {index}"` shape note from vLLM's doc comment. Signed-off-by: Roman Sirokov --- apps/rocm/src/advised_commands.rs | 9 +++------ engines/vllm/src/state.rs | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/apps/rocm/src/advised_commands.rs b/apps/rocm/src/advised_commands.rs index c33adbf72a..f7c0e529c9 100644 --- a/apps/rocm/src/advised_commands.rs +++ b/apps/rocm/src/advised_commands.rs @@ -1282,13 +1282,10 @@ const NOT_INVOCATIONS: &[(&str, &str, &str)] = &[ // it selected as ` [gpu ] [()]` -- `vulkan (AMD Radeon // Graphics)`, `rocm gpu 1 (AMD Instinct MI300X)` -- so the ROCm half of // that value reads as a `rocm` invocation to this scan while naming no - // command at all. These are the engine side of it and the two docs that + // command at all. The engines no longer need an entry each: they build the + // string through `rocm_engine_protocol::format_reported_device` rather + // than holding a `"rocm gpu ..."` literal. These are the two docs that // quote the format back. - ( - "engines/vllm/src/state.rs", - "rocm gpu", - "the shape note for the `device:` value vLLM reports, not a command", - ), ( "README.md", "rocm gpu 0 (AMD Instinct MI300X)", diff --git a/engines/vllm/src/state.rs b/engines/vllm/src/state.rs index e56768106f..bcb2c447d9 100644 --- a/engines/vllm/src/state.rs +++ b/engines/vllm/src/state.rs @@ -77,7 +77,7 @@ pub(crate) fn healthcheck_service(request: HealthcheckRequest) -> Result Date: Tue, 3 Nov 2026 14:23:16 +0200 Subject: [PATCH 10/37] ROCMAI-550: pin the support line with a scenario, and stop the device step failing a correct run Review round 3 on #678. - The `device:` e2e step failed a legitimately correct run. Lemonade reports `unknown` until its own inference probe returns, and the serve summary reads the device once right after `ready`, so the row can be absent on a correct implementation. Only serve-07 being xfail'd on lemonade+linux hid it. The step now tolerates the absence and the window is documented rather than asserted away. - `rocm examine` grew a `rocm_support` line for RocmHip and Unknown with no scenario asserting it. examine-31 covers it, deliberately untagged so each lane's own hardware is the coverage: the claim is cross-checked against `detected_gfx_target` and `compatible_therock_family` from the same report, so "full ROCm/HIP support" on the mock lane or "not supported by ROCm" on an MI300X both fail. Signed-off-by: Roman Sirokov --- README.md | 14 +- apps/rocm/src/engine_ipc.rs | 133 +++++++++ apps/rocm/src/main.rs | 13 +- apps/rocm/src/serve_cmd.rs | 11 +- apps/rocm/src/serve_summary.rs | 15 +- crates/rocm-core/src/host_gpu.rs | 272 +++++++++++++++++- docs/manual-testing.md | 12 +- docs/testing.md | 46 +++ engines/aim/src/state.rs | 7 +- engines/lemonade/src/state.rs | 17 +- engines/vllm/src/state.rs | 13 +- tests/e2e-cucumber/features/examine.feature | 21 ++ tests/e2e-cucumber/tests/e2e/examine_steps.rs | 125 ++++++++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 45 +-- 14 files changed, 694 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index f3bc28de17..46294dc3af 100644 --- a/README.md +++ b/README.md @@ -888,9 +888,17 @@ example `rocm gpu 0 (AMD Instinct MI300X)`, or `vulkan (AMD Radeon Graphics)` where the engine pinned no GPU ordinal — read back from the running engine), the full inference endpoint, the API-qualified model name, and a quick smoke test (time to first token and approximate tokens per second). `rocm services` reports -the same `device` for each running service. Control -returns to your shell with the server still running; manage it later with `rocm -services` (below). +the same `device` for each running service. + +The device line is omitted when the engine cannot say yet. A server is reported +ready as soon as it answers, which can be before it has finished loading the +model, and until then the engine does not know which device it ended up on; +`rocm serve` waits a few seconds for it rather than holding your terminal, and +`rocm services` shows it once it is known. The GPU name is omitted on a machine +with more than one AMD GPU, where naming one of them could name the wrong one. + +Control returns to your shell with the server still running; manage it later +with `rocm services` (below). `--verbose` (or `--foreground`) instead attaches to the server in the current terminal and streams every engine log line. Use it to debug a startup problem. diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index 94ce6c719c..918adccf6f 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -333,6 +333,81 @@ pub(crate) fn fetch_service_device( (!response.device.is_empty() && response.device != "unknown").then_some(response.device) } +/// How long `rocm serve`'s deployment summary may keep asking the engine which +/// device it selected, and how long it waits between asks. +/// +/// A managed launch reports `ready` as soon as its server answers, but +/// `engines/lemonade` reports `device: "unknown"` until its own +/// `inference_verified` probe has come back — so asking once, immediately, +/// loses the row on exactly the Vulkan case ROCMAI-550 is about. One retry +/// window closes the common case: the default smoke test has just put a real +/// completion through the server, so the engine's own probe is a cache hit. +/// +/// **It cannot close every case, and is not meant to.** A server that reported +/// `ready` while weights are still loading can stay unverified for minutes, and +/// waiting that out would hold the terminal hostage for an informational row +/// that `rocm services list` will show as soon as it exists. So the row is +/// *optional by contract* — see [`crate::serve_summary::DeploymentSummary`]'s +/// `device` field, `docs/manual-testing.md` and README.md, which all say it can +/// be absent. +/// +/// The budget is a start gate, like [`SERVICE_DEVICE_PROBE_BUDGET`]: it is +/// tested before each attempt, never inside one, so the worst-case addition to +/// `rocm serve` is this budget plus one in-flight probe — about 14s with +/// Lemonade's 8s `rocm_core::INFERENCE_PROBE_TIMEOUT`, and only on a launch +/// that was going to have no row at all. +const SERVE_DEVICE_PROBE_BUDGET: Duration = Duration::from_secs(6); +/// Pause between the attempts [`SERVE_DEVICE_PROBE_BUDGET`] allows. Short +/// enough that the common case (the engine is a second away from verifying) +/// resolves quickly, long enough not to spin on an engine that answers +/// instantly. +const SERVE_DEVICE_PROBE_INTERVAL: Duration = Duration::from_secs(2); + +/// [`fetch_service_device`], retried briefly, for `rocm serve`'s one-shot +/// deployment summary. +/// +/// `None` is a legitimate answer and must stay cheap to get: see +/// [`SERVE_DEVICE_PROBE_BUDGET`] for why the wait is bounded and why the +/// summary prints no `device:` row rather than blocking on one. +pub(crate) fn fetch_service_device_briefly( + paths: &AppPaths, + engine: &str, + service_id: &str, +) -> Option { + let started = Instant::now(); + retry_service_device( + || started.elapsed(), + || fetch_service_device(paths, engine, service_id), + std::thread::sleep, + ) +} + +/// The retry loop of [`fetch_service_device_briefly`], with the clock, the +/// probe and the wait injected. +/// +/// Split out for the same reason as [`collect_service_devices`]: driving this +/// against the live clock would mean a test that really sleeps out +/// [`SERVE_DEVICE_PROBE_BUDGET`] against a real engine. What has to be +/// guaranteed is not just that a late answer is reported but that the loop +/// *stops* — an unbounded retry here would hang every `rocm serve` whose engine +/// never verifies. +fn retry_service_device(mut elapsed: C, mut probe: P, mut wait: S) -> Option +where + C: FnMut() -> Duration, + P: FnMut() -> Option, + S: FnMut(Duration), +{ + loop { + if let Some(device) = probe() { + return Some(device); + } + if elapsed() >= SERVE_DEVICE_PROBE_BUDGET { + return None; + } + wait(SERVE_DEVICE_PROBE_INTERVAL); + } +} + /// How long [`fetch_service_devices`] may spend on live `Healthcheck` RPCs in /// total, across however many services were listed. /// @@ -724,6 +799,64 @@ mod tests { ); } + /// The serve summary asks again when the first answer is `None`: the + /// engine can be seconds from verifying its own inference probe, which is + /// the `device: "unknown"` window `engines/lemonade` has right after a + /// launch reports ready. + #[test] + fn the_serve_summary_probe_waits_for_a_device_that_is_not_ready_yet() { + let mut attempts = 0; + let mut waited = Duration::ZERO; + let device = retry_service_device( + || Duration::ZERO, + || { + attempts += 1; + (attempts > 2).then(|| "vulkan gpu 0".to_owned()) + }, + |interval| waited += interval, + ); + + assert_eq!(device.as_deref(), Some("vulkan gpu 0")); + assert_eq!(attempts, 3); + assert_eq!( + waited, + SERVE_DEVICE_PROBE_INTERVAL * 2, + "the retries must pace themselves rather than spin" + ); + } + + /// And it gives up. A lemonade service whose weights are still loading can + /// stay `unknown` for minutes; the summary prints no `device:` row rather + /// than holding the terminal, which is why the row is documented as + /// optional. + #[test] + fn the_serve_summary_probe_gives_up_within_its_budget() { + let mut attempts = 0; + // Inside the budget for the first attempt, past it from then on. + let mut tick = 0; + let device = retry_service_device( + || { + tick += 1; + if tick <= 1 { + Duration::ZERO + } else { + SERVE_DEVICE_PROBE_BUDGET + } + }, + || { + attempts += 1; + None + }, + |_| {}, + ); + + assert_eq!(device, None); + assert_eq!( + attempts, 2, + "the budget must end the retries, not merely discard their results" + ); + } + /// A service whose engine cannot answer costs its own probe and nothing /// more: it is skipped, and the services after it are still probed. #[test] diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 7b41bb348d..decf1ec074 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -506,8 +506,11 @@ rocm model --verbose" /// /// Picks an engine and ROCm runtime automatically unless overridden. By default the /// server runs as a managed background service and prints a deployment summary - /// (status, endpoint, model, and smoke-test metrics); use --verbose to stream engine - /// logs in this terminal instead. When streaming, press Ctrl-D to detach and leave the + /// (status, endpoint, model, smoke-test metrics, and the backend device the engine + /// reports it is running on). The device line is read back from the running server, so + /// it is omitted when the engine cannot say yet: a server that answers is not always + /// done loading its weights. `rocm services list` shows it once it is known. Use + /// --verbose to stream engine logs in this terminal instead. When streaming, press Ctrl-D to detach and leave the /// server running, or Ctrl-C to stop it. Inspect or stop servers later with /// `rocm services`. /// @@ -1124,6 +1127,12 @@ enum ComfyuiCommand { enum ServicesCommand { /// Show currently running local model servers, and count the records that /// are no longer running. + /// + /// Each running server is asked which backend and GPU it is actually on, so + /// this takes a few seconds rather than returning instantly. The asking is + /// capped in total — about 18 seconds worst case, however many servers are + /// listed — and a server that does not answer in time simply has no device + /// line. List { /// Include failed, stopped, and old service records. #[arg(short, long)] diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index fd6e9586bf..145a4908c3 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -50,7 +50,7 @@ use rocm_engine_protocol::{ use crate::DevicePolicyArg; use crate::endpoint_keys; -use crate::engine_ipc::{engine_request, fetch_service_device}; +use crate::engine_ipc::{engine_request, fetch_service_device_briefly}; use crate::engine_runtime_select::{ EngineSelection, resolve_engine_selection, validate_engine_selection_runtime, }; @@ -1256,8 +1256,15 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // Vulkan-only GPU that ROCm/amd-smi never enumerates, even though // the server is running fine on it. Only meaningful once the // server is ready to answer, same gate as the smoke test. + // + // Asked more than once, briefly: `engines/lemonade` answers + // `"unknown"` until its own inference probe has come back, which is + // a real window right after a launch reports ready. The wait is + // bounded (`engine_ipc::SERVE_DEVICE_PROBE_BUDGET`) and `None` is a + // supported outcome — the summary then has no `device:` row, and + // `rocm services list` shows it once the engine can say. let device = (report.status == "ready") - .then(|| fetch_service_device(&paths, &selected_engine, &report.service_id)) + .then(|| fetch_service_device_briefly(&paths, &selected_engine, &report.service_id)) .flatten(); let mut notes = collect_serve_notes( diff --git a/apps/rocm/src/serve_summary.rs b/apps/rocm/src/serve_summary.rs index e0ee77df4d..3424650ff4 100644 --- a/apps/rocm/src/serve_summary.rs +++ b/apps/rocm/src/serve_summary.rs @@ -56,8 +56,19 @@ pub(crate) struct DeploymentSummary { /// `"ready"`, `"starting"`, or an existing service's status. pub status: String, /// The backend/device the engine actually selected (e.g. `"vulkan gpu 0"`, - /// `"rocm gpu 0"`), read from the engine's own `Healthcheck` response. - /// `None` when the query failed or the service was not ready to ask. + /// `"rocm gpu 0 (AMD Instinct MI300X)"`), read from the engine's own + /// `Healthcheck` response. + /// + /// **`None` is a supported outcome, and renders no `device:` row at all.** + /// A launch is `ready` once its server answers, but an engine can still be + /// unable to say which device it is on: `engines/lemonade` reports + /// `"unknown"` until its own inference probe returns, which for a model + /// still loading its weights can be minutes away. `rocm serve` retries + /// within `engine_ipc::SERVE_DEVICE_PROBE_BUDGET` and then prints the + /// summary without the row rather than making the user wait for it; + /// `rocm services list` shows it as soon as the engine can answer. So + /// neither a caller nor a test may read a missing row as a failure to + /// select a device. pub device: Option, /// True when an equivalent server was already running and nothing was spawned. pub already_running: bool, diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index 1a2eab0f4a..b02f8193ae 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -1812,6 +1812,29 @@ impl WindowsExamineInventory { .is_some_and(|d| d.driver_version.is_none() || d.driver_date.is_none()) } + /// How many of the listed adapters are recognizably AMD GPUs. + /// + /// Used only by [`pinned_gpu_name`], to answer "could the preferred + /// adapter's name belong to a card other than the pinned one?". Counts an + /// adapter when either half of [`Self::preferred_amd_display`]'s + /// recognition applies (an AMD PCI device id, or an AMD marketing name) — + /// deliberately *not* that function's last-resort "any adapter with a + /// name", which matches non-AMD displays and would read a laptop's Intel + /// panel as a second AMD GPU and drop a name that was fine. + fn amd_display_count(&self) -> usize { + self.displays + .iter() + .filter(|display| { + display + .pnp_device_id + .as_deref() + .and_then(amd_pci_device_id_from_pnp_id) + .is_some() + || gfx_target_from_amd_marketing_name(&display.name).is_some() + }) + .count() + } + fn amd_display_name(&self) -> Option { self.preferred_amd_display() .map(|display| display.name.trim()) @@ -1977,11 +2000,28 @@ impl ExamineSummary { // The covering family is the fact; it is already printed on its own // line above, so this one says what it means rather than repeating // it. - GpuSupportLevel::RocmHip => format!( - " rocm_support: GPU detected ({}) with full ROCm/HIP support; \ - install the SDK for it with `rocm install sdk`\n", - detected.unwrap_or("") - ), + // + // The advice is conditional because AGENTS.md wants a named command + // to be able to clear the condition it is printed for. On a host + // that already has a managed SDK, `rocm install sdk` clears + // nothing — it is the install that is already done — so the line + // says so and points at what is left to do instead. + // `detected_therock_family` is the installed managed runtime (see + // `detect_managed_therock_family`), as opposed to + // `compatible_therock_family`, which is what this GPU *could* run. + GpuSupportLevel::RocmHip => match self.detected_therock_family.as_deref() { + Some(installed) => format!( + " rocm_support: GPU detected ({}) with full ROCm/HIP support, and the \ + managed ROCm SDK for {installed} is already installed - serve on it with \ + `rocm serve `\n", + detected.unwrap_or("") + ), + None => format!( + " rocm_support: GPU detected ({}) with full ROCm/HIP support; \ + install the SDK for it with `rocm install sdk`\n", + detected.unwrap_or("") + ), + }, // Not folded in with `VulkanOnly`: a detection gap is not a // confirmed absence of ROCm support, and saying "Vulkan" here would // be the same false claim in the other direction. What the user can @@ -3365,6 +3405,116 @@ pub fn detect_host_gpu_summary(paths: Option<&AppPaths>) -> HostGpuSummary { detect_host_gpu_summary_fast(paths) } +/// The GPU name an engine may record at launch for a service pinned to +/// `gpu_indices`, for the `device:` row `rocm serve` and `rocm services list` +/// print (ROCMAI-550). +/// +/// `None` whenever the name cannot be attributed to the *pinned* device, which +/// is most of what this function is for. Every name probe on this host — +/// [`detect_host_gpu_summary`] on Linux, the Windows display inventory — answers +/// for the host's primary/preferred adapter, not for an ordinal. On a multi-GPU +/// host that would print `gpu 1 ()`: a row that looks specific +/// and is wrong, which is worse than the bare `gpu 1` it replaced. So the name +/// is reported only when the host has exactly **one** AMD GPU, where "the +/// primary one" and "the pinned one" cannot differ. +/// +/// The three built-in engines all call this instead of probing themselves, so +/// the rule lives in one place and `engines/aim` (which records no ordinal at +/// all) is held to it as well. +/// +/// Why not map the ordinal to a device properly: the ordinals in `gpu_indices` +/// are HIP ordinals *after* the visibility mask +/// ([`usable_amd_gpu_indices`]), so ordinal 0 under `ROCR_VISIBLE_DEVICES=1` +/// is the host's second card. Attributing a name would mean enumerating +/// devices in HIP order and re-applying that mask, and neither KFD topology +/// nodes nor DRM cards carry a marketing name — so it is a real probe to build +/// and one no lane here can verify (none of the CI hosts has two AMD GPUs). +/// Until that exists, a mask in effect is treated exactly like a second GPU: +/// no name. +#[must_use] +pub fn pinned_gpu_name(gpu_indices: &[u32]) -> Option { + gpu_name_for_single_gpu_host( + gpu_indices, + amd_gpu_count_for_naming(), + gpu_visibility_mask_in_effect(), + host_primary_gpu_name, + ) +} + +/// The decision [`pinned_gpu_name`] makes, with the host probes lifted out. +/// +/// Split out because every input here is a host fact a unit test cannot plant +/// (a second GPU, a Windows display inventory), while the rule itself — "a name +/// only when nothing else on this host could be the one that was pinned" — is +/// the whole point and is exactly what a regression would silently undo. +fn gpu_name_for_single_gpu_host( + gpu_indices: &[u32], + present_amd_gpus: Option, + mask_in_effect: bool, + name: F, +) -> Option +where + F: FnOnce() -> Option, +{ + // A pinned ordinal other than 0 cannot be the only GPU on the host, so it + // is already outside what the probes can attribute. Checked separately from + // the count so a disagreement between the two (a `--gpu 1` that got past + // validation on a host now reporting one GPU) also declines to name. + if gpu_indices.iter().any(|&index| index != 0) { + return None; + } + if mask_in_effect || present_amd_gpus != Some(1) { + return None; + } + name() +} + +/// How many AMD GPUs this host presents, for [`pinned_gpu_name`]'s +/// "could this name belong to another card?" test. `None` when it cannot be +/// established, which declines the name rather than guessing. +fn amd_gpu_count_for_naming() -> Option { + #[cfg(target_os = "linux")] + { + if !runtime_is_windows() { + // WSL2 exposes no per-device topology; the ROCm-over-dxg path + // reaches a single device, which is what `usable_amd_gpu_indices` + // reports there too. + if is_wsl_host() { + return Some(1); + } + return combine_amd_gpu_counts( + linux_kfd_gpu_node_count(), + linux_drm_amdgpu_card_count(), + ); + } + } + detect_windows_examine_inventory().map(|inventory| inventory.amd_display_count()) +} + +/// The host's primary/preferred AMD GPU name — correct only on a host with one +/// AMD GPU, which is why [`pinned_gpu_name`] is the only caller. +fn host_primary_gpu_name() -> Option { + if runtime_is_windows() { + return detect_windows_examine_inventory().and_then(|i| i.amd_display_name()); + } + detect_host_gpu_summary(None).name +} + +/// Whether a GPU visibility mask is narrowing what the engine can see, so a +/// HIP ordinal no longer indexes the host's own device order. +#[cfg(target_os = "linux")] +fn gpu_visibility_mask_in_effect() -> bool { + visibility_mask_from_env().is_some() +} + +/// Platform stub: the mask readers are Linux-only (see +/// [`visibility_mask_from_env`]), so nothing here can observe a mask. A mask +/// reader for another platform must extend this rather than leave it `false`. +#[cfg(not(target_os = "linux"))] +const fn gpu_visibility_mask_in_effect() -> bool { + false +} + #[cfg(windows)] fn detect_host_gpu_summary_fast(_paths: Option<&AppPaths>) -> HostGpuSummary { let windows_inventory = detect_windows_examine_inventory(); @@ -3949,6 +4099,99 @@ mod tests { ); } + /// The one host shape where the probes can attribute a name to the pinned + /// device: a single AMD GPU, unmasked, pinned to ordinal 0 (or to nothing, + /// which is how `engines/aim` records every launch). + #[test] + fn a_single_unmasked_gpu_is_named() { + for indices in [&[][..], &[0][..]] { + assert_eq!( + gpu_name_for_single_gpu_host(indices, Some(1), false, || Some( + "AMD Radeon Graphics".to_owned() + )), + Some("AMD Radeon Graphics".to_owned()) + ); + } + } + + /// The failure this exists to prevent: on a two-GPU host the name probe + /// answers for the primary adapter whatever ordinal was pinned, so + /// `gpu 1 ()` would be a specific-looking lie. The probe + /// must not even be called. + #[test] + fn a_second_gpu_on_the_host_withholds_the_name() { + let mut probed = false; + let name = gpu_name_for_single_gpu_host(&[1], Some(2), false, || { + probed = true; + Some("AMD Instinct MI300X".to_owned()) + }); + assert_eq!(name, None); + assert!(!probed, "a name that cannot be attributed must not be probed"); + + // Ordinal 0 on that same host is no safer: the probe answers for the + // host's primary adapter, which the mask/enumeration order need not + // agree with. + assert_eq!( + gpu_name_for_single_gpu_host(&[0], Some(2), false, || Some("AMD".to_owned())), + None + ); + } + + /// A visibility mask renumbers the devices, so ordinal 0 need not be the + /// host's first GPU and a one-device *view* is not a one-GPU host. + #[test] + fn a_visibility_mask_withholds_the_name() { + assert_eq!( + gpu_name_for_single_gpu_host(&[0], Some(1), true, || Some("AMD".to_owned())), + None + ); + } + + /// An unprobeable host count declines rather than guessing, so a platform + /// whose GPU count cannot be read reports the backend without a name. + #[test] + fn an_unknown_gpu_count_withholds_the_name() { + assert_eq!( + gpu_name_for_single_gpu_host(&[0], None, false, || Some("AMD".to_owned())), + None + ); + // And a host with no AMD GPU at all has nothing to name. + assert_eq!( + gpu_name_for_single_gpu_host(&[], Some(0), false, || Some("AMD".to_owned())), + None + ); + } + + /// Only AMD adapters count toward "is this host ambiguous?" — a laptop's + /// Intel panel alongside one Radeon is still one AMD GPU, and counting it + /// would drop a name that was correct. + #[test] + fn the_windows_display_count_counts_amd_adapters_only() { + let inventory = WindowsExamineInventory { + cpu_model: None, + system_ram_gib: None, + displays: vec![ + WindowsDisplayAdapter { + name: "Intel(R) UHD Graphics".to_owned(), + driver_version: None, + driver_date: None, + pnp_device_id: Some("PCI\\VEN_8086&DEV_9A49".to_owned()), + }, + WindowsDisplayAdapter { + name: "AMD Radeon(TM) Graphics".to_owned(), + driver_version: None, + driver_date: None, + pnp_device_id: Some("PCI\\VEN_1002&DEV_15BF".to_owned()), + }, + ], + }; + assert_eq!(inventory.amd_display_count(), 1); + assert_eq!( + inventory.amd_display_name(), + Some("AMD Radeon(TM) Graphics".to_owned()) + ); + } + /// The predicate must agree with `normalize_therock_family`, not merely /// happen to. /// @@ -5227,6 +5470,25 @@ Class Name: Display ); assert!(rendered.contains("rocm install sdk"), "{rendered}"); + // Same GPU, but the SDK it was being told to install is already there: + // the advice has to be something that is still left to do (AGENTS.md + // wants a named command to be able to clear the condition). + let mut installed = supported; + installed.detected_therock_family = Some("gfx110X-dgpu".to_owned()); + let rendered = installed.render_text(); + assert!( + rendered.contains("rocm_support: GPU detected (gfx1103) with full ROCm/HIP support"), + "{rendered}" + ); + assert!( + rendered.contains("managed ROCm SDK for gfx110X-dgpu is already installed"), + "{rendered}" + ); + assert!( + !rendered.contains("rocm install sdk"), + "an install that is already done must not be the advice: {rendered}" + ); + // No GPU found: the level is reported as unknown, and nothing about the // hardware is asserted in either direction. let missing = examine_summary_with_gpu(None, None); diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 435af13729..8b49d19273 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -289,9 +289,15 @@ Expected result: reports it in the same ` [gpu ] [()]` shape — so expect something like `rocm gpu 0 (AMD Instinct MI300X)`, with the ordinal omitted when the launch pinned none (no `--gpu`, nothing recorded in the engine's - `gpu_indices`) and the name omitted when the host probe could not read one. - `rocm services` shows the same `device` while it runs. The first word is the - half that decides this check. + `gpu_indices`) and the name omitted when the host probe could not read one or + the machine has more than one AMD GPU (where naming one could name the wrong + one). `rocm services` shows the same `device` while it runs. The first word is + the half that decides this check. +- The whole `device:` row may be missing from the deployment summary, and that + is not a failure of this check: the engine reports no device until it has + verified it can serve, which a model still loading its weights has not. Run + `rocm services list` and read the row there instead — the check is on what the + row says, not on which surface said it first. - The run does not fall back to CPU or Vulkan — a `device:` whose first word is `vulkan` fails this check even though the serve itself succeeds. - If the GPU cannot be used, the command fails with a clear error. diff --git a/docs/testing.md b/docs/testing.md index 1892bf2482..b5d15ef3c5 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -2077,6 +2077,52 @@ hardware: helper, not the call site, and the four no-GPU scenarios above run where the gate opens either way. +## The Reported Backend Device And Support Level + +ROCMAI-550 made two surfaces report what this host can actually do instead of a +static ROCm claim. Both are worth knowing about before reading a failure in +them. + +**`rocm serve` and `rocm services list` print a `device:` row** — the backend +and GPU the engine says it selected, read back over the engine's own +`Healthcheck` RPC rather than inferred from the launch plan. Every engine +renders it through `rocm_engine_protocol::format_reported_device`, so the shape +is one and the same everywhere: a backend word, then an optional ` gpu `, +then an optional ` ()`. + +**The row is optional, and a missing one is not a defect.** An engine answers +`unknown` until it can really say — `engines/lemonade` until its own inference +probe returns, which for a model still loading its weights is minutes rather +than seconds — and the CLI drops `unknown` rather than printing it. `rocm serve` +retries within `engine_ipc::SERVE_DEVICE_PROBE_BUDGET` (~6s, plus one in-flight +probe) and then prints the summary without the row; `rocm services list` shows +it once the engine can answer. So: + +- a test may assert that a row, where present, is well-formed and that the two + surfaces agree — which is what `the services list reports the engine's actual + backend device` (serve-05, serve-07) does; +- a test may **not** assert that a `ready` serve printed one. + +The GPU name is optional in the same way, and deliberately so: +`rocm_core::pinned_gpu_name` reports a name only on a host with exactly one AMD +GPU and no visibility mask, because every name probe here answers for the +host's primary adapter rather than for a HIP ordinal. On a multi-GPU host the +row is the backend and ordinal with no name — correct and unspecific, rather +than specific and wrong. No CI lane has two AMD GPUs, so that branch is covered +by unit tests in [host_gpu.rs](../crates/rocm-core/src/host_gpu.rs) only. + +**`rocm examine` prints a `rocm_support` line on every host**, at one of three +levels (`gpu_support_level`): ROCm/HIP-supported, Vulkan-only, or unknown. +Unknown covers both "nothing detected" and "a target this build's catalog does +not recognise" — conflating the latter with "confirmed to have no ROCm build" +would turn a detection gap into a claim about the hardware. `examine-31` holds +the line against the `detected_gfx_target` and `compatible_therock_family` the +same report prints, so every lane checks the level its own hardware implies: + +```bash +cargo xtask e2e -- -n '(examine-31)' +``` + ## Windows Tool Notes The TheRock SDK wheel install path should not require users to install global diff --git a/engines/aim/src/state.rs b/engines/aim/src/state.rs index 7756e40eb0..4437ddfa2f 100644 --- a/engines/aim/src/state.rs +++ b/engines/aim/src/state.rs @@ -157,8 +157,11 @@ pub(crate) fn write_running_state( // so without it its row would be the bare word `rocm` on every // host. Captured once here rather than probed per healthcheck: // `rocm services list` healthchecks every ready service on its - // render path. - "gpu_name": rocm_core::detect_host_gpu_summary(None).name, + // render path. The empty ordinal list is the literal truth — the + // container is handed a device list, not an ordinal — and + // `rocm_core::pinned_gpu_name` still declines on any host where + // more than one AMD GPU could be the one that answered. + "gpu_name": rocm_core::pinned_gpu_name(&[]), "engine_recipe": engine_recipe, "started_at_unix_ms": current_unix_millis(), }), diff --git a/engines/lemonade/src/state.rs b/engines/lemonade/src/state.rs index 4f7e61f5b1..ec8410ca9f 100644 --- a/engines/lemonade/src/state.rs +++ b/engines/lemonade/src/state.rs @@ -71,13 +71,16 @@ pub(crate) fn write_running_state( "endpoint_url": endpoint_url(&request.host, request.port), "device_policy": device_policy_name(&request.device_policy), "gpu_indices": request.gpu_indices, - // The host's GPU name, so `healthcheck_service` can name a device - // and not just a backend (ROCMAI-550). Captured here, once, rather - // than probed per healthcheck: `rocm services list` healthchecks - // every ready service on its render path. Recorded even when it is - // `null`, so the key's absence means an older state file rather - // than a probe that came back empty. - "gpu_name": rocm_core::detect_host_gpu_summary(None).name, + // The name of the GPU this service was pinned to, so + // `healthcheck_service` can name a device and not just a backend + // (ROCMAI-550). Captured here, once, rather than probed per + // healthcheck: `rocm services list` healthchecks every ready + // service on its render path. `rocm_core::pinned_gpu_name` answers + // `null` whenever the host probe cannot be attributed to the pinned + // ordinal (a second AMD GPU on the host, a visibility mask); the + // key is recorded either way, so its absence means an older state + // file rather than a probe that declined. + "gpu_name": rocm_core::pinned_gpu_name(&request.gpu_indices), "runtime_id": request.runtime_id.as_deref().unwrap_or(runtime.manifest.env_id.as_str()), "env_id": request.env_id.as_deref().unwrap_or(runtime.manifest.env_id.as_str()), "runtime_kind": "lemonade_embeddable", diff --git a/engines/vllm/src/state.rs b/engines/vllm/src/state.rs index bcb2c447d9..355f7d3345 100644 --- a/engines/vllm/src/state.rs +++ b/engines/vllm/src/state.rs @@ -184,11 +184,14 @@ pub(crate) fn write_running_state( // reports. Introspection only, like the rest of this file — never // read back to relaunch. "gpu_indices": request.gpu_indices, - // The host's GPU name, so the `device:` row names a device and not - // only a backend (ROCMAI-550). Captured once here rather than - // probed per healthcheck, because `rocm services list` - // healthchecks every ready service on its render path. - "gpu_name": rocm_core::detect_host_gpu_summary(None).name, + // The name of the GPU this service was pinned to, so the `device:` + // row names a device and not only a backend (ROCMAI-550). Captured + // once here rather than probed per healthcheck, because `rocm + // services list` healthchecks every ready service on its render + // path. `rocm_core::pinned_gpu_name` answers `null` when the host + // probe cannot be attributed to the pinned ordinal, so the row + // falls back to `rocm gpu ` rather than naming another card. + "gpu_name": rocm_core::pinned_gpu_name(&request.gpu_indices), "runtime_id": runtime.runtime_id, "requested_runtime_id": request.runtime_id, "env_id": request.env_id.as_deref().unwrap_or(runtime.env_id.as_str()), diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index 34c732c3cc..8566f2c9c1 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -464,3 +464,24 @@ Feature: GPU detection and system inspection Given a machine with a ROCm install that was not set up by the CLI When the user inspects the system in machine-readable form Then the inspection reports a current install-method name + + # ROCMAI-550. `rocm examine` answers "can this machine run ROCm?" with a + # `rocm_support` line on EVERY host -- the ROCm-capable runners and the no-GPU + # mock lane included, not only the Vulkan-only hosts the ticket started from. + # Other surfaces send people here for that answer (`lemonade_engine_note` in + # apps/rocm/src/main.rs tells an unclassifiable host to run `rocm examine`), so + # silence there is a dead end rather than a smaller report. + # + # Deliberately untagged: the line's content is keyed to what this host turns + # out to be, so the lanes disagreeing about their GPUs is the coverage rather + # than an obstacle to it. The assertion is a cross-check and not a tautology -- + # the claim is held against `detected_gfx_target` and + # `compatible_therock_family` from the same report, so a line reading "full + # ROCm/HIP support" on the mock lane, or "not supported by ROCm" on an MI300X, + # fails here. + @id:examine-reports-this-hosts-rocm-support-level + Scenario: examine-31 - System inspection says whether this machine's GPU is supported by ROCm + When the user inspects the system + Then the inspection states this host's ROCm support level + And the support level agrees with the GPU the inspection detected + And the support level leaves a command that fits this host diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 67898e4079..95dd0831b9 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -2062,3 +2062,128 @@ async fn assert_examine_driver_age_follows_the_date(world: &mut E2eWorld) { "{age}" ); } + +/// The `rocm_support` line's value, or `None` when the report has no such line. +fn rocm_support_line(output: &str) -> Option<&str> { + field_value(output, "rocm_support") +} + +/// What `rocm examine` reported as the detected GPU target and the ROCm package +/// family covering it, with the report's `` placeholder read back as +/// "absent" so the two can be reasoned about as the CLI's own +/// `gpu_support_level` inputs. +fn detected_gpu_facts(output: &str) -> (Option<&str>, Option<&str>) { + let field = |name| field_value(output, name).filter(|value| *value != ""); + ( + field("detected_gfx_target"), + field("compatible_therock_family"), + ) +} + +/// ROCMAI-550: the report answers the support question on every host, including +/// the ones where the honest answer is "this build cannot tell". +#[then("the inspection states this host's ROCm support level")] +async fn assert_examine_states_a_support_level(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + let line = rocm_support_line(output).unwrap_or_else(|| { + panic!( + "`rocm examine` printed no `rocm_support` line. Other surfaces send \ + the user here to find out whether this GPU is supported, so there \ + is no host on which the answer may be silence:\n{output}" + ) + }); + assert!( + !line.is_empty(), + "the `rocm_support` line states nothing:\n{output}" + ); +} + +/// The cross-check. The claim is judged against the two facts the same report +/// prints, not against a hardcoded expectation, so this says something on every +/// lane: the mock lane detects no GPU and must not be told it has ROCm support; +/// the GPU runners have a covering package family and must not be told their +/// card has no ROCm build. +#[then("the support level agrees with the GPU the inspection detected")] +async fn assert_support_level_agrees_with_detection(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + let line = rocm_support_line(output).expect("no rocm_support line"); + let (detected, family) = detected_gpu_facts(output); + + match (detected, family) { + // A package family covers this GPU: full ROCm/HIP support, and the + // report must not hedge it into either of the weaker answers. + (_, Some(_)) => { + assert!( + line.contains("full ROCm/HIP support"), + "a GPU with a covering ROCm package family is supported, and the \ + support line must say so: {line}\n{output}" + ); + assert!( + !line.contains("not supported by ROCm") && !line.starts_with("unknown"), + "{line}\n{output}" + ); + } + // Nothing detected at all. The honest answer is "unknown"; claiming + // either support or a working Vulkan backend would be inventing + // hardware. + (None, None) => { + assert!( + line.starts_with("unknown"), + "no GPU was detected, so neither support nor its absence can be \ + claimed: {line}\n{output}" + ); + assert!( + !line.contains("Vulkan backend can still be used"), + "no backend may be promised for a GPU that was not found: \ + {line}\n{output}" + ); + } + // Detected, with no family covering it. Both answers are legitimate + // here and which one is correct depends on the catalog: `gfx90c` is + // *known* to have no ROCm build (Vulkan-only), anything unrecognised is + // a gap in what this build knows (unknown). What is never legitimate is + // claiming ROCm support for a GPU no package family covers. + (Some(gfx), None) => { + assert!( + line.contains(gfx), + "the support line must name the target it is judging: {line}\n{output}" + ); + assert!( + line.contains("not supported by ROCm") || line.contains("unknown"), + "no ROCm package family covers {gfx}, so the line must report \ + either a Vulkan-only GPU or an unknown one: {line}\n{output}" + ); + assert!( + !line.contains("full ROCm/HIP support"), + "a GPU with no covering package family must not be reported as \ + ROCm-supported: {line}\n{output}" + ); + } + } +} + +/// AGENTS.md wants a line that tells the user what they can do, and a named +/// command that can actually clear the condition it was printed for. The +/// specific pairing checked here is the one this round fixed: a host whose +/// managed SDK is already installed must not be told to install it. +#[then("the support level leaves a command that fits this host")] +async fn assert_support_level_advice_fits(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + let line = rocm_support_line(output).expect("no rocm_support line"); + assert!( + line.contains("rocm "), + "the support line must leave the user somewhere to go: {line}\n{output}" + ); + + // `detected_therock_family` is the managed SDK that is installed, as + // opposed to `compatible_therock_family`, which is what this GPU could run. + let sdk_installed = field_value(output, "detected_therock_family") + .is_some_and(|value| value != "" && value != ""); + if sdk_installed { + assert!( + !line.contains("rocm install sdk"), + "this host already has a managed ROCm SDK, so installing one clears \ + nothing: {line}\n{output}" + ); + } +} diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index c8c3ce1472..113958ea87 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1916,8 +1916,9 @@ fn device_row_value(text: &str) -> Option<&str> { /// Lemonade has a Vulkan path, so `vulkan` from `vllm` or `aim` is wrong, /// and `unknown` is wrong from any of them; /// 3. it equals what the `rocm serve` deployment summary printed for the same -/// service, which is the row the preceding Given produced and which nothing -/// else in these scenarios looks at. +/// service — *when* that summary printed a device row at all, which is a +/// documented maybe rather than a promise (see the comment at that +/// comparison, and `serve_summary::DeploymentSummary::device`). /// The row is polled rather than read once: see [`DEVICE_ROW_DEADLINE`] for /// the window in which a correct implementation legitimately has no row yet. /// Running out of that window is an assertion naming what was last seen, not a @@ -1986,30 +1987,36 @@ async fn assert_services_list_reports_device(world: &mut E2eWorld) { // at launch; nothing else in these scenarios inspects that summary, so // without this the summary half of ROCMAI-550 has no live assertion at all. // - // The summary row is conditional by design: `serve` only asks the engine - // once the launch already reported `ready`, and a managed launch can return - // while the weights are still loading (which is why the Given then polls - // `/v1/models`). So the assertion is two-sided rather than unconditional - - // a device row must agree with this one, and its absence must be explained - // by a not-yet-ready launch rather than by the row having been dropped. + // Only when the summary HAS the row. Its absence is not a defect and must + // not be asserted against, in either direction: + // + // * `serve` asks the engine once the launch reports `ready` and retries + // only within `engine_ipc::SERVE_DEVICE_PROBE_BUDGET` (~6s), because the + // row is informational and the alternative is holding a terminal; + // * `engines/lemonade` answers `"unknown"` until its own + // `inference_verified` probe returns, which for a model still loading can + // be far longer than that - which is exactly why the Given step then + // polls `/v1/models` for up to minutes; + // * so a correct `rocm serve` can print `status ready` with no `device:` + // row, while `rocm services list` - asked later, after this step has + // waited - has one. + // + // An earlier revision asserted that a `ready` summary must carry the row. + // That is the exact false failure this note describes, and it was invisible + // only because serve-07 is xfail'd on lemonade+linux (see the comment on + // that scenario in model_serving.feature). What remains asserted is the + // half that is a real invariant: the two surfaces read one engine over one + // RPC, so where both speak they must say the same thing. let summary = world .serve_summary .as_deref() .expect("the Given step must have recorded the serve summary"); - match device_row_value(summary) { - Some(summary_device) => assert_eq!( + if let Some(summary_device) = device_row_value(summary) { + assert_eq!( summary_device, device, "`rocm serve` and `rocm services list` must report the same device \ for one service; they read it from the same Healthcheck RPC" - ), - None => assert!( - !summary - .lines() - .any(|line| line.trim_start().starts_with("status") - && line.trim_end().ends_with("ready")), - "the serve summary reported `ready` but printed no `device:` row, so \ - the row was dropped rather than deferred:\n{summary}" - ), + ); } } From c65224a5385b31acde666564e860c4d05e9aab43 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 6 Nov 2026 09:06:45 +0200 Subject: [PATCH 11/37] fix(examine): point the unknown-tier advice at a surface that shows the row The note promised that `rocm serve --engine lemonade` reports the backend on its `device:` row, while this PR's own docs say that row can be absent: the serve summary reads the device once, right after `ready`, and lemonade answers `unknown` until its own inference probe returns. The advice now names `rocm services list`, which shows the row once it is known, so the command can actually clear the condition it is printed for. The reasoning is recorded beside the arm rather than left implied. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/host_gpu/render.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index d9e4ff32c3..476cca612e 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -69,6 +69,14 @@ impl ExamineSummary { detected.unwrap_or("") ), }, + // The command names `rocm services list` rather than the serve + // summary on purpose. The summary reads the device once, right + // after `ready`, and lemonade answers `unknown` until its own + // inference probe returns — so the row it prints is legitimately + // absent for a short window. Pointing at the surface that shows it + // *once known* keeps the advice able to clear the condition it is + // printed for, which naming the summary would not. + // // Not folded in with `VulkanOnly`: a detection gap is not a // confirmed absence of ROCm support, and saying "Vulkan" here would // be the same false claim in the other direction. What the user can @@ -77,8 +85,9 @@ impl ExamineSummary { // thing on this host that can actually settle the question. GpuSupportLevel::Unknown => format!( " rocm_support: unknown - {}, so this build can confirm neither \ - ROCm support nor its absence; `rocm serve --engine \ - lemonade` reports the backend it picked on its `device:` row\n", + ROCm support nor its absence; serve a model with `rocm serve \ + --engine lemonade` and read the backend it picked from \ + the `device:` row of `rocm services list`\n", match detected { Some(gfx) => format!("GPU target {gfx} is not in this build's catalog"), None => "no AMD GPU was detected".to_owned(), From 34845bf6e46de3d75e12c5b67b44ff472440b1e9 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Mon, 9 Nov 2026 12:01:32 +0200 Subject: [PATCH 12/37] fix(examine): stop sending a GPU-less host to a command serve would refuse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `Unknown` tier covers two different hosts and was giving both the same advice. A detected-but-unrecognised gfx target has a GPU, so `rocm serve` runs and the engine's own `device:` row settles what this catalog could not. A host with nothing detected is refused outright by the no-usable-GPU pre-flight under the default `gpu_required` policy, so naming that command was advice that could not clear the condition it was printed for (AGENTS.md section 3). The no-detection arm now points at `rocm diagnose`, whose `fix-3-rocm-kernel`, `fix-4-render-group` and `fix-5-amdgpu-load` entries are exactly the reasons a present GPU goes undetected. The sibling arm keeps the serve advice. `no_gpu_advice_names_a_command_that_host_can_run` asserts the behaviour rather than the wording, as asked: it pins the tier through `gpu_support_level` and then checks the rendered line does not name `rocm serve`. Falsified by reverting the arm — the test fails with the offending line quoted. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/host_gpu/render.rs | 118 ++++++++++++++++++++---- 1 file changed, 101 insertions(+), 17 deletions(-) diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index 476cca612e..ec2dedb9c2 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -83,16 +83,32 @@ impl ExamineSummary { // do is let an engine decide and then read back what it chose - // which is the `device:` row this same ticket added, and the only // thing on this host that can actually settle the question. - GpuSupportLevel::Unknown => format!( - " rocm_support: unknown - {}, so this build can confirm neither \ - ROCm support nor its absence; serve a model with `rocm serve \ - --engine lemonade` and read the backend it picked from \ - the `device:` row of `rocm services list`\n", - match detected { - Some(gfx) => format!("GPU target {gfx} is not in this build's catalog"), - None => "no AMD GPU was detected".to_owned(), - } - ), + // Two different hosts reach `Unknown`, and they need different + // advice because only one of them can serve anything. + // + // A detected-but-unrecognised target means a GPU *is* present, so + // `rocm serve` runs and the engine's own `device:` row settles what + // this catalog could not. With nothing detected, that same command + // is refused outright by the no-usable-GPU pre-flight under the + // default `gpu_required` policy — naming it would be advice that + // cannot clear the condition it is printed for (AGENTS.md section 3). + // `rocm diagnose` is the command that can: its `fix-3-rocm-kernel`, + // `fix-4-render-group` and `fix-5-amdgpu-load` entries are exactly + // the reasons a present GPU goes undetected. + GpuSupportLevel::Unknown => match detected { + Some(gfx) => format!( + " rocm_support: unknown - GPU target {gfx} is not in this \ + build's catalog, so this build can confirm neither ROCm \ + support nor its absence; serve a model with `rocm serve \ + --engine lemonade` and read the backend it picked \ + from the `device:` row of `rocm services list`\n" + ), + None => " rocm_support: unknown - no AMD GPU was detected, so \ + this build can confirm neither ROCm support nor its \ + absence; run `rocm diagnose` to find out why no GPU is \ + visible\n" + .to_owned(), + }, } } @@ -502,7 +518,18 @@ mod tests { #[test] fn examine_answers_the_support_question_the_unknown_engine_note_sends_users_here_for() { - for detected in [None, Some("gfx803")] { + // Both `Unknown` sub-cases must answer, and both must leave somewhere + // to go — but not the same somewhere. A detected-but-unrecognised + // target has a GPU, so `rocm serve` runs and its `device:` row settles + // the backend. With nothing detected that command is refused by the + // no-usable-GPU pre-flight, so the only honest next step is finding out + // why the GPU is invisible. See + // `no_gpu_advice_names_a_command_that_host_can_run`, which pins that + // distinction against the behaviour rather than the wording. + for (detected, expected_command) in [ + (None, "rocm diagnose"), + (Some("gfx803"), "rocm serve --engine lemonade"), + ] { let rendered = examine_summary_with_gpu(detected, None).render_text(); let line = rendered .lines() @@ -516,14 +543,71 @@ mod tests { // Says which level it is, and leaves a command that can settle it. assert!(line.contains("unknown"), "{line}"); assert!( - line.contains("rocm serve --engine lemonade"), - "an `unknown` answer still has to leave somewhere to go: {line}" - ); - assert!( - line.contains("device:"), - "and name where the real backend shows up: {line}" + line.contains(expected_command), + "an `unknown` answer still has to leave somewhere to go, and it \ + has to be somewhere this host can actually go: {line}" ); } + + // The detected sub-case also names where the backend shows up; the + // undetected one has no server to read it from. + let unrecognised = examine_summary_with_gpu(Some("gfx803"), None).render_text(); + assert!( + unrecognised.contains("device:"), + "a host that can serve must be told where the real backend shows up: \ + {unrecognised}" + ); + } + + /// The advice a host is given must name a command that host can actually + /// run — asserted against the serve pre-flight's own predicate, not against + /// the wording. + /// + /// `rocm serve` refuses on a host with no usable AMD GPU under the default + /// `gpu_required` policy, so telling an undetected host to serve a model is + /// advice that cannot clear the condition it is printed for. Both `Unknown` + /// sub-cases are driven here because they differ precisely on this: a + /// detected-but-unrecognised target has a GPU and may serve, nothing + /// detected may not. + /// + /// Written against `gpu_support_level` + the rendered line rather than a + /// string match on the sentence: a reworded message that still sent a + /// GPU-less host to `rocm serve` would pass a wording assertion and fail + /// this one. + #[test] + fn no_gpu_advice_names_a_command_that_host_can_run() { + let undetected = examine_summary_with_gpu(None, None).render_text(); + let line = undetected + .lines() + .find(|l| l.trim_start().starts_with("rocm_support:")) + .expect("every host gets a support line"); + assert_eq!( + gpu_support_level(None, None), + GpuSupportLevel::Unknown, + "precondition: nothing detected is the Unknown tier" + ); + assert!( + !line.contains("rocm serve"), + "`rocm serve` is refused on a host with no usable AMD GPU, so it \ + cannot be the next step offered to one: {line}" + ); + assert!( + line.contains("rocm diagnose"), + "dropping the unusable command must still leave a usable one: {line}" + ); + + // The sibling sub-case keeps the serve advice, because that host has a + // GPU and the engine's own device row is what settles the question. + let unrecognised = examine_summary_with_gpu(Some("gfx-not-in-catalog"), None).render_text(); + let line = unrecognised + .lines() + .find(|l| l.trim_start().starts_with("rocm_support:")) + .expect("every host gets a support line"); + assert!( + line.contains("rocm serve"), + "a detected GPU can serve, so the advice that settles its backend \ + must not be withheld: {line}" + ); } #[test] From 8ffeca68127d295430be98e2eb2e503f505e5ff8 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 10 Nov 2026 09:35:28 +0200 Subject: [PATCH 13/37] test(serve): show the Lemonade advice names a command this host is not refused `rocm install sdk` and `rocm examine` tell a gfx90c APU to run `rocm serve --engine lemonade`, but nothing showed serve's no-usable-GPU pre-flight accepts that host. It does: a zero KFD compute-node count falls back to the DRM amdgpu card count, so the host the refusal was printed on resolves ordinal 0. Pin that composition off one planted /sys, in a unit test and in an e2e scenario that runs the real binary down the route the CLI sends the user. Also close the test gaps review 5826143385 named: the positive `device:` row and the `unknown` filter now have assertions that run in default CI, the endpoint key file's scoping is pinned, and the examine support-line check asserts the specific command each tier owes instead of any `rocm ` word. Windows reads the display inventory once per `pinned_gpu_name` instead of twice, and the docs say the engine *reports* a device rather than naming the one it observed. Signed-off-by: Roman Sirokov --- README.md | 12 +- apps/rocm/src/engine_ipc.rs | 86 +++++++++++- apps/rocm/src/services_text.rs | 129 +++++++++++++++++- apps/rocm/src/therock/family.rs | 13 ++ crates/rocm-core/src/host_gpu.rs | 25 +++- .../rocm-core/src/host_gpu/gpu_visibility.rs | 85 ++++++++++++ crates/rocm-core/src/host_gpu/linux.rs | 36 ++++- crates/rocm-core/src/host_gpu/render.rs | 48 +++++-- docs/manual-testing.md | 3 +- engines/lemonade/src/device_policy.rs | 14 +- .../features/model_serving.feature | 24 ++++ tests/e2e-cucumber/tests/e2e/examine_steps.rs | 59 ++++++-- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 124 +++++++++++++++-- 13 files changed, 594 insertions(+), 64 deletions(-) diff --git a/README.md b/README.md index a5a2f518e6..1aff586922 100644 --- a/README.md +++ b/README.md @@ -1024,12 +1024,14 @@ the running service, or stop it first to serve on the new port. By default the server runs in the background under ROCm CLI's supervision and prints a deployment summary: a progress indicator while it starts, then a table -with the status, the engine and the backend device it actually selected (for +with the status, the engine and the backend device the engine reports (for example `rocm gpu 0 (AMD Instinct MI300X)`, or `vulkan (AMD Radeon Graphics)` -where the engine pinned no GPU ordinal — read back from the running engine), the -full inference endpoint, the API-qualified model name, and a quick smoke test -(time to first token and approximate tokens per second). `rocm services` reports -the same `device` for each running service. +where the engine pinned no GPU ordinal). It is read back from the running +engine, so it is that engine's own account of the backend it was launched on +rather than an independent measurement of the hardware. The summary also carries +the full inference endpoint, the API-qualified model name, and a quick smoke +test (time to first token and approximate tokens per second). `rocm services` +reports the same `device` for each running service. The device line is omitted when the engine cannot say yet. A server is reported ready as soon as it answers, which can be before it has finished loading the diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index 1f3876f753..f8603d13ff 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -321,11 +321,23 @@ pub(crate) fn fetch_service_device( endpoint_key_file.as_deref(), ) .ok()?; - // `"unknown"` is the engines' own device value when no state is on disk for - // this service id yet (see `engines/vllm/src/state.rs` and - // `engines/lemonade/src/lib.rs`), which is not more informative than - // showing nothing. - (!response.device.is_empty() && response.device != "unknown").then_some(response.device) + device_worth_reporting(response.device) +} + +/// The engine's reported device, or `None` when it said nothing usable. +/// +/// `"unknown"` is the engines' own device value when no state is on disk for +/// this service id yet (see `engines/vllm/src/state.rs` and +/// `engines/lemonade/src/lib.rs`), and it is not more informative than showing +/// nothing — a `device: unknown` row claims the CLI asked and learned +/// something, which is the opposite of what happened. +/// +/// A named function rather than an inline `then_some` because the only caller +/// is an RPC to a live engine: in default CI that call never succeeds, so the +/// filter is unreachable there and deleting it would be invisible until a +/// freshly-launched Lemonade service printed `device: unknown` on a GPU lane. +fn device_worth_reporting(device: String) -> Option { + (!device.trim().is_empty() && device != "unknown").then_some(device) } /// How long `rocm serve`'s deployment summary may keep asking the engine which @@ -718,6 +730,70 @@ mod tests { use super::*; use crate::tests::test_paths; + /// The `unknown` filter, which nothing else can reach: its caller is an RPC + /// to a live engine, so in default CI it never runs and deleting it stays + /// green. + #[test] + fn a_device_the_engine_does_not_know_is_not_reported_as_one() { + for nothing in ["unknown", "", " "] { + assert_eq!( + device_worth_reporting(nothing.to_owned()), + None, + "`{nothing}` is the engine saying it has no answer yet, and a \ + `device:` row would claim otherwise" + ); + } + for device in ["rocm gpu 0", "vulkan (AMD Radeon Graphics)"] { + assert_eq!( + device_worth_reporting(device.to_owned()), + Some(device.to_owned()) + ); + } + } + + /// The endpoint key file reaches a built-in engine through the process + /// environment, and is gone again afterwards. + /// + /// Both halves matter and neither was covered. Without the variable a + /// `--require-api-key` service's own `/v1/models` probe answers 401, the + /// engine reports the service as unlisted, and the `device:` row silently + /// vanishes for exactly the protected services. Without the restore, one + /// service's key would leak into the next service's probe in the same + /// `rocm services list` — `with_scoped_builtin_engine_env` mutates + /// process-wide state, which is why the probes are serialized in the first + /// place. + #[test] + fn the_endpoint_key_file_is_scoped_to_one_builtin_engine_call() { + let key = rocm_engine_protocol::ENDPOINT_API_KEY_FILE_ENV; + // SAFETY-adjacent: the scoped setter takes the same process-wide lock + // this test observes through, so no other engine call can interleave. + let before = std::env::var_os(key); + let seen = with_scoped_builtin_engine_env( + None, + None, + Some(Path::new("/tmp/rocm-e2e-endpoint-key")), + || std::env::var_os(key), + ); + assert_eq!( + seen.as_deref(), + Some(std::ffi::OsStr::new("/tmp/rocm-e2e-endpoint-key")), + "the engine must be able to read the key file path it was given" + ); + assert_eq!( + std::env::var_os(key), + before, + "the key must not outlive the call it was scoped to" + ); + + // And nothing is set when no key file was supplied: a loopback service + // has none, and inventing one would point the engine at a file that + // does not exist. + assert_eq!( + with_scoped_builtin_engine_env(None, None, None, || std::env::var_os(key)), + before + ); + } + /// Every service inside the budget is probed and reported. #[test] fn service_device_probes_run_while_the_budget_holds() { diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index cbd64a9cab..e17f4ce77c 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -77,7 +77,7 @@ //! classification respectively — stay in `main.rs`'s shared `mod tests` //! instead. -use std::collections::VecDeque; +use std::collections::{HashMap, VecDeque}; use std::fmt::Write as _; use std::fs; use std::io::{self, BufRead, Read as _, Seek as _, SeekFrom}; @@ -379,6 +379,23 @@ fn write_past_attempts_hint(output: &mut String, past_attempts: usize, newest_id /// bounded and flat in the number of services, which is what makes it /// acceptable there. pub(crate) fn render_services_text(paths: &AppPaths, all: bool) -> Result { + render_services_text_with_devices(paths, all, |ready| fetch_service_devices(paths, ready)) +} + +/// [`render_services_text`] with the live device probe injected. +/// +/// Split out because the probe is the one thing a unit test cannot have: it is +/// a `Healthcheck` RPC to a running engine, so in default CI (no GPU, no +/// servers) it answers nothing and the `device:` row is never rendered at all — +/// deleting the row would stay green everywhere except the GPU lanes, which is +/// exactly the gap this closes. The seam takes only the probe, not the row: +/// which services are offered to it (`status == "ready"`) and where the row +/// lands both stay in the shared body, so a test can pin them. +fn render_services_text_with_devices( + paths: &AppPaths, + all: bool, + probe_devices: impl FnOnce(Vec<(&str, &str)>) -> HashMap, +) -> Result { // Counts describe the WHOLE registry, not the rows this view keeps. The // default view hides records that are not live, and counting the filtered // list made the header say "none ready" on a host whose only local servers @@ -434,12 +451,12 @@ pub(crate) fn render_services_text(paths: &AppPaths, all: bool) -> Result Result<()> { + use std::io::{Read, Write}; + use std::net::TcpListener; + + let (root, paths) = test_paths("services-list-device"); + paths.ensure()?; + let listener = TcpListener::bind(("127.0.0.1", 0))?; + let ready_port = listener.local_addr()?.port(); + let server = thread::spawn(move || -> Result<()> { + for _ in 0..2 { + let (mut stream, _) = listener.accept()?; + stream.set_read_timeout(Some(Duration::from_secs(2)))?; + let mut request = [0_u8; 512]; + let read = stream.read(&mut request)?; + let request = String::from_utf8_lossy(&request[..read]).into_owned(); + let body = if request.starts_with("POST /v1/chat/completions ") { + r#"{"choices":[{"message":{"content":"ok"}}]}"# + } else { + r#"{"data":[{"id":"Qwen/Qwen3.5"}]}"# + }; + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + )?; + } + Ok(()) + }); + let current_pid = std::process::id(); + for (service_id, status, port) in [ + ("svc-ready", "ready", ready_port), + ("svc-starting", "starting", 11438_u16), + ] { + let mut record = ManagedServiceRecord::new( + &paths, + service_id, + "vllm", + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + port, + "managed", + current_pid, + Some("therock-release".to_owned()), + None, + Some("gpu_required".to_owned()), + ); + record.status = status.to_owned(); + record.write()?; + } + + let mut offered: Vec<(String, String)> = Vec::new(); + let rendered = render_services_text_with_devices(&paths, false, |ready| { + offered = ready + .iter() + .map(|(id, engine)| ((*id).to_owned(), (*engine).to_owned())) + .collect(); + HashMap::from([( + "svc-ready".to_owned(), + "rocm gpu 0 (AMD Instinct MI300X)".to_owned(), + )]) + })?; + server + .join() + .expect("fake models server should not panic")?; + let _ = fs::remove_dir_all(root); + + assert_eq!( + offered, + [("svc-ready".to_owned(), "vllm".to_owned())], + "only a service ready to answer may be probed" + ); + assert!( + rendered.contains(" device: rocm gpu 0 (AMD Instinct MI300X)\n"), + "the device the engine reported must reach the list:\n{rendered}" + ); + assert!( + rendered + .contains(" engine: vllm\n device: rocm gpu 0 (AMD Instinct MI300X)\n model: "), + "the device belongs with the engine that selected it, above the \ + model:\n{rendered}" + ); + // The service the probe said nothing about keeps no row of its own. + assert_eq!( + rendered.matches(" device: ").count(), + 1, + "only the service with a reported device gets a row:\n{rendered}" + ); + Ok(()) + } + /// The default view lists only live servers. The counts used to be taken /// from that already-filtered list, so a host whose local servers had all /// failed was told "none ready" and shown an empty list - the records were diff --git a/apps/rocm/src/therock/family.rs b/apps/rocm/src/therock/family.rs index f8659bb67f..efcfaea7ce 100644 --- a/apps/rocm/src/therock/family.rs +++ b/apps/rocm/src/therock/family.rs @@ -186,6 +186,19 @@ pub(super) fn resolve_family( /// on a host with no ROCm, and fail under the no-CPU-fallback policy. Naming /// the engine makes the sentence true whatever the config says, because the /// explicit flag out-ranks every other selection source. +/// +/// **The named command is not refused on the host that is shown it.** `serve`'s +/// no-usable-GPU pre-flight is `host_has_no_usable_gpu` (`serve_cmd.rs`) over +/// `rocm_core::usable_amd_gpu_indices`, and on Linux that composes the KFD +/// compute-node count with the DRM `amdgpu` card count: a *zero* KFD count +/// falls back to DRM rather than refusing. A gfx90c APU whose kernel exposes no +/// compute node can only have been named by the DRM `ip_discovery` read, which +/// walks the same `card*/device` entries that count — so this branch is +/// reachable only on a host with at least one such card, and the pre-flight +/// therefore sees ordinal 0. `rocm_core`'s +/// `a_gfx90c_host_the_kernel_gives_no_compute_node_still_has_a_usable_gpu` +/// drives both halves off one planted `/sys`, and serve-50 runs the real binary +/// against that same host shape on a GPU-less lane. fn unsupported_host_gpu_message(raw_arch: &str) -> String { format!( "no ROCm SDK build exists for the detected GPU target `{raw_arch}`, so \ diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index cb18f3028d..4eaf10c80e 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -449,11 +449,20 @@ pub fn detect_host_gpu_summary(paths: Option<&AppPaths>) -> HostGpuSummary { /// no name. #[must_use] pub fn pinned_gpu_name(gpu_indices: &[u32]) -> Option { + // One Windows display-inventory read per call, shared by the two questions + // that need it. Both the count and the name come from the same `pnputil` / + // CIM query, and asking twice meant two PowerShell/WMI probes on the serve + // launch path. The read is eager rather than lazy only on the platform that + // uses it: off Windows nothing is probed here at all, and on Windows the + // count already needed it before any of the decisions below could run. + let windows_inventory = runtime_is_windows() + .then(detect_windows_examine_inventory) + .flatten(); gpu_name_for_single_gpu_host( gpu_indices, - amd_gpu_count_for_naming(), + amd_gpu_count_for_naming(windows_inventory.as_ref()), gpu_visibility_mask_in_effect(), - host_primary_gpu_name, + || host_primary_gpu_name(windows_inventory.as_ref()), ) } @@ -488,7 +497,11 @@ where /// How many AMD GPUs this host presents, for [`pinned_gpu_name`]'s /// "could this name belong to another card?" test. `None` when it cannot be /// established, which declines the name rather than guessing. -fn amd_gpu_count_for_naming() -> Option { +/// +/// `windows_inventory` is the display listing the caller already read, or +/// `None` off Windows (and when the listing failed); see [`pinned_gpu_name`] +/// for why it is passed in rather than read here. +fn amd_gpu_count_for_naming(windows_inventory: Option<&WindowsExamineInventory>) -> Option { #[cfg(target_os = "linux")] { if !runtime_is_windows() { @@ -504,14 +517,14 @@ fn amd_gpu_count_for_naming() -> Option { ); } } - detect_windows_examine_inventory().map(|inventory| inventory.amd_display_count()) + windows_inventory.map(WindowsExamineInventory::amd_display_count) } /// The host's primary/preferred AMD GPU name — correct only on a host with one /// AMD GPU, which is why [`pinned_gpu_name`] is the only caller. -fn host_primary_gpu_name() -> Option { +fn host_primary_gpu_name(windows_inventory: Option<&WindowsExamineInventory>) -> Option { if runtime_is_windows() { - return detect_windows_examine_inventory().and_then(|i| i.amd_display_name()); + return windows_inventory.and_then(WindowsExamineInventory::amd_display_name); } detect_host_gpu_summary(None).name } diff --git a/crates/rocm-core/src/host_gpu/gpu_visibility.rs b/crates/rocm-core/src/host_gpu/gpu_visibility.rs index 9a7be9a7f9..8d8fdcacf9 100644 --- a/crates/rocm-core/src/host_gpu/gpu_visibility.rs +++ b/crates/rocm-core/src/host_gpu/gpu_visibility.rs @@ -324,6 +324,91 @@ pub(super) fn visibility_mask_in_effect() -> bool { #[cfg(test)] mod tests { use super::*; + use crate::gfx_target::gfx_target_has_no_rocm_build; + use crate::test_support::temp_app_paths; + use std::fs; + + /// ROCMAI-550, and the one claim the Lemonade remediation rests on: the host + /// that gets told `rocm serve --engine lemonade` is a host + /// `rocm serve` will actually launch on. + /// + /// The shape is a Renoir/Cezanne laptop: the Vega iGPU is on the bus and + /// bound to `amdgpu`, but the kernel's KFD topology lists no compute node + /// for it, so `gfx90c` can only have been named by the DRM `ip_discovery` + /// read. `apps/rocm/src/therock/family.rs` prints + /// `unsupported_host_gpu_message` for exactly that target, and + /// `apps/rocm/src/serve_cmd.rs`'s `host_has_no_usable_gpu` — + /// `matches!(visible, Some([]))` over [`usable_amd_gpu_indices`] — decides + /// whether the command it names is refused. Both halves are driven here off + /// **one** planted `/sys`, so this is the composition and not two + /// independent claims: KFD contributes `Some(0)`, DRM contributes `Some(1)`, + /// and [`combine_amd_gpu_counts`]'s zero-KFD fallback is what turns that + /// into a usable ordinal. Delete that fallback and the detection half still + /// passes while the pre-flight half goes red, which is the failure the + /// reviewer asked to see ruled out. + /// + /// Planted rather than run on hardware because no CI lane has a gfx90c APU; + /// see `linux::linux_drm_amdgpu_card_count_in`. + #[test] + fn a_gfx90c_host_the_kernel_gives_no_compute_node_still_has_a_usable_gpu() { + let (root, _paths) = temp_app_paths("gfx90c-drm-only-host"); + let drm = root.join("sys/class/drm"); + let device = drm.join("card0/device"); + fs::create_dir_all(&device).expect("plant the DRM card"); + fs::write(device.join("vendor"), "0x1002\n").expect("plant the PCI vendor"); + // GC 9.0.12 is the Vega iGPU in Renoir / Lucienne / Cezanne / Barcelo. + let gc = device.join("ip_discovery/die/0/GC/0"); + fs::create_dir_all(&gc).expect("plant the GC IP block"); + for (file, value) in [("major", "9"), ("minor", "0"), ("revision", "12")] { + fs::write(gc.join(file), value).expect("plant the GC IP version"); + } + // A connector sub-node, which is not a card and must not be counted. + fs::create_dir_all(drm.join("card0-eDP-1")).expect("plant the DRM connector"); + // The KFD topology is readable and lists only the CPU node: this host's + // iGPU has no compute node, which is the whole premise. + let nodes = root.join("sys/class/kfd/kfd/topology/nodes"); + fs::create_dir_all(nodes.join("0")).expect("plant the KFD CPU node"); + fs::write( + nodes.join("0").join("properties"), + "cpu_cores_count 16\nsimd_count 0\ngfx_target_version 0\n", + ) + .expect("plant the KFD CPU node properties"); + + let detected = super::super::linux::detect_linux_drm_ip_discovery_gfx_target_in(&drm); + let kfd = super::super::linux::linux_kfd_gpu_node_count_in(&nodes); + let cards = super::super::linux::linux_drm_amdgpu_card_count_in(&drm); + fs::remove_dir_all(&root).ok(); + + // Half one: this is a host `unsupported_host_gpu_message` speaks for. + assert_eq!( + detected.as_deref(), + Some("gfx90c"), + "the DRM ip-discovery read is the only route left on a host with no \ + KFD compute node; without it there is no detected target and the \ + Lemonade advice is never printed" + ); + assert!(gfx_target_has_no_rocm_build("gfx90c")); + + // Half two: and `rocm serve`'s pre-flight does not refuse it. + assert_eq!( + (kfd, cards), + (Some(0), Some(1)), + "the kernel must report no compute node and the bus exactly one card" + ); + let present = combine_amd_gpu_counts(kfd, cards); + let usable = usable_amd_gpu_indices_from(present.expect("GPU presence"), None); + assert_eq!( + usable, + Some(vec![0]), + "serve pins HIP ordinal 0 on this host" + ); + assert!( + !matches!(usable.as_deref(), Some([])), + "`serve_cmd::host_has_no_usable_gpu` is `matches!(visible, Some([]))`; \ + anything else here means the advised command is refused for the very \ + condition it was printed to clear" + ); + } fn wsl_summary(librocdxg_in_cache: bool, on_ld_library_path: bool) -> super::super::WslSummary { super::super::WslSummary { diff --git a/crates/rocm-core/src/host_gpu/linux.rs b/crates/rocm-core/src/host_gpu/linux.rs index 701071d859..47dd97682d 100644 --- a/crates/rocm-core/src/host_gpu/linux.rs +++ b/crates/rocm-core/src/host_gpu/linux.rs @@ -204,7 +204,21 @@ fn natural_node_order(name: &str) -> (u64, String) { /// DRM class dir can't be read; `Some(0)` when it is readable with no AMD card. #[cfg(target_os = "linux")] pub(super) fn linux_drm_amdgpu_card_count() -> Option { - let entries = fs::read_dir(host_path("/sys/class/drm")).ok()?; + linux_drm_amdgpu_card_count_in(&host_path("/sys/class/drm")) +} + +/// The DRM card count, against a caller-supplied `/sys/class/drm`. +/// +/// Split out for the same reason as `linux_kfd_gpu_node_count_in`: this half of +/// the serve pre-flight's "is there a GPU here" answer is what carries a host +/// whose KFD reports no compute node — the gfx90c APUs +/// `unsupported_host_gpu_message` sends to Lemonade among them — and no CI lane +/// has that hardware. Planting the directory is the only way the composed +/// answer can be asserted at all; see +/// `a_gfx90c_host_the_kernel_gives_no_compute_node_still_has_a_usable_gpu`. +#[cfg(any(target_os = "linux", test))] +pub(super) fn linux_drm_amdgpu_card_count_in(class_dir: &Path) -> Option { + let entries = fs::read_dir(class_dir).ok()?; let count = entries .flatten() .filter(|entry| { @@ -236,7 +250,7 @@ pub(crate) fn linux_kfd_gpu_node_count() -> Option { /// branch is driven by tests -- the unreadable branches key off the real /// `/dev/kfd`, which a planted directory cannot stand in for. #[cfg(any(target_os = "linux", test))] -fn linux_kfd_gpu_node_count_in(nodes_dir: &Path) -> Option { +pub(super) fn linux_kfd_gpu_node_count_in(nodes_dir: &Path) -> Option { match fs::read_dir(nodes_dir) { Ok(entries) => Some( entries @@ -305,8 +319,22 @@ fn kfd_gfx_target_version_is_gpu(value: &str) -> bool { #[cfg(target_os = "linux")] fn detect_linux_drm_ip_discovery_gfx_target() -> Option { - let drm_dir = host_path("/sys/class/drm"); - let entries = fs::read_dir(drm_dir).ok()?; + detect_linux_drm_ip_discovery_gfx_target_in(&host_path("/sys/class/drm")) +} + +/// The DRM ip-discovery target read, against a caller-supplied `/sys/class/drm`. +/// +/// Split out for the same reason as [`linux_drm_amdgpu_card_count_in`], and +/// against the *same* directory on purpose: on a host whose KFD exposes no +/// compute node this is the only route left that can name a target, so it is +/// what puts a GPU in front of `unsupported_host_gpu_message` — and it walks +/// `card*/device` under `is_amdgpu_device`, exactly the cards that count then +/// hands to the serve pre-flight. Pointing one planted tree at both is what +/// makes "the host that gets the Lemonade advice is a host serve accepts" +/// assertable rather than argued. +#[cfg(any(target_os = "linux", test))] +pub(super) fn detect_linux_drm_ip_discovery_gfx_target_in(class_dir: &Path) -> Option { + let entries = fs::read_dir(class_dir).ok()?; for entry in entries.flatten() { let card_path = entry.path(); let Some(card_name) = card_path.file_name().and_then(|value| value.to_str()) else { diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index de83cb68c6..be70236505 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -95,18 +95,21 @@ impl ExamineSummary { // `rocm diagnose` is the command that can: its `fix-3-rocm-kernel`, // `fix-4-render-group` and `fix-5-amdgpu-load` entries are exactly // the reasons a present GPU goes undetected. + // + // Kept to one sentence each: the level word already says the CLI + // cannot answer, so spelling out "can confirm neither ROCm support + // nor its absence" in both arms was length without information. + // The distinction it was carrying lives in this comment and in + // `GpuSupportLevel`'s own docs, which is where it is load-bearing. GpuSupportLevel::Unknown => match detected { Some(gfx) => format!( - " rocm_support: unknown - GPU target {gfx} is not in this \ - build's catalog, so this build can confirm neither ROCm \ - support nor its absence; serve a model with `rocm serve \ - --engine lemonade` and read the backend it picked \ - from the `device:` row of `rocm services list`\n" + " rocm_support: unknown - {gfx} is not in this build's \ + catalog; serve with `rocm serve --engine lemonade`, \ + then read the `device:` row of `rocm services list` for the \ + backend it used\n" ), - None => " rocm_support: unknown - no AMD GPU was detected, so \ - this build can confirm neither ROCm support nor its \ - absence; run `rocm diagnose` to find out why no GPU is \ - visible\n" + None => " rocm_support: unknown - no AMD GPU was detected; run \ + `rocm diagnose` to find out why none is visible\n" .to_owned(), }, } @@ -272,6 +275,29 @@ mod tests { assert!(rendered.contains("model_cache_entries: 3")); } + /// The exact placeholders the report prints for the three GPU fields when + /// it has no value, pinned because they are read back as data. + /// + /// examine-35 decides which support tier a host is in by filtering these + /// out of `detected_gfx_target`, `compatible_therock_family` and + /// `detected_therock_family`. Reword one and that scenario stops seeing an + /// absent value as absent: a lane would be classified into the wrong tier + /// and its tier-specific command assertion would silently stop describing + /// the host it ran on. The two strings differ on purpose — "not detected" + /// is about a managed install, "unknown" about a probe — so neither may + /// stand in for the other. + #[test] + fn examine_render_spells_its_absent_value_placeholders() { + let rendered = examine_summary_with_gpu(None, None).render_text(); + for field in [ + "detected_gfx_target: ", + "compatible_therock_family: ", + "detected_therock_family: ", + ] { + assert!(rendered.contains(field), "missing `{field}`:\n{rendered}"); + } + } + fn examine_summary_with_gpu( detected_gfx_target: Option<&str>, compatible_therock_family: Option<&str>, @@ -660,9 +686,7 @@ mod tests { let unknown = examine_summary_with_gpu(Some("gfx803"), None); let rendered = unknown.render_text(); assert!( - rendered.contains( - "rocm_support: unknown - GPU target gfx803 is not in this build's catalog" - ), + rendered.contains("rocm_support: unknown - gfx803 is not in this build's catalog"), "{rendered}" ); assert!( diff --git a/docs/manual-testing.md b/docs/manual-testing.md index e74fa79950..d82a5d666e 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -306,7 +306,8 @@ rocm serve qwen --engine lemonade --device gpu_required --foreground --port 1143 Expected result: - The engine uses ROCm GPU execution. The deployment summary's `device:` row - reports what the engine actually selected, so check it by reading that row + is the engine's own account of the backend it was launched on (Lemonade reads + back the `backend_requested` it recorded), so check it by reading that row rather than inferring it from the absence of a fallback warning. Every engine reports it in the same ` [gpu ] [()]` shape — so expect something like `rocm gpu 0 (AMD Instinct MI300X)`, with the ordinal omitted diff --git a/engines/lemonade/src/device_policy.rs b/engines/lemonade/src/device_policy.rs index e0b65449d5..11a68d70bc 100644 --- a/engines/lemonade/src/device_policy.rs +++ b/engines/lemonade/src/device_policy.rs @@ -133,10 +133,16 @@ pub(crate) const fn device_policy_name(policy: &DevicePolicy) -> &'static str { } } -/// The device string reported once a model is loaded. Reflects the backend that -/// actually ran (`rocm`, `vulkan`, …), the pinned GPU ordinal when known, and -/// the GPU name recorded at launch, rather than a hardcoded value, so the -/// report matches observed execution. +/// The device string reported once a model is loaded: the backend this service +/// was launched on (`rocm`, `vulkan`, …), the pinned GPU ordinal when known, and +/// the GPU name recorded at launch. +/// +/// Read back from the service's own recorded `backend_requested` rather than +/// from a hardcoded value — so it follows the backend selection this launch +/// actually made, including a ROCm-to-Vulkan fallback. It is this engine's +/// account of what it asked llama.cpp for, not an independent measurement of +/// which device executed, which is why the CLI's docs say the row *reports* a +/// device rather than naming the one observed. /// /// Formatted by [`rocm_engine_protocol::format_reported_device`] so all three /// engines render one shape; see that function for why the ordinal-less row diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 1d5a8e194c..775f637127 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -820,3 +820,27 @@ Feature: Model serving And a stopped managed service is recorded on the default serve port When the user restarts the stopped service Then the restart is refused naming the service that holds the port + + # ROCMAI-550, and the evidence that the Vulkan remediation is not a dead end. + # `rocm install sdk` on a gfx90c APU refuses and tells the user to run `rocm + # serve --engine lemonade`; AGENTS.md section 3 only allows that if + # serve's no-usable-GPU pre-flight accepts the very host the refusal was + # printed on. That host is a Renoir/Cezanne laptop: the iGPU is on the bus and + # bound to amdgpu, but the kernel's KFD topology lists no compute node for it, + # so the target can only have come from the DRM ip-discovery read. + # + # The assertion is positive rather than "the GPU refusal is absent": an + # out-of-range `--gpu` is validated AFTER the pre-flight and prints the usable + # ordinals it found, so the refusal text here can only be produced by a + # pre-flight that resolved exactly one usable GPU on this host. Take away + # `combine_amd_gpu_counts`'s zero-KFD fallback to DRM and this turns into "no + # usable AMD GPU detected", which is what the step rejects by name. + # + # Linux only, and needs no AMD hardware: the host is planted under + # ROCM_CLI_TEST_HOST_ROOT (an `e2e-test-hooks` build), so the GPU-less mock + # lane is where this runs. + @id:serve-accepts-an-apu-the-kernel-gives-no-compute-node @requires-os:linux + Scenario: serve-50 - Serving is not refused on a Vulkan-only APU the kernel gives no compute node + Given a host whose only GPU is a gfx90c Vega iGPU the kernel reports no compute node for + When the user is told to serve on Lemonade because this GPU has no ROCm build + Then serving got past the no-GPU pre-flight and reported this host's usable GPU diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 786145426a..399146a688 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -2531,27 +2531,58 @@ async fn assert_support_level_agrees_with_detection(world: &mut E2eWorld) { } /// AGENTS.md wants a line that tells the user what they can do, and a named -/// command that can actually clear the condition it was printed for. The -/// specific pairing checked here is the one this round fixed: a host whose -/// managed SDK is already installed must not be told to install it. +/// command that can actually clear the condition it was printed for. +/// +/// So the assertion is the *specific* command each tier owes, not "some `rocm` +/// word appears": `contains("rocm ")` is satisfied by `rocm examine` echoing +/// its own name, by a stray `rocm diagnose` on a ROCm-capable host, and by +/// every wrong-tier command there is. Which tier applies is read from the same +/// three fields the renderer classifies on, so this says something different — +/// and falsifiable — on each lane rather than collapsing to one weak check. +/// +/// The placeholders those fields fall back to are pinned by +/// `rocm_core`'s `examine_render_spells_its_absent_value_placeholders`, so this +/// cannot silently start reading every host as "nothing detected" if the +/// renderer's wording changes. #[then("the support level leaves a command that fits this host")] async fn assert_support_level_advice_fits(world: &mut E2eWorld) { let output = world.cli_output.as_ref().expect("no command was run"); let line = rocm_support_line(output).expect("no rocm_support line"); + let (detected, family) = detected_gpu_facts(output); + // `detected_therock_family` is the managed SDK that *is* installed, as + // opposed to `compatible_therock_family`, which is what this GPU could run. + let installed_sdk = field_value(output, "detected_therock_family") + .filter(|value| *value != "" && *value != ""); + + let (expected, forbidden): (&str, &[&str]) = match (detected, family, installed_sdk) { + // ROCm-capable with the managed SDK already in place: installing one + // clears nothing, so the line must point at serving instead. + (_, Some(_), Some(_)) => ("rocm serve ", &["rocm install sdk"]), + // ROCm-capable with no managed SDK yet: that install is the next step. + (_, Some(_), None) => ("rocm install sdk", &[]), + // A GPU no package family covers — confirmed Vulkan-only (gfx90c) or + // merely unrecognized. Either way the SDK has nothing for it and + // Lemonade's Vulkan backend is what can serve on it. + (Some(_), None, _) => ( + "rocm serve --engine lemonade", + &["rocm install sdk"], + ), + // Nothing detected: `rocm serve` is refused outright by the + // no-usable-GPU pre-flight, so naming it would be advice that cannot + // clear the condition it is printed for. + (None, None, _) => ("rocm diagnose", &["rocm serve", "rocm install sdk"]), + }; + assert!( - line.contains("rocm "), - "the support line must leave the user somewhere to go: {line}\n{output}" + line.contains(expected), + "this host's support tier owes `{expected}`, and the line has to name a \ + command that can actually clear the condition it was printed for: \ + {line}\n{output}" ); - - // `detected_therock_family` is the managed SDK that is installed, as - // opposed to `compatible_therock_family`, which is what this GPU could run. - let sdk_installed = field_value(output, "detected_therock_family") - .is_some_and(|value| value != "" && value != ""); - if sdk_installed { + for command in forbidden { assert!( - !line.contains("rocm install sdk"), - "this host already has a managed ROCm SDK, so installing one clears \ - nothing: {line}\n{output}" + !line.contains(command), + "`{command}` cannot clear anything on this host: {line}\n{output}" ); } } diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 8026963b2a..f516b5bb59 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1287,12 +1287,7 @@ fn docker_call_log(world: &E2eWorld) -> std::path::PathBuf { /// by `e2e-test-hooks` builds) so the family the CLI detects does not depend on the /// runner's hardware. `gfx_target_version` is the kernel's decimal encoding. fn plant_single_gpu(world: &mut E2eWorld, gfx_target_version: u32) { - let host_root = world - .isolated_root - .as_ref() - .expect("scenario has no isolated root") - .path() - .join("host-root"); + let host_root = planted_host_root(world); let node = host_root.join("sys/class/kfd/kfd/topology/nodes/1"); std::fs::create_dir_all(&node).expect("failed to create the planted KFD node"); // `cpu_cores_count 0` with a nonzero `simd_count` makes the node a GPU. @@ -1301,9 +1296,7 @@ fn plant_single_gpu(world: &mut E2eWorld, gfx_target_version: u32) { format!("cpu_cores_count 0\nsimd_count 56\ngfx_target_version {gfx_target_version}\n"), ) .expect("failed to write the planted KFD node properties"); - world - .command_env - .push(("ROCM_CLI_TEST_HOST_ROOT", host_root.into_os_string())); + use_planted_host_root(world); } /// gfx942, TheRock family `gfx94X-dcgpu`. @@ -1318,6 +1311,119 @@ async fn host_with_a_w7900(world: &mut E2eWorld) { plant_single_gpu(world, 110_000); } +/// A Renoir/Cezanne laptop, planted under the simulated host root: the Vega +/// iGPU is on the PCI bus and bound to `amdgpu`, and the kernel's KFD topology +/// lists **only a CPU node** for it. +/// +/// Not `plant_single_gpu`: that writes a KFD GPU node, which is precisely the +/// thing this host does not have. With no compute node the target can only be +/// named by the DRM `ip_discovery` read (GC 9.0.12 -> `gfx90c`), and the usable +/// GPU count can only come from `combine_amd_gpu_counts`' zero-KFD fallback to +/// the DRM card count. Both are driven off this one directory, which is what +/// makes the scenario a statement about one host rather than two. +fn plant_gfx90c_apu_without_a_kfd_node(world: &mut E2eWorld) { + let host_root = planted_host_root(world); + let device = host_root.join("sys/class/drm/card0/device"); + std::fs::create_dir_all(&device).expect("failed to create the planted DRM card"); + std::fs::write(device.join("vendor"), "0x1002\n").expect("failed to write the planted vendor"); + let gc = device.join("ip_discovery/die/0/GC/0"); + std::fs::create_dir_all(&gc).expect("failed to create the planted GC IP block"); + for (file, value) in [("major", "9"), ("minor", "0"), ("revision", "12")] { + std::fs::write(gc.join(file), value).expect("failed to write the planted GC IP version"); + } + let cpu_node = host_root.join("sys/class/kfd/kfd/topology/nodes/0"); + std::fs::create_dir_all(&cpu_node).expect("failed to create the planted KFD node"); + std::fs::write( + cpu_node.join("properties"), + "cpu_cores_count 16\nsimd_count 0\ngfx_target_version 0\n", + ) + .expect("failed to write the planted KFD node properties"); + use_planted_host_root(world); +} + +/// Where a scenario's simulated `/` lives, by the same convention +/// [`plant_single_gpu`] uses. +fn planted_host_root(world: &E2eWorld) -> std::path::PathBuf { + world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .join("host-root") +} + +/// Point the *next* `rocm` invocation at the planted host. +/// +/// Needed once per invocation: `run_rocm_with_scenario_env` takes +/// `world.command_env` by `mem::take`, so a step that runs two commands would +/// silently run the second one against the real machine — which for this +/// scenario would mean asserting the serve pre-flight's answer for the runner's +/// own hardware instead of for the host the refusal was printed on. +fn use_planted_host_root(world: &mut E2eWorld) { + let host_root = planted_host_root(world); + world + .command_env + .push(("ROCM_CLI_TEST_HOST_ROOT", host_root.into_os_string())); +} + +#[given("a host whose only GPU is a gfx90c Vega iGPU the kernel reports no compute node for")] +async fn host_with_a_kfd_less_gfx90c(world: &mut E2eWorld) { + plant_gfx90c_apu_without_a_kfd_node(world); +} + +/// Take the route the CLI itself sends this host down: let the SDK install +/// refuse, then run the command that refusal names. +/// +/// Both halves run against the same planted host, in order, so the scenario +/// cannot pass by printing the advice on one machine and testing it on another. +/// `--gpu 99` is what makes the result readable: it is validated *after* the +/// no-usable-GPU pre-flight and reports the ordinals that pre-flight resolved, +/// turning "the host was accepted" into something the output states rather than +/// something inferred from the absence of a refusal. The model ref is arbitrary +/// and nothing is launched: both bails are reached before any runtime or engine +/// work. +#[when("the user is told to serve on Lemonade because this GPU has no ROCm build")] +async fn user_follows_the_lemonade_advice(world: &mut E2eWorld) { + let (install_stdout, install_stderr, install_rc) = + crate::run_rocm_with_scenario_env(world, &["install", "sdk", "--dry-run"]); + let refusal = format!("{install_stdout}\n{install_stderr}"); + assert_ne!( + install_rc, 0, + "a gfx90c host has no ROCm SDK build to install:\n{refusal}" + ); + assert!( + refusal.contains("gfx90c") && refusal.contains("rocm serve --engine lemonade"), + "the planted host must be detected as gfx90c and advised to serve on \ + Lemonade; without that this scenario is testing nothing:\n{refusal}" + ); + + use_planted_host_root(world); + let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env( + world, + &["serve", "some-model", "--engine", "lemonade", "--gpu", "99"], + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[then("serving got past the no-GPU pre-flight and reported this host's usable GPU")] +async fn assert_serve_accepted_the_drm_only_gpu(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + !output.contains("no usable AMD GPU"), + "`rocm install sdk` told this host to run `rocm serve --engine lemonade`, \ + and serve's pre-flight refused it for having no GPU - advice that cannot \ + clear the condition it was printed for:\n{output}" + ); + assert!( + output.contains("--gpu index 99 is not present on this host") + && output.contains("usable GPU indices: [0]"), + "the pre-flight must have resolved exactly the iGPU's ordinal, and said \ + so where `--gpu` is validated:\n{output}" + ); +} + fn run_serve_aim(world: &mut E2eWorld, args: &[&str]) { let mut full = vec!["serve"]; full.extend_from_slice(args); From 6a928436800b07be0a9336f60eb708fa62b278b0 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 11:10:41 +0200 Subject: [PATCH 14/37] feat(examine): probe Vulkan instead of inferring it from the gfx table `GpuSupportLevel::VulkanOnly` was derived from the gfx-target table alone: "no ROCm build" was read as "Vulkan then", on a machine nobody had looked at. Every surface that renders that level tells the user to go and serve a model on it, so a host with no Vulkan driver installed -- or with only a software rasteriser -- was sent to a backend that cannot start, with no CPU fallback to catch it. Adds a real probe (host_gpu/vulkan.rs): ICD manifests through `host_path` on Linux, the Khronos registry on Windows, and `vulkaninfo --summary` for device enumeration. Filesystem and subprocess evidence rather than dlopening libvulkan, because the loader reads the real ICD list and so could never be faked on the no-GPU lane -- and because running vendor driver code inside `examine` can hang a headless runner. `gpu_support_level` now takes that evidence. An installed driver and an enumerated device are kept apart (`DriverPresentDevicesUnverified`), since collapsing them would replace one unverified promise with another; a platform that cannot be probed keeps `VulkanOnly`, because withholding the only path such a host has, on the strength of a probe that never ran, would be a worse answer than the assumption being replaced. The negative case gets its own level. `NoGpuBackend` is not folded into `Unknown`: both halves there are established (the catalog confirms no ROCm build, the probe finds no driver), whereas `Unknown` means neither could be. Collapsing them commits, in the other direction, the conflation `Unknown` exists to prevent. A software rasteriser is never a GPU path. Mesa lavapipe is a complete Vulkan implementation running on the CPU, so counting it would reintroduce the silent CPU fallback through a backend rather than through a policy change. `rocm engines` and the codex bridge deliberately do not probe: they pass `VulkanStatus::Unknown` and answer exactly as before, so a listing command does not grow a subprocess. `rocm examine` already gathers the probe and is the surface those messages send people to. examine-49/50/51 cover the three answers on the ungated mock lane, which needs neither a Renoir APU nor a Vulkan driver -- only the KFD node and ICD state the harness can plant. Falsified: both negative scenarios fail when `gpu_support_level` is made to ignore the probe. Later superseded: The Vulkan probe this added was later split out of this branch into PR #1036; what remains here is the support-level classifier it was built on. Signed-off-by: Roman Sirokov --- apps/rocm/src/bridge_snapshot.rs | 1 + apps/rocm/src/driver_plan.rs | 1 + apps/rocm/src/examine_report_text.rs | 10 +- apps/rocm/src/main.rs | 24 +- crates/rocm-core/src/gfx_target.rs | 133 +++- crates/rocm-core/src/host_gpu.rs | 17 + crates/rocm-core/src/host_gpu/render.rs | 207 +++++- crates/rocm-core/src/host_gpu/vulkan.rs | 686 ++++++++++++++++++ crates/rocm-core/src/lib.rs | 12 +- docs/architecture.md | 2 +- tests/e2e-cucumber/features/examine.feature | 37 + tests/e2e-cucumber/tests/e2e/examine_steps.rs | 224 ++++++ 12 files changed, 1319 insertions(+), 35 deletions(-) create mode 100644 crates/rocm-core/src/host_gpu/vulkan.rs diff --git a/apps/rocm/src/bridge_snapshot.rs b/apps/rocm/src/bridge_snapshot.rs index 741c18ede7..115279e42a 100644 --- a/apps/rocm/src/bridge_snapshot.rs +++ b/apps/rocm/src/bridge_snapshot.rs @@ -40,6 +40,7 @@ pub(crate) fn build_codex_bridge_snapshot(paths: &AppPaths) -> Result) render_engine_inventory_text_for_host( paths, rocm_core::default_engine_for_host(&host_gpu), - rocm_core::gpu_support_level_for_host_gpu(&host_gpu), + // No Vulkan probe on this listing path, for the reason given on + // `host_gpu_support_level`: `rocm engines` is cheap by design, and + // `rocm examine` is where the verified answer lives. + rocm_core::gpu_support_level_for_host_gpu(&host_gpu, rocm_core::VulkanStatus::Unknown), ) } @@ -634,6 +641,7 @@ mod tests { detected_gfx_target: None, compatible_therock_family: None, detected_therock_family: None, + vulkan: None, driver: rocm_core::DriverSummary { policy: "linux_official_amd_dkms_wrapper".to_owned(), status: "amdgpu_available".to_owned(), diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index c18432bc61..3887afd02e 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -1081,6 +1081,15 @@ const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str rocm_core::GpuSupportLevel::RocmHip => { "default embedded Lemonade server with ROCm llama.cpp backend" } + // Only `rocm examine` reaches this level — the cheap probe behind + // `host_gpu_support_level` passes `VulkanStatus::Unknown` and so lands + // on `VulkanOnly` — but the arm is spelled out rather than folded into + // `Unknown`, because what it means is the opposite: `Unknown` promises + // that a command can still answer, and here the question is already + // answered and the answer is no. + rocm_core::GpuSupportLevel::NoGpuBackend => { + "default embedded Lemonade server; this GPU has no ROCm/HIP support and no Vulkan driver was found, so no GPU backend can run here" + } rocm_core::GpuSupportLevel::Unknown => { "default embedded Lemonade server; run `rocm examine` for this host's GPU support level, which says how to find the backend" } @@ -1093,8 +1102,21 @@ const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str /// The same two inputs `rocm examine` derives it from, read through /// `detect_host_gpu_summary` rather than the full examine gather — exactly what /// `render_engine_inventory_text_with_paths` (i.e. `rocm engines`) already does. +/// +/// **The Vulkan probe is deliberately not run here**, so this stays the cheap +/// probe its callers chose it for: the probe spawns `vulkaninfo` on Linux and +/// PowerShell on Windows, and `rocm engines` is a listing command that would +/// pay that on every invocation. Passing [`VulkanStatus::Unknown`] keeps these +/// surfaces answering exactly as they did before this probe existed — a +/// Vulkan-capable-by-the-table host still reads `VulkanOnly` — while `rocm +/// examine`, which already gathers the probe, is the surface that can tell a +/// host with a Vulkan driver from one without. That split is why +/// `lemonade_engine_note` sends the user to `rocm examine` for a real answer. pub(crate) fn host_gpu_support_level(paths: Option<&AppPaths>) -> rocm_core::GpuSupportLevel { - rocm_core::gpu_support_level_for_host_gpu(&rocm_core::detect_host_gpu_summary(paths)) + rocm_core::gpu_support_level_for_host_gpu( + &rocm_core::detect_host_gpu_summary(paths), + rocm_core::VulkanStatus::Unknown, + ) } fn refresh_startup_update_check_quietly() { diff --git a/crates/rocm-core/src/gfx_target.rs b/crates/rocm-core/src/gfx_target.rs index bd0a6bd6b0..391c145fd3 100644 --- a/crates/rocm-core/src/gfx_target.rs +++ b/crates/rocm-core/src/gfx_target.rs @@ -946,7 +946,7 @@ mod tests { therock_family: Some("gfx94X-dcgpu".to_owned()), }; assert_eq!( - gpu_support_level_for_host_gpu(&host_gpu), + gpu_support_level_for_host_gpu(&host_gpu, VulkanStatus::Available), GpuSupportLevel::RocmHip ); } @@ -959,7 +959,7 @@ mod tests { therock_family: None, }; assert_eq!( - gpu_support_level_for_host_gpu(&host_gpu), + gpu_support_level_for_host_gpu(&host_gpu, VulkanStatus::Available), GpuSupportLevel::VulkanOnly ); } @@ -967,32 +967,86 @@ mod tests { #[test] fn gpu_support_level_for_host_gpu_reads_an_undetected_summary_as_unknown() { assert_eq!( - gpu_support_level_for_host_gpu(&HostGpuSummary::default()), + gpu_support_level_for_host_gpu(&HostGpuSummary::default(), VulkanStatus::Available), GpuSupportLevel::Unknown ); } + use crate::host_gpu::VulkanStatus; + #[test] fn gpu_support_level_is_rocm_hip_when_a_family_covers_it() { assert_eq!( - gpu_support_level(Some("gfx90a"), Some("gfx90a")), + gpu_support_level(Some("gfx90a"), Some("gfx90a"), VulkanStatus::Available), + GpuSupportLevel::RocmHip + ); + // A covering family wins outright: a host whose Vulkan probe found + // nothing still has ROCm, and must not be downgraded for it. + assert_eq!( + gpu_support_level(Some("gfx90a"), Some("gfx90a"), VulkanStatus::Absent), GpuSupportLevel::RocmHip ); } #[test] fn gpu_support_level_is_unknown_without_detection_or_for_an_unrecognized_target() { - assert_eq!(gpu_support_level(None, None), GpuSupportLevel::Unknown); assert_eq!( - gpu_support_level(Some("gfx_totally_unrecognized"), None), + gpu_support_level(None, None, VulkanStatus::Available), + GpuSupportLevel::Unknown + ); + // Finding a Vulkan driver says nothing about whether ROCm would work on + // an unrecognized target, so it must not promote a detection gap into a + // support claim. + assert_eq!( + gpu_support_level( + Some("gfx_totally_unrecognized"), + None, + VulkanStatus::Available + ), GpuSupportLevel::Unknown ); } #[test] - fn gpu_support_level_is_vulkan_only_for_gfx90c() { + fn gpu_support_level_is_vulkan_only_for_gfx90c_with_a_vulkan_driver() { + assert_eq!( + gpu_support_level(Some("gfx90c"), None, VulkanStatus::Available), + GpuSupportLevel::VulkanOnly + ); + // An installed driver that enumerated no device is still worth routing + // to; the level says "Vulkan", and `examine`'s own line carries how sure + // the probe was. assert_eq!( - gpu_support_level(Some("gfx90c"), None), + gpu_support_level( + Some("gfx90c"), + None, + VulkanStatus::DriverPresentDevicesUnverified + ), + GpuSupportLevel::VulkanOnly + ); + } + + /// The defect this guards against: `VulkanOnly` used to be inferred from the + /// gfx table alone, so a Renoir laptop with no Vulkan driver installed — or + /// with only a CPU rasteriser — was told to go and serve a model on a + /// backend that cannot start. + #[test] + fn gpu_support_level_reports_no_gpu_backend_when_the_vulkan_probe_found_none() { + assert_eq!( + gpu_support_level(Some("gfx90c"), None, VulkanStatus::Absent), + GpuSupportLevel::NoGpuBackend, + "a confirmed-absent Vulkan driver must not read as a usable Vulkan path" + ); + } + + /// A platform the probe cannot read is not a platform without Vulkan. + /// Withholding the only path such a host has, on the strength of a probe + /// that never ran, would be a worse answer than the assumption this change + /// replaces. + #[test] + fn gpu_support_level_keeps_vulkan_when_the_probe_could_not_run() { + assert_eq!( + gpu_support_level(Some("gfx90c"), None, VulkanStatus::Unknown), GpuSupportLevel::VulkanOnly ); } @@ -1013,23 +1067,63 @@ mod tests { pub enum GpuSupportLevel { /// A TheRock package family covers this GPU: full ROCm/HIP support. RocmHip, - /// Recognized, confirmed to have no ROCm/HIP build, but Vulkan-based - /// engines (the default Lemonade backend) still work on it. + /// Recognized, confirmed to have no ROCm/HIP build, and a Vulkan GPU driver + /// was **found on this host**, so Vulkan-based engines (the default Lemonade + /// backend) run on it. + /// + /// The second half of that sentence is evidence, not inference. This variant + /// used to be returned from the gfx table alone — "no ROCm build" was taken + /// to mean "Vulkan then", on a machine nobody had looked at. Every surface + /// that renders this level tells the user to go and serve a model on it, so + /// a host with no Vulkan driver installed, or with only a software + /// rasteriser, was sent to a backend that cannot start. VulkanOnly, + /// Recognized, confirmed to have no ROCm/HIP build, **and** no Vulkan GPU + /// driver was found: this host has no GPU path at all. + /// + /// Deliberately not folded into [`Self::Unknown`]. Both halves here are + /// confirmed — the catalog establishes the missing ROCm build, and the + /// Vulkan probe looked and found nothing — whereas `Unknown` means the CLI + /// could not establish either. Collapsing them would commit, in the other + /// direction, the exact conflation this enum's `Unknown` exists to prevent: + /// a fact about the machine would be reported as a gap in what this build + /// knows, and the user would be sent to probe a question already answered. + NoGpuBackend, /// No GPU detected yet, or a gfx target this catalog doesn't recognize. Unknown, } -/// Classify the host's GPU support level from the same two facts -/// `ExamineSummary` and `resolve_family` already compute: the detected gfx -/// target and whichever TheRock family (if any) covers it. +/// Classify the host's GPU support level. +/// +/// Reads the two facts `ExamineSummary` and `resolve_family` already compute — +/// the detected gfx target and whichever TheRock family (if any) covers it — +/// plus what the Vulkan probe found. +/// +/// `vulkan` only ever decides between [`GpuSupportLevel::VulkanOnly`] and +/// [`GpuSupportLevel::NoGpuBackend`], and is consulted only once the catalog has +/// confirmed there is no ROCm build. A host with a covering family is `RocmHip` +/// whatever Vulkan reports, and an unrecognized target stays `Unknown`: finding +/// a Vulkan driver says nothing about whether ROCm would also work there, so it +/// must not be allowed to turn a detection gap into a support claim. +/// +/// [`VulkanStatus::Unknown`] — the platform could not be probed at all — keeps +/// `VulkanOnly`, because withholding the one path such a host has on the +/// strength of a probe that never ran would be worse than the old assumption, +/// not better. pub fn gpu_support_level( detected_gfx_target: Option<&str>, compatible_therock_family: Option<&str>, + vulkan: crate::host_gpu::VulkanStatus, ) -> GpuSupportLevel { match (detected_gfx_target, compatible_therock_family) { (_, Some(_)) => GpuSupportLevel::RocmHip, - (Some(gfx), None) if gfx_target_has_no_rocm_build(gfx) => GpuSupportLevel::VulkanOnly, + (Some(gfx), None) if gfx_target_has_no_rocm_build(gfx) => { + if vulkan.is_usable() || vulkan == crate::host_gpu::VulkanStatus::Unknown { + GpuSupportLevel::VulkanOnly + } else { + GpuSupportLevel::NoGpuBackend + } + } _ => GpuSupportLevel::Unknown, } } @@ -1041,10 +1135,19 @@ pub fn gpu_support_level( /// site that transposed them would classify a Vulkan-only host as `RocmHip` /// and still compile. `rocm engines` and the codex-bridge `engines` tool both /// derive the level from this struct, so both go through this. +/// +/// `vulkan` is passed in rather than probed here so this stays a pure function +/// of what the caller already holds: the probe runs a subprocess and reads the +/// filesystem, and burying that in a helper this cheap would make every caller +/// pay for it without saying so. #[must_use] -pub fn gpu_support_level_for_host_gpu(host_gpu: &HostGpuSummary) -> GpuSupportLevel { +pub fn gpu_support_level_for_host_gpu( + host_gpu: &HostGpuSummary, + vulkan: crate::host_gpu::VulkanStatus, +) -> GpuSupportLevel { gpu_support_level( host_gpu.gfx_target.as_deref(), host_gpu.therock_family.as_deref(), + vulkan, ) } diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index 87b6ad185f..db45107033 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -54,6 +54,7 @@ mod gpu_visibility; pub(crate) mod linux; mod os_facts; mod render; +mod vulkan; mod windows_inventory; mod wsl; @@ -73,6 +74,11 @@ pub(crate) use linux::detect_linux_sysfs_gfx_target; pub use linux::is_amdgpu_device; #[cfg(any(target_os = "linux", test))] pub(crate) use linux::{KfdGpuNode, kfd_gpu_nodes_in}; +pub use vulkan::{ + VulkanDevice, VulkanStatus, VulkanSupport, classify_vulkan_support, detect_vulkan_support, + icd_driver_label, icd_is_gpu_driver, parse_vulkaninfo_summary, + parse_windows_vulkan_icd_listing, +}; pub(crate) use windows_inventory::detect_windows_amd_driver_info; #[cfg(test)] use windows_inventory::parse_windows_examine_inventory; @@ -133,6 +139,15 @@ pub struct ExamineSummary { pub compatible_therock_family: Option, #[serde(default)] pub detected_therock_family: Option, + /// What the Vulkan probe found, when it could run. + /// + /// `#[serde(default)]` is load-bearing in the *reverse* direction: `rocm + /// remote doctor` deserializes summaries produced by other hosts, which may + /// be running a build from before this field existed. `None` means "not + /// probed", which [`ExamineSummary::vulkan_status`] maps to + /// [`VulkanStatus::Unknown`] — never to "no Vulkan". + #[serde(default)] + pub vulkan: Option, pub driver: DriverSummary, pub legacy_rocm: LegacyRocmSummary, #[serde(default)] @@ -284,6 +299,7 @@ impl ExamineSummary { detected_gfx_target, compatible_therock_family, detected_therock_family, + vulkan: Some(detect_vulkan_support()), driver: detect_driver_summary_with_windows_inventory( windows_inventory.as_ref(), wsl.as_ref(), @@ -730,6 +746,7 @@ mod tests { detected_gfx_target: None, compatible_therock_family: None, detected_therock_family: None, + vulkan: None, driver: windows_driver_summary( Some("AMD Radeon(TM) Graphics driver 30.0.13014.8".to_owned()), info, diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index 82eb8b8391..df88230d12 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -6,9 +6,42 @@ use super::ExamineSummary; use super::os_facts::format_gib_value; +use super::vulkan::VulkanStatus; use crate::gfx_target::{GpuSupportLevel, gpu_support_level}; impl ExamineSummary { + /// What the Vulkan probe found, or [`VulkanStatus::Unknown`] when this + /// summary carries no probe result — an older host's JSON read back by + /// `rocm remote doctor`, or a caller that built the struct by hand. + /// + /// Public because the `rocm` binary classifies its own support level from a + /// summary it already gathered, and must read the probe result the same way + /// this module does rather than reaching into the field and inventing its + /// own `None` handling. + #[must_use] + pub fn vulkan_status(&self) -> VulkanStatus { + self.vulkan + .as_ref() + .map_or(VulkanStatus::Unknown, |vulkan| vulkan.status) + } + + /// The `vulkan` line: the status, and the evidence behind it. + fn vulkan_line(&self) -> String { + let Some(vulkan) = self.vulkan.as_ref() else { + return "".to_owned(); + }; + let status = match vulkan.status { + VulkanStatus::Available => "available", + VulkanStatus::DriverPresentDevicesUnverified => "driver_present_devices_unverified", + VulkanStatus::Absent => "absent", + VulkanStatus::Unknown => "unknown", + }; + vulkan.detail.as_deref().map_or_else( + || status.to_owned(), + |detail| format!("{status} ({detail})"), + ) + } + /// The `rocm_support` line of the examine text. /// /// ROCMAI-550 asks `rocm examine` to "report the support level with one @@ -34,14 +67,49 @@ impl ExamineSummary { /// blank line. fn rocm_support_line(&self) -> String { let detected = self.detected_gfx_target.as_deref(); - match gpu_support_level(detected, self.compatible_therock_family.as_deref()) { + let vulkan = self.vulkan_status(); + match gpu_support_level(detected, self.compatible_therock_family.as_deref(), vulkan) { // `VulkanOnly` is only ever returned for a detected target, so the // fallback below is unreachable; it keeps the arm total without a // panic path. + // + // The claim is graded by what the probe actually established. An + // enumerated device is named, because naming it is what lets the + // reader check the claim rather than take it; an installed-but- + // unenumerated driver says so instead of borrowing the stronger + // sentence; and a platform that could not be probed admits it. All + // three still point at the same command, because all three are + // hosts worth trying. GpuSupportLevel::VulkanOnly => format!( - " rocm_support: GPU detected ({}) but not supported by ROCm; \ - the Vulkan backend can still be used - serve on it with \ - `rocm serve --engine lemonade`\n", + " rocm_support: GPU detected ({}) but not supported by ROCm; {} - serve on it \ + with `rocm serve --engine lemonade`\n", + detected.unwrap_or_default(), + match vulkan { + VulkanStatus::Available => self.vulkan.as_ref().map_or_else( + || "the Vulkan backend can be used".to_owned(), + |support| support.driver_label().map_or_else( + || "the Vulkan backend can be used".to_owned(), + |driver| format!("the Vulkan backend can be used ({driver})"), + ), + ), + VulkanStatus::DriverPresentDevicesUnverified => + "a Vulkan driver is installed, so the Vulkan backend is expected to work" + .to_owned(), + VulkanStatus::Absent | VulkanStatus::Unknown => + "Vulkan was not probed on this platform, so the Vulkan backend is \ + unconfirmed" + .to_owned(), + } + ), + // Both halves are established facts: the catalog confirmed there is + // no ROCm build, and the probe looked for a Vulkan driver and found + // none. Unlike `Unknown` below, there is nothing left to go and find + // out on this host, so the advice is to install a driver rather than + // to serve a model and read back what happened. + GpuSupportLevel::NoGpuBackend => format!( + " rocm_support: GPU detected ({}) but not supported by ROCm, and no Vulkan GPU \ + driver was found - install one (Mesa RADV on Linux, the AMD display driver on \ + Windows) to run models on this GPU\n", detected.unwrap_or_default() ), // The covering family is the fact; it is already printed on its own @@ -137,7 +205,7 @@ impl ExamineSummary { "false (this run's output is captured, so the CLI will not prompt)" }; format!( - "rocm examine\n os: {}\n arch: {}\n kernel: {}\n distro: {}\n cpu: {}\n system_ram: {}\n interactive_terminal: {}\n default_engine: {}\n detected_gfx_target: {}\n compatible_therock_family: {}\n{} detected_therock_family: {}\n driver_policy: {}\n driver_status: {}\n driver_detail: {}\n{} legacy_rocm_status: {}\n legacy_rocm_paths: {}\n legacy_rocm_version: {}\n legacy_rocm_detail: {}\n legacy_rocm_guidance: {}\n wsl: {}\n wsl_dxg_device: {}\n wsl_dxcore: {}\n wsl_librocdxg: {}\n wsl_rocdxg_dids: {}\n wsl_ldconfig_librocdxg: {}\n wsl_librocdxg_on_ld_library_path: {}\n wsl_librocdxg_beside_runtime: {}\n wsl_global_rocminfo: {}\n wsl_cargo: {}\n wsl_detail: {}\n managed_runtimes: {}\n managed_services: {}\n model_cache_entries: {}\n config_dir: {}\n data_dir: {}\n cache_dir: {}\n", + "rocm examine\n os: {}\n arch: {}\n kernel: {}\n distro: {}\n cpu: {}\n system_ram: {}\n interactive_terminal: {}\n default_engine: {}\n detected_gfx_target: {}\n compatible_therock_family: {}\n{} detected_therock_family: {}\n vulkan: {}\n driver_policy: {}\n driver_status: {}\n driver_detail: {}\n{} legacy_rocm_status: {}\n legacy_rocm_paths: {}\n legacy_rocm_version: {}\n legacy_rocm_detail: {}\n legacy_rocm_guidance: {}\n wsl: {}\n wsl_dxg_device: {}\n wsl_dxcore: {}\n wsl_librocdxg: {}\n wsl_rocdxg_dids: {}\n wsl_ldconfig_librocdxg: {}\n wsl_librocdxg_on_ld_library_path: {}\n wsl_librocdxg_beside_runtime: {}\n wsl_global_rocminfo: {}\n wsl_cargo: {}\n wsl_detail: {}\n managed_runtimes: {}\n managed_services: {}\n model_cache_entries: {}\n config_dir: {}\n data_dir: {}\n cache_dir: {}\n", self.os, self.arch, self.kernel.as_deref().unwrap_or(""), @@ -155,6 +223,7 @@ impl ExamineSummary { self.detected_therock_family .as_deref() .unwrap_or(""), + self.vulkan_line(), self.driver.policy, self.driver.status, self.driver.detail.as_deref().unwrap_or(""), @@ -217,8 +286,8 @@ mod tests { use super::*; use crate::LegacyRocmSummary; use crate::gfx_target::normalize_therock_family; - use crate::host_gpu::DriverSummary; use crate::host_gpu::tests::windows_summary_with_driver; + use crate::host_gpu::{DriverSummary, VulkanDevice, VulkanSupport}; use std::path::PathBuf; #[test] @@ -235,6 +304,7 @@ mod tests { detected_gfx_target: None, compatible_therock_family: Some("gfx120X-all".to_owned()), detected_therock_family: None, + vulkan: None, driver: DriverSummary { policy: "windows_validate_only".to_owned(), status: "amd_display_driver_detected".to_owned(), @@ -314,6 +384,7 @@ mod tests { detected_gfx_target: detected_gfx_target.map(str::to_owned), compatible_therock_family: compatible_therock_family.map(str::to_owned), detected_therock_family: None, + vulkan: None, driver: DriverSummary { policy: "windows_validate_only".to_owned(), status: "amd_display_driver_detected".to_owned(), @@ -336,12 +407,41 @@ mod tests { } } + /// A gfx90c summary carrying the given Vulkan evidence. + /// + /// gfx90c is detected from its PCI ID but has no ROCm package family, so it + /// is the target that reaches the Vulkan-graded arms at all. + fn gfx90c_summary_with_vulkan(vulkan: Option) -> ExamineSummary { + let family = normalize_therock_family("gfx90c"); + let mut summary = examine_summary_with_gpu(Some("gfx90c"), family.as_deref()); + summary.vulkan = vulkan; + summary + } + + fn vulkan_support(status: VulkanStatus, detail: &str) -> Option { + Some(VulkanSupport { + status, + detail: Some(detail.to_owned()), + devices: vec![VulkanDevice { + name: "AMD Radeon Graphics (RADV RENOIR)".to_owned(), + driver_name: Some("radv".to_owned()), + vendor_id: Some("0x1002".to_owned()), + device_type: Some("PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU".to_owned()), + ..VulkanDevice::default() + }], + ..VulkanSupport::default() + }) + } + #[test] fn examine_render_says_a_gpu_without_a_rocm_family_is_unsupported() { - // gfx90c is detected from its PCI ID but has no ROCm package family. - let family = normalize_therock_family("gfx90c"); - let summary = examine_summary_with_gpu(Some("gfx90c"), family.as_deref()); + let summary = gfx90c_summary_with_vulkan(vulkan_support( + VulkanStatus::Available, + "AMD Radeon Graphics (RADV RENOIR) through radv", + )); + let rendered = summary.render_text(); + assert!( rendered.contains("detected_gfx_target: gfx90c"), "{rendered}" @@ -351,9 +451,92 @@ mod tests { "{rendered}" ); assert!( - rendered.contains("the Vulkan backend can still be used"), + rendered.contains("the Vulkan backend can be used (radv)"), + "a confirmed device must be named, so the claim can be checked: {rendered}" + ); + assert!( + rendered.contains("vulkan: available"), + "the evidence line must carry the verdict too: {rendered}" + ); + } + + /// The defect this guards against: the `VulkanOnly` line used to promise + /// "the Vulkan backend can still be used" on every gfx90c host, derived from + /// the gfx table alone. A Renoir laptop with no Vulkan driver installed was + /// told to go and serve a model on a backend that cannot start — and under + /// the GPU-required policy there is nothing for it to fall back to. + #[test] + fn examine_render_does_not_promise_vulkan_on_a_host_that_has_none() { + let summary = gfx90c_summary_with_vulkan(Some(VulkanSupport { + status: VulkanStatus::Absent, + detail: Some("no Vulkan driver found".to_owned()), + ..VulkanSupport::default() + })); + + let rendered = summary.render_text(); + + assert!( + !rendered.contains("the Vulkan backend can be used"), + "a host with no Vulkan driver must not be promised one: {rendered}" + ); + assert!( + rendered.contains("no Vulkan GPU driver was found"), "{rendered}" ); + assert!( + rendered.contains("install one"), + "the advice must be able to clear the condition it is printed for: {rendered}" + ); + assert!(rendered.contains("vulkan: absent"), "{rendered}"); + } + + /// An installed driver that enumerated no device is a weaker claim than a + /// device that answered, and says so rather than borrowing the stronger + /// sentence. + #[test] + fn examine_render_separates_an_installed_driver_from_a_confirmed_device() { + let summary = gfx90c_summary_with_vulkan(Some(VulkanSupport { + status: VulkanStatus::DriverPresentDevicesUnverified, + detail: Some("Mesa RADV is installed, but no device was enumerated".to_owned()), + ..VulkanSupport::default() + })); + + let rendered = summary.render_text(); + + assert!( + rendered.contains("a Vulkan driver is installed, so the Vulkan backend is expected"), + "{rendered}" + ); + assert!( + rendered.contains("vulkan: driver_present_devices_unverified"), + "{rendered}" + ); + } + + /// A summary with no probe result at all — an older host's JSON read back + /// through `rocm remote doctor` — must not read as "no Vulkan". It keeps + /// the Vulkan advice, because withholding the only path such a host has on + /// the strength of a probe that never ran would be worse than the + /// assumption this change replaces. + #[test] + fn examine_render_treats_an_unprobed_summary_as_unconfirmed_not_absent() { + let summary = gfx90c_summary_with_vulkan(None); + + let rendered = summary.render_text(); + + assert!(rendered.contains("vulkan: "), "{rendered}"); + assert!( + rendered.contains("the Vulkan backend is unconfirmed"), + "{rendered}" + ); + assert!( + !rendered.contains("no Vulkan GPU driver was found"), + "an unprobed host is not a host without Vulkan: {rendered}" + ); + assert!( + rendered.contains("rocm serve --engine lemonade"), + "it is still worth trying, so the command stays: {rendered}" + ); } /// An unmapped target must not be *claimed* unsupported, even though every @@ -461,6 +644,7 @@ mod tests { detected_gfx_target: None, compatible_therock_family: None, detected_therock_family: None, + vulkan: None, driver: DriverSummary { policy: "linux_official_amd_dkms_wrapper".to_owned(), status: "amdgpu_available".to_owned(), @@ -515,6 +699,7 @@ mod tests { detected_gfx_target: None, compatible_therock_family: None, detected_therock_family: None, + vulkan: None, driver: DriverSummary { policy: "linux_official_amd_dkms_wrapper".to_owned(), status: "amdgpu_available".to_owned(), @@ -610,7 +795,7 @@ mod tests { .find(|l| l.trim_start().starts_with("rocm_support:")) .expect("every host gets a support line"); assert_eq!( - gpu_support_level(None, None), + gpu_support_level(None, None, VulkanStatus::Unknown), GpuSupportLevel::Unknown, "precondition: nothing detected is the Unknown tier" ); diff --git a/crates/rocm-core/src/host_gpu/vulkan.rs b/crates/rocm-core/src/host_gpu/vulkan.rs new file mode 100644 index 0000000000..cc8cdeeaf3 --- /dev/null +++ b/crates/rocm-core/src/host_gpu/vulkan.rs @@ -0,0 +1,686 @@ +// Copyright © Advanced Micro Devices, Inc., or its affiliates. +// +// SPDX-License-Identifier: MIT + +//! Whether this host can run a model through Vulkan. +//! +//! The CLI has always *asserted* this for a GPU with no ROCm build -- `examine` +//! told a gfx90c user "the Vulkan backend can still be used" without ever +//! looking. This probe is what makes that sentence evidence rather than a +//! promise, and it is what lets a host outside the ROCm compatibility matrix be +//! described by what it can do instead of only by what it cannot. +//! +//! Two sources, in increasing order of strength: +//! +//! 1. An **installable driver**: an ICD manifest on Linux (`/usr/share/vulkan/ +//! icd.d` and friends), or the Khronos registry key on Windows. This proves a +//! driver is *installed*, not that a device enumerates. +//! 2. An **enumerated device**: `vulkaninfo --summary`, when it is on PATH. +//! This proves a device actually answers. +//! +//! The two are kept apart by [`VulkanStatus`] rather than collapsed, because +//! collapsing them would replace one unverified promise with another. +//! +//! **A software rasteriser is not a GPU.** Mesa's lavapipe (`lvp_icd.*.json`, +//! `PHYSICAL_DEVICE_TYPE_CPU`) presents a complete Vulkan implementation that +//! runs entirely on the CPU. Counting it here would route a user onto CPU +//! execution while reporting a GPU path -- precisely the silent CPU fallback +//! `AGENTS.md` §6 forbids -- so it is recognised in order to be excluded, and +//! named in `detail` so the exclusion is visible rather than mysterious. + +use super::capture::capture_optional_command; +#[cfg(target_os = "linux")] +use crate::host_path; +use serde::{Deserialize, Serialize}; + +/// The directories a Linux Vulkan loader reads ICD manifests from. +/// +/// Read through [`host_path`] so a simulated host root can plant them, which is +/// what lets the no-GPU CI lane cover this probe at all. +#[cfg(target_os = "linux")] +const LINUX_ICD_DIRS: [&str; 4] = [ + "/usr/share/vulkan/icd.d", + "/usr/local/share/vulkan/icd.d", + "/etc/vulkan/icd.d", + "/usr/lib/x86_64-linux-gnu/vulkan/icd.d", +]; + +/// How sure the CLI is that a model can run through Vulkan here. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "snake_case")] +pub enum VulkanStatus { + /// A Vulkan device answered and it is a real GPU. + Available, + /// A driver is installed, but no device was enumerated -- `vulkaninfo` is + /// not on PATH, so nothing confirmed a device answers. Likely to work; + /// not established. + DriverPresentDevicesUnverified, + /// The probe looked and found no GPU Vulkan driver. A software rasteriser + /// alone lands here, not in [`Self::Available`]. + Absent, + /// The probe could not look (an unsupported platform). + /// + /// The default: a value nobody has filled in is one nobody has probed, and + /// defaulting to `Absent` would let an unset field read as a fact about the + /// machine. + #[default] + Unknown, +} + +impl VulkanStatus { + /// Whether a model may be routed through Vulkan on the strength of this. + #[must_use] + pub const fn is_usable(self) -> bool { + matches!(self, Self::Available | Self::DriverPresentDevicesUnverified) + } +} + +/// One physical device as `vulkaninfo --summary` reported it. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct VulkanDevice { + pub name: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub driver_name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub driver_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub api_version: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vendor_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub device_id: Option, + /// The raw `deviceType`, e.g. `PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU`. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub device_type: Option, +} + +impl VulkanDevice { + /// AMD's PCI vendor id, as `vulkaninfo` prints it. + const AMD_VENDOR_ID: &'static str = "0x1002"; + + #[must_use] + pub fn is_amd(&self) -> bool { + self.vendor_id + .as_deref() + .is_some_and(|id| id.eq_ignore_ascii_case(Self::AMD_VENDOR_ID)) + } + + /// Whether this device is real hardware rather than a software rasteriser. + /// + /// A missing `deviceType` reads as hardware: the field is optional in the + /// summary output, and treating an unreported type as CPU would discard a + /// working GPU. Lavapipe always reports its type, so the only entries this + /// lets through unverified are ones that named no type at all. + #[must_use] + pub fn is_hardware(&self) -> bool { + !self + .device_type + .as_deref() + .is_some_and(|kind| kind.to_ascii_uppercase().contains("TYPE_CPU")) + } +} + +/// What the host offers for Vulkan, and the evidence behind it. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct VulkanSupport { + pub status: VulkanStatus, + /// The ICD manifests or registry entries found, as the loader names them. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub icds: Vec, + /// The devices `vulkaninfo` enumerated, when it was available. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub devices: Vec, + /// A one-line human summary, including why anything was excluded. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +impl VulkanSupport { + /// The driver this host would use, for the `examine` line. + /// + /// Prefers what a device actually reported (`AMD proprietary driver`, + /// `radv`) over what an ICD filename implies, because the filename is a + /// guess about a vendor and the device's own `driverName` is not. + #[must_use] + pub fn driver_label(&self) -> Option { + self.devices + .iter() + .find(|device| device.is_amd() && device.is_hardware()) + .and_then(|device| device.driver_name.clone()) + .or_else(|| { + self.icds + .iter() + .find_map(|icd| icd_driver_label(icd)) + .map(str::to_owned) + }) + } +} + +/// The driver an ICD manifest's file name implies, or `None` when the name is +/// not one this table knows. +/// +/// Deliberately includes the software rasterisers so a caller can *exclude* +/// them by name; [`icd_is_gpu_driver`] is the predicate that does so. +#[must_use] +pub fn icd_driver_label(icd: &str) -> Option<&'static str> { + let name = icd + .rsplit(['/', '\\']) + .next() + .unwrap_or(icd) + .to_ascii_lowercase(); + if name.starts_with("radeon_icd") { + return Some("Mesa RADV"); + } + if name.starts_with("amd_icd") || name.starts_with("amdvlk") { + return Some("AMDVLK"); + } + if name.starts_with("lvp_icd") { + return Some("Mesa lavapipe (software)"); + } + if name.starts_with("vk_swiftshader") { + return Some("SwiftShader (software)"); + } + if name.starts_with("nvidia_icd") { + return Some("NVIDIA"); + } + if name.starts_with("intel_icd") || name.starts_with("intel_hasvk") { + return Some("Intel"); + } + None +} + +/// Whether an ICD manifest names a driver that runs on a GPU. +/// +/// An unrecognised manifest counts as a GPU driver: a vendor the table does not +/// list yet is far more likely to be real hardware than a new software +/// rasteriser, and the cost of the two errors is not symmetric -- missing a +/// working GPU path sends the user to CPU, while admitting an unknown one is +/// corrected by the device enumeration that follows. +#[must_use] +pub fn icd_is_gpu_driver(icd: &str) -> bool { + !icd_driver_label(icd).is_some_and(|label| label.contains("(software)")) +} + +/// Parse `vulkaninfo --summary` output into its physical devices. +/// +/// Tolerates the loader warnings `vulkaninfo` prints to stdout before the +/// report, and the `Instance Extensions` / `Instance Layers` sections ahead of +/// `Devices:` -- only `GPUn:` blocks and their indented `key = value` lines are +/// read. +#[must_use] +pub fn parse_vulkaninfo_summary(output: &str) -> Vec { + let mut devices = Vec::new(); + let mut current: Option = None; + for line in output.lines() { + let trimmed = line.trim(); + if trimmed.starts_with("GPU") && trimmed.ends_with(':') { + if let Some(device) = current.take() { + devices.push(device); + } + current = Some(VulkanDevice::default()); + continue; + } + let Some(device) = current.as_mut() else { + continue; + }; + let Some((key, value)) = trimmed.split_once('=') else { + // A non-indented, non-`key = value` line ends the block: the + // summary's device section is the last thing printed, but a caller + // may hand us trailing output. + if !trimmed.is_empty() && !line.starts_with([' ', '\t']) { + devices.push(current.take().unwrap_or_default()); + } + continue; + }; + let value = value.trim().to_owned(); + if value.is_empty() { + continue; + } + match key.trim() { + "deviceName" => device.name = value, + "driverName" => device.driver_name = Some(value), + "driverID" => device.driver_id = Some(value), + "apiVersion" => device.api_version = Some(value), + "vendorID" => device.vendor_id = Some(value), + "deviceID" => device.device_id = Some(value), + "deviceType" => device.device_type = Some(value), + _ => {} + } + } + if let Some(device) = current.take() { + devices.push(device); + } + devices.retain(|device| !device.name.is_empty()); + devices +} + +/// The ICD manifest paths listed under the Windows Khronos registry key, as +/// [`WINDOWS_VULKAN_ICD_SCRIPT`] prints them (one `ICD\t` row each). +#[must_use] +pub fn parse_windows_vulkan_icd_listing(output: &str) -> Vec { + output + .lines() + .filter_map(|line| line.trim().strip_prefix("ICD\t")) + .map(str::trim) + .filter(|path| !path.is_empty()) + .map(str::to_owned) + .collect() +} + +/// Enumerate both Khronos ICD keys. The WOW6432Node key carries the 32-bit +/// drivers; a 64-bit-only install lists nothing there, which is not an error. +#[cfg(any(windows, test))] +pub(super) const WINDOWS_VULKAN_ICD_SCRIPT: &str = r#"foreach ($key in 'HKLM:\SOFTWARE\Khronos\Vulkan\Drivers','HKLM:\SOFTWARE\WOW6432Node\Khronos\Vulkan\Drivers') { if (Test-Path $key) { foreach ($name in (Get-Item $key).GetValueNames()) { "ICD`t$name" } } }"#; + +/// Every Vulkan ICD manifest this host's loader would read. +#[cfg(target_os = "linux")] +fn detect_icds() -> Vec { + let mut icds = Vec::new(); + for dir in LINUX_ICD_DIRS { + let Ok(entries) = std::fs::read_dir(host_path(dir)) else { + continue; + }; + for entry in entries.flatten() { + let path = entry.path(); + if path.extension().is_some_and(|ext| ext == "json") + && let Some(name) = path.file_name().and_then(|name| name.to_str()) + { + icds.push(name.to_owned()); + } + } + } + icds.sort(); + icds.dedup(); + icds +} + +#[cfg(windows)] +fn detect_icds() -> Vec { + capture_optional_command( + "powershell", + &[ + "-NoLogo", + "-NoProfile", + "-NonInteractive", + "-Command", + WINDOWS_VULKAN_ICD_SCRIPT, + ], + ) + .map(|output| parse_windows_vulkan_icd_listing(&output)) + .unwrap_or_default() +} + +#[cfg(not(any(target_os = "linux", windows)))] +const fn detect_icds() -> Vec { + Vec::new() +} + +/// The devices `vulkaninfo --summary` reports, or an empty list when the tool +/// is not installed. Absence of the tool is not absence of a driver, which is +/// why this cannot decide [`VulkanStatus`] on its own. +fn detect_vulkan_devices() -> Vec { + capture_optional_command("vulkaninfo", &["--summary"]) + .map(|output| parse_vulkaninfo_summary(&output)) + .unwrap_or_default() +} + +/// Whether this platform's Vulkan surfaces can be read at all. +const fn vulkan_is_probeable() -> bool { + cfg!(any(target_os = "linux", windows)) +} + +/// What this host offers for Vulkan. +#[must_use] +pub fn detect_vulkan_support() -> VulkanSupport { + classify_vulkan_support( + detect_icds(), + detect_vulkan_devices(), + vulkan_is_probeable(), + ) +} + +/// The pure decision behind [`detect_vulkan_support`], split out so every +/// combination of evidence can be tested without a Vulkan install. +#[must_use] +pub fn classify_vulkan_support( + icds: Vec, + devices: Vec, + probeable: bool, +) -> VulkanSupport { + if !probeable { + return VulkanSupport { + status: VulkanStatus::Unknown, + icds, + devices, + detail: Some("Vulkan support is not probed on this platform".to_owned()), + }; + } + + let amd_gpu = devices + .iter() + .find(|device| device.is_amd() && device.is_hardware()); + if let Some(device) = amd_gpu { + let driver = device.driver_name.as_deref().unwrap_or("an unnamed driver"); + return VulkanSupport { + detail: Some(format!("{} through {driver}", device.name)), + status: VulkanStatus::Available, + icds, + devices, + }; + } + + let software_only = !devices.is_empty() && devices.iter().all(|device| !device.is_hardware()); + if software_only { + return VulkanSupport { + status: VulkanStatus::Absent, + detail: Some( + "only a software (CPU) Vulkan device was found; that is not a GPU path".to_owned(), + ), + icds, + devices, + }; + } + + let gpu_icds: Vec<&String> = icds.iter().filter(|icd| icd_is_gpu_driver(icd)).collect(); + if gpu_icds.is_empty() { + let detail = if icds.is_empty() { + "no Vulkan driver found".to_owned() + } else { + "only a software (CPU) Vulkan driver is installed; that is not a GPU path".to_owned() + }; + return VulkanSupport { + status: VulkanStatus::Absent, + detail: Some(detail), + icds, + devices, + }; + } + + let named = gpu_icds + .iter() + .find_map(|icd| icd_driver_label(icd)) + .map_or_else( + || "a Vulkan driver is installed".to_owned(), + |label| format!("{label} is installed"), + ); + VulkanSupport { + status: VulkanStatus::DriverPresentDevicesUnverified, + detail: Some(format!( + "{named}, but no device was enumerated (install `vulkaninfo` to confirm)" + )), + icds, + devices, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The real `vulkaninfo --summary` output from a Ryzen 7 4700U (gfx90c, + /// Renoir) on Windows 11 with Adrenalin 31.0.21925.1001 -- the machine + /// ROCMAI-517 is about. Note the loader warnings ahead of the report and + /// the instance sections before `Devices:`; both must be skipped. + const RENOIR_WINDOWS_SUMMARY: &str = "\ +WARNING: [Loader Message] Code 0 : windows_read_data_files_in_registry: Registry lookup failed to get layer manifest files. +========== +VULKANINFO +========== + +Vulkan Instance Version: 1.3.301 + + +Instance Extensions: count = 13 +------------------------------- +VK_EXT_debug_report : extension revision 10 +VK_KHR_surface : extension revision 25 + +Instance Layers: count = 1 +-------------------------- +VK_LAYER_AMD_switchable_graphics AMD switchable graphics layer 1.3.260 version 1 + +Devices: +======== +GPU0: +\tapiVersion = 1.3.260 +\tdriverVersion = 2.0.279 +\tvendorID = 0x1002 +\tdeviceID = 0x1636 +\tdeviceType = PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU +\tdeviceName = AMD Radeon(TM) Graphics +\tdriverID = DRIVER_ID_AMD_PROPRIETARY +\tdriverName = AMD proprietary driver +\tdriverInfo = 26.5.2 (AMD proprietary shader compiler) +\tconformanceVersion = 1.3.3.1 +"; + + #[test] + fn the_renoir_igpu_is_read_out_of_a_real_vulkaninfo_summary() { + let devices = parse_vulkaninfo_summary(RENOIR_WINDOWS_SUMMARY); + + assert_eq!(devices.len(), 1, "{devices:?}"); + let gpu = &devices[0]; + assert_eq!(gpu.name, "AMD Radeon(TM) Graphics"); + assert_eq!(gpu.vendor_id.as_deref(), Some("0x1002")); + // The same device id the gfx-target table maps to gfx90c. + assert_eq!(gpu.device_id.as_deref(), Some("0x1636")); + assert_eq!(gpu.driver_name.as_deref(), Some("AMD proprietary driver")); + assert_eq!(gpu.driver_id.as_deref(), Some("DRIVER_ID_AMD_PROPRIETARY")); + assert_eq!(gpu.api_version.as_deref(), Some("1.3.260")); + assert!(gpu.is_amd(), "vendor 0x1002 is AMD"); + assert!(gpu.is_hardware(), "an integrated GPU is hardware"); + } + + #[test] + fn a_real_renoir_host_reports_vulkan_available_and_names_its_driver() { + let devices = parse_vulkaninfo_summary(RENOIR_WINDOWS_SUMMARY); + + let support = classify_vulkan_support(Vec::new(), devices, true); + + assert_eq!(support.status, VulkanStatus::Available); + assert!(support.status.is_usable()); + assert_eq!( + support.driver_label().as_deref(), + Some("AMD proprietary driver"), + "the device's own driverName must win over any ICD filename guess" + ); + } + + /// The defect this guards against: lavapipe is a complete Vulkan + /// implementation that runs on the CPU. Reporting it as a usable Vulkan + /// path would route a GPU-required launch onto CPU execution while telling + /// the user a GPU path was chosen -- the silent CPU fallback AGENTS.md §6 + /// forbids, reached through a backend rather than a policy bug. + #[test] + fn a_software_rasteriser_is_never_a_usable_vulkan_path() { + let lavapipe = "\ +GPU0: +\tapiVersion = 1.3.255 +\tvendorID = 0x10005 +\tdeviceType = PHYSICAL_DEVICE_TYPE_CPU +\tdeviceName = llvmpipe (LLVM 15.0.7, 256 bits) +\tdriverName = llvmpipe +"; + let devices = parse_vulkaninfo_summary(lavapipe); + assert_eq!(devices.len(), 1); + assert!(!devices[0].is_hardware()); + + let support = + classify_vulkan_support(vec!["lvp_icd.x86_64.json".to_owned()], devices, true); + + assert_eq!( + support.status, + VulkanStatus::Absent, + "a CPU rasteriser must not read as a Vulkan GPU path: {support:?}" + ); + assert!(!support.status.is_usable()); + assert!( + support + .detail + .as_deref() + .is_some_and(|detail| detail.contains("not a GPU path")), + "{support:?}" + ); + } + + #[test] + fn a_lavapipe_only_install_is_absent_even_before_any_device_is_enumerated() { + let support = + classify_vulkan_support(vec!["lvp_icd.x86_64.json".to_owned()], Vec::new(), true); + + assert_eq!(support.status, VulkanStatus::Absent); + assert!(!icd_is_gpu_driver("lvp_icd.x86_64.json")); + assert!(icd_is_gpu_driver("radeon_icd.x86_64.json")); + } + + #[test] + fn an_installed_driver_with_no_enumerated_device_stays_unverified() { + let support = + classify_vulkan_support(vec!["radeon_icd.x86_64.json".to_owned()], Vec::new(), true); + + assert_eq!(support.status, VulkanStatus::DriverPresentDevicesUnverified); + assert!( + support.status.is_usable(), + "an installed GPU driver is still worth routing to" + ); + assert_eq!(support.driver_label().as_deref(), Some("Mesa RADV")); + assert!( + support + .detail + .as_deref() + .is_some_and(|detail| detail.contains("no device was enumerated")), + "the gap must be stated, not papered over: {support:?}" + ); + } + + #[test] + fn a_host_with_nothing_installed_is_absent_and_a_platform_that_cannot_look_is_unknown() { + let nothing = classify_vulkan_support(Vec::new(), Vec::new(), true); + assert_eq!(nothing.status, VulkanStatus::Absent); + assert_eq!(nothing.detail.as_deref(), Some("no Vulkan driver found")); + assert!(nothing.driver_label().is_none()); + + // macOS and friends: no evidence either way, which must not read as + // "no Vulkan" any more than it reads as "Vulkan works". + let unprobeable = classify_vulkan_support(Vec::new(), Vec::new(), false); + assert_eq!(unprobeable.status, VulkanStatus::Unknown); + assert!(!unprobeable.status.is_usable()); + } + + #[test] + fn an_unknown_vendors_icd_is_assumed_to_be_hardware() { + // A vendor the table does not list yet must not be mistaken for a + // software rasteriser; the device enumeration corrects it if wrong. + assert!(icd_is_gpu_driver("some_new_vendor_icd.x86_64.json")); + assert_eq!(icd_driver_label("some_new_vendor_icd.x86_64.json"), None); + + let support = classify_vulkan_support( + vec!["some_new_vendor_icd.x86_64.json".to_owned()], + Vec::new(), + true, + ); + assert_eq!(support.status, VulkanStatus::DriverPresentDevicesUnverified); + } + + #[test] + fn a_discrete_amd_card_is_preferred_over_a_software_device_in_the_same_report() { + let mixed = "\ +GPU0: +\tvendorID = 0x10005 +\tdeviceType = PHYSICAL_DEVICE_TYPE_CPU +\tdeviceName = llvmpipe +\tdriverName = llvmpipe +GPU1: +\tvendorID = 0x1002 +\tdeviceType = PHYSICAL_DEVICE_TYPE_DISCRETE_GPU +\tdeviceName = AMD Radeon RX 7900 XTX +\tdriverName = radv +"; + let support = classify_vulkan_support(Vec::new(), parse_vulkaninfo_summary(mixed), true); + + assert_eq!(support.status, VulkanStatus::Available); + assert_eq!(support.driver_label().as_deref(), Some("radv")); + assert!( + support + .detail + .as_deref() + .is_some_and(|detail| detail.contains("AMD Radeon RX 7900 XTX")), + "{support:?}" + ); + } + + #[test] + fn a_non_amd_gpu_does_not_make_vulkan_an_amd_path() { + let nvidia = "\ +GPU0: +\tvendorID = 0x10de +\tdeviceType = PHYSICAL_DEVICE_TYPE_DISCRETE_GPU +\tdeviceName = NVIDIA GeForce GTX 1050 Ti +\tdriverName = NVIDIA +"; + let support = classify_vulkan_support( + vec!["nvidia_icd.json".to_owned()], + parse_vulkaninfo_summary(nvidia), + true, + ); + + // The driver is installed and is not software, so the honest answer is + // "unverified for AMD", not "available". + assert_eq!(support.status, VulkanStatus::DriverPresentDevicesUnverified); + } + + #[test] + fn the_windows_registry_listing_is_read_into_icd_paths() { + let output = "ICD\tC:\\Windows\\System32\\DriverStore\\FileRepository\\u0397342.inf_amd64_x\\amdvlk64.json\nICD\tC:\\Windows\\SysWOW64\\amdvlk32.json\n"; + + let icds = parse_windows_vulkan_icd_listing(output); + + assert_eq!(icds.len(), 2, "{icds:?}"); + assert!(icds[0].ends_with("amdvlk64.json")); + assert_eq!(icd_driver_label(&icds[0]), Some("AMDVLK")); + assert!( + icd_is_gpu_driver(&icds[0]), + "a Windows path separator must not defeat the filename table" + ); + } + + /// Both Khronos keys must be queried. A 32-bit AMDVLK registration lives + /// only under `WOW6432Node`, so reading the 64-bit key alone would report + /// "no Vulkan driver" on a host that has one. + #[test] + fn the_windows_icd_query_reads_both_the_native_and_the_wow6432_key() { + assert!( + WINDOWS_VULKAN_ICD_SCRIPT.contains(r"HKLM:\SOFTWARE\Khronos\Vulkan\Drivers"), + "{WINDOWS_VULKAN_ICD_SCRIPT}" + ); + assert!( + WINDOWS_VULKAN_ICD_SCRIPT + .contains(r"HKLM:\SOFTWARE\WOW6432Node\Khronos\Vulkan\Drivers"), + "{WINDOWS_VULKAN_ICD_SCRIPT}" + ); + // A missing key is ordinary on a host with no Vulkan at all, so the + // script must test for it rather than let PowerShell error out. + assert!( + WINDOWS_VULKAN_ICD_SCRIPT.contains("Test-Path"), + "{WINDOWS_VULKAN_ICD_SCRIPT}" + ); + // The rows the parser consumes. + assert!( + WINDOWS_VULKAN_ICD_SCRIPT.contains("\"ICD`t$name\""), + "{WINDOWS_VULKAN_ICD_SCRIPT}" + ); + } + + #[test] + fn an_empty_registry_key_yields_no_icds_rather_than_a_blank_entry() { + assert!(parse_windows_vulkan_icd_listing("").is_empty()); + assert!(parse_windows_vulkan_icd_listing("ICD\t\n").is_empty()); + assert!(parse_windows_vulkan_icd_listing("unrelated output\n").is_empty()); + } + + #[test] + fn a_vulkaninfo_report_with_no_devices_yields_none() { + assert!(parse_vulkaninfo_summary("Vulkan Instance Version: 1.3.301\n").is_empty()); + assert!(parse_vulkaninfo_summary("").is_empty()); + } +} diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 28623e3ff1..9c64329a8e 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -137,12 +137,12 @@ pub use gfx_target::{ }; pub use hardware_root::host_path; pub use host_gpu::{ - DriverSummary, ExamineSummary, HostGpuSummary, WSL_ROCDXG_READY, WindowsGpuListing, - WslHostDriverProbe, WslSummary, default_engine_for_host, default_engine_for_platform, - detect_gpu_driver_version, detect_host_gfx_target, detect_host_gpu_diagnostics, - detect_host_gpu_summary, detect_system_ram_gib, has_usable_amd_gpu, is_amdgpu_device, - is_wsl_host, pinned_gpu_name, preferred_serve_engine_for_host_gpu_summary, - usable_amd_gpu_indices, windows_amd_gpu_listing, + DriverSummary, ExamineSummary, HostGpuSummary, VulkanDevice, VulkanStatus, VulkanSupport, + WSL_ROCDXG_READY, WindowsGpuListing, WslHostDriverProbe, WslSummary, default_engine_for_host, + default_engine_for_platform, detect_gpu_driver_version, detect_host_gfx_target, + detect_host_gpu_diagnostics, detect_host_gpu_summary, detect_system_ram_gib, + detect_vulkan_support, has_usable_amd_gpu, is_amdgpu_device, is_wsl_host, pinned_gpu_name, + preferred_serve_engine_for_host_gpu_summary, usable_amd_gpu_indices, windows_amd_gpu_listing, }; #[cfg(any(target_os = "linux", test))] use host_gpu::{KfdGpuNode, kfd_gpu_nodes_in}; diff --git a/docs/architecture.md b/docs/architecture.md index 124a86c6e7..c551f8a5d5 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -202,7 +202,7 @@ Already-extracted subsystem modules, following the same full-domain-extraction p - [`gguf.rs`](../crates/rocm-core/src/gguf.rs) — reads the shape of a GGUF model (layers, KV heads, head width, parameter count, data offset) from its header over any `Read + Seek`, never touching the weights. - [`gguf_fit.rs`](../crates/rocm-core/src/gguf_fit.rs) — pure arithmetic over that header: weights + KV cache at a context size and KV type + compute buffer + runtime reserve against dedicated VRAM, the largest context that fits, and what would make a model that does not fit, fit. - [`gguf_source.rs`](../crates/rocm-core/src/gguf_source.rs) — where a header comes from: a local file, or a Hugging Face file, found by one repository-listing request (and a `HEAD` request for a file the listing does not size) and read by HTTP range requests (`HF_ENDPOINT` selects a mirror; redirects are followed by hand, up to five, and never from https to plain http). The only code in the estimate that reads a file or the network. -- [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) — (ROCMAI-85, Phase 7a of ROCMAI-27's sequencing) GPU/host hardware detection — see the module's own doc comment for what it owns. Its leaf concerns are child modules under `host_gpu/`, with `host_gpu.rs` kept as the module root: [`capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs) (running an optional tool under a timeout, trying each spelling [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs) gives for it), [`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) (kernel version, distro name, installed RAM, and the CPU model and vendor, which one reader returns together for both the summary and `Examination::probe`: Linux from `/proc/cpuinfo`, Windows from the CPU row of the system inventory), [`linux.rs`](../crates/rocm-core/src/host_gpu/linux.rs) (the sysfs, KFD and DRM probes), [`gpu_visibility.rs`](../crates/rocm-core/src/host_gpu/gpu_visibility.rs) (which AMD GPUs this process may use, from the KFD/DRM counts and the `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` masks, plus `windows_amd_gpu_listing`, the native-Windows adapter bound `rocm serve --gpu ` is checked against), [`windows_inventory.rs`](../crates/rocm-core/src/host_gpu/windows_inventory.rs) (the Windows display-adapter inventory and its parsers, including the AMD-adapter count behind `windows_amd_gpu_listing` and the system query whose CPU row the CPU reader in `os_facts.rs` takes), [`wsl.rs`](../crates/rocm-core/src/host_gpu/wsl.rs) (WSL detection, the librocdxg chain and the Windows host driver as seen from the guest), [`render.rs`](../crates/rocm-core/src/host_gpu/render.rs) (the `rocm examine` text of an `ExamineSummary`, as a second `impl` block), [`diagnostics.rs`](../crates/rocm-core/src/host_gpu/diagnostics.rs) (the GPU detection diagnostics text) and [`engine_default.rs`](../crates/rocm-core/src/host_gpu/engine_default.rs) (the default serve engine, and the vLLM preference for the TheRock families that serve through it). +- [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) — (ROCMAI-85, Phase 7a of ROCMAI-27's sequencing) GPU/host hardware detection — see the module's own doc comment for what it owns. Its leaf concerns are child modules under `host_gpu/`, with `host_gpu.rs` kept as the module root: [`capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs) (running an optional tool under a timeout, trying each spelling [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs) gives for it), [`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) (kernel version, distro name, installed RAM, and the CPU model and vendor, which one reader returns together for both the summary and `Examination::probe`: Linux from `/proc/cpuinfo`, Windows from the CPU row of the system inventory), [`linux.rs`](../crates/rocm-core/src/host_gpu/linux.rs) (the sysfs, KFD and DRM probes), [`gpu_visibility.rs`](../crates/rocm-core/src/host_gpu/gpu_visibility.rs) (which AMD GPUs this process may use, from the KFD/DRM counts and the `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` masks, plus `windows_amd_gpu_listing`, the native-Windows adapter bound `rocm serve --gpu ` is checked against), [`windows_inventory.rs`](../crates/rocm-core/src/host_gpu/windows_inventory.rs) (the Windows display-adapter inventory and its parsers, including the AMD-adapter count behind `windows_amd_gpu_listing` and the system query whose CPU row the CPU reader in `os_facts.rs` takes), [`wsl.rs`](../crates/rocm-core/src/host_gpu/wsl.rs) (WSL detection, the librocdxg chain and the Windows host driver as seen from the guest), [`render.rs`](../crates/rocm-core/src/host_gpu/render.rs) (the `rocm examine` text of an `ExamineSummary`, as a second `impl` block), [`vulkan.rs`](../crates/rocm-core/src/host_gpu/vulkan.rs) (whether a model can run through Vulkan here, read from ICD manifests, the Windows Khronos registry and `vulkaninfo --summary` — the evidence `gpu_support_level` in [`gfx_target.rs`](../crates/rocm-core/src/gfx_target.rs) needs to tell a GPU with no ROCm build that still has a Vulkan driver from one with no GPU path at all, and where a software rasteriser is recognised in order to be excluded), [`diagnostics.rs`](../crates/rocm-core/src/host_gpu/diagnostics.rs) (the GPU detection diagnostics text) and [`engine_default.rs`](../crates/rocm-core/src/host_gpu/engine_default.rs) (the default serve engine, and the vLLM preference for the TheRock families that serve through it). - [`host-probing.md`](host-probing.md) — a design note, not a module: [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) and [`examine.rs`](../crates/rocm-core/src/examine.rs) are two separate host-probing stacks that read some of the same facts twice (the Windows GPU query, the marketing-name table and the subprocess runners). It proposes which of each pair should survive, how the two differ and the order in which they could converge, and records what each convergence decided; it changes no code itself. Three pairs are converged: the `PATH` lookup ([`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs)'s `which` is now the only copy) and the kernel release ([`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs)'s `detect_kernel_version` is now the only `uname -r` reader, which [`host_env.rs`](../crates/rocm-core/src/examine/host_env.rs)'s `probe_os` calls for `kernel_release`, and `rocm examine --json` runs it once: [`examine.rs`](../crates/rocm-core/src/examine.rs)'s `Examination::probe_with_summary` probes the host and hands the examination's kernel release to the summary), and the CPU model ([`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) has one reader per platform for the model and vendor, which `probe_cpu_linux` and `probe_cpu_windows` record and the summary reads). It is a snapshot, to be retired when the convergence tickets close. - [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs) — the JSONL store engine behind the automation stores: the sibling `.lock` file (`jsonl_lock_path`, re-exported at the crate root), single-line appends, torn-tail repair, size-based rotation (`JsonlRotation`) and the tail-first readers that span rotated generations. It is generic over the record type; [`automation.rs`](../crates/rocm-core/src/automation.rs) supplies the record types and each store's policy, and nothing in this module depends on it, in production or in its own tests: those drive the engine with a small record type of their own. The module is private; its engine items are `pub(crate)`. - [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) — (ROCMAI-85, Phase 7a) the TheRock managed-runtime-environment surface — see the module's own doc comment for what it owns. diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index b0f1d34c59..0e38e084fc 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -736,3 +736,40 @@ Feature: GPU detection and system inspection Then the inspection states this host's ROCm support level And the support level agrees with the GPU the inspection detected And the support level leaves a command that fits this host + + # The three below close the gap this PR's own notes call unclosable: the + # `VulkanOnly` door needs a Renoir/Cezanne Vega APU, and no runner has one. + # It does not need the hardware, only the two signals the CLI reads for it -- + # a KFD node naming the target, and the host's Vulkan driver state -- both of + # which the harness can plant. So these run on the ungated mock lane, which is + # the lane that has neither. + # + # They exist because the support level used to be inferred: "no ROCm build" + # was taken to mean "Vulkan then", on a machine nobody had looked at. Each + # scenario pins one of the three answers that inference could not tell apart. + @id:examine-names-the-vulkan-driver-an-unsupported-gpu-can-use @requires-os:linux + Scenario: examine-49 - A GPU the official stack rejects is told which way it can still run models + Given a machine whose GPU has no official support but has a working Vulkan driver + When the user inspects that machine + Then the inspection reports the GPU as unsupported by the official stack + And the inspection reports that models can still run through Vulkan + And the inspection names the Vulkan driver it found + + @id:examine-does-not-promise-vulkan-without-a-driver @requires-os:linux + Scenario: examine-50 - A machine with no way to run models is not told it has one + Given a machine whose GPU has no official support and no Vulkan driver + When the user inspects that machine + Then the inspection reports the GPU as unsupported by the official stack + And the inspection does not claim models can run through Vulkan + And the inspection says no Vulkan driver was found + + # A software renderer is a complete Vulkan implementation that runs on the + # CPU. Counting it would put a GPU-required run on the processor while + # reporting a GPU path - the one outcome the no-fallback rule exists to + # prevent, reached through a backend rather than a policy change. + @id:examine-does-not-count-a-software-renderer-as-a-gpu-path @requires-os:linux + Scenario: examine-51 - A software renderer does not count as a way to run on the GPU + Given a machine whose GPU has no official support and only a software renderer + When the user inspects that machine + Then the inspection does not claim models can run through Vulkan + And the inspection says the only renderer it found runs on the CPU diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index f63af84819..6d92fc5769 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -1704,6 +1704,25 @@ async fn assert_no_amdgpu_fault_evidence(world: &mut E2eWorld) { } } +/// The human-readable twin of the machine-readable planted-machine step, for +/// scenarios whose subject is a sentence the report prints rather than a field +/// it serialises. +/// +/// It has to be its own step rather than reusing "the user inspects the +/// system", because that one runs through `run_rocm`, which does not consume +/// `World::command_env` -- so a planted host root never reaches the child and +/// the CLI silently describes the *runner* instead. Not hypothetical: it is +/// what made the first draft of examine-49/50/51 report the lane's own MI300X +/// as `gfx942` while claiming to be about a Renoir APU. +#[when("the user inspects that machine")] +async fn user_inspects_planted_machine_text(world: &mut E2eWorld) { + let (stdout, stderr, rc) = + crate::run_rocm_with_scenario_env(world, &["examine", "--framework", "skip"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + #[when("the user inspects that machine in machine-readable form")] async fn user_inspects_planted_machine(world: &mut E2eWorld) { let (stdout, stderr, rc) = @@ -3244,3 +3263,208 @@ async fn assert_support_level_advice_fits(world: &mut E2eWorld) { ); } } + +/// A KFD `gfx_target_version` naming gfx90c, the Vega iGPU in Renoir / +/// Lucienne / Cezanne / Barcelo APUs. +/// +/// The kernel packs the target as `major * 10000 + minor * 100 + step`, so +/// `9.0.12` is `90012` and decodes to `gfx90c` (step 12 is the hex `c`). This +/// is the one target the CLI recognizes *and* refuses to map to a ROCm package +/// family, so it is what reaches the support level these scenarios are about. +const KFD_GFX90C_TARGET_VERSION: &str = "90012"; + +/// `vulkaninfo --summary` on a host whose Vega iGPU is driven by Mesa RADV. +const RADV_VULKANINFO_SUMMARY: &str = "\ +Devices: +======== +GPU0: +\tapiVersion = 1.3.274 +\tvendorID = 0x1002 +\tdeviceID = 0x1636 +\tdeviceType = PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU +\tdeviceName = AMD Radeon Graphics (RADV RENOIR) +\tdriverID = DRIVER_ID_MESA_RADV +\tdriverName = radv +"; + +/// `vulkaninfo --summary` on a host with only Mesa's CPU rasteriser: a complete +/// Vulkan implementation, and not a GPU. +const LAVAPIPE_VULKANINFO_SUMMARY: &str = "\ +Devices: +======== +GPU0: +\tapiVersion = 1.3.255 +\tvendorID = 0x10005 +\tdeviceType = PHYSICAL_DEVICE_TYPE_CPU +\tdeviceName = llvmpipe (LLVM 15.0.7, 256 bits) +\tdriverName = llvmpipe +"; + +/// Plant a host whose GPU is identified but has no ROCm package family, +/// carrying the Vulkan state the caller describes. +/// +/// `icd_manifest` is the ICD file name the loader would find and +/// `vulkaninfo_summary` what `vulkaninfo --summary` prints; `None` for either +/// means that half of the evidence is absent. +/// +/// The `vulkaninfo` stub is planted in **both** directions. When a scenario +/// wants the tool absent it is planted as a failing command rather than left +/// out, because `PATH` still reaches the rest of the machine: a runner with a +/// real `vulkaninfo` installed would otherwise answer for it, and a scenario +/// about a host with no Vulkan driver has to be hermetic about the host and not +/// only about the planted root. +fn plant_unsupported_gpu( + world: &mut E2eWorld, + icd_manifest: Option<&str>, + vulkaninfo_summary: Option<&str>, +) { + let root = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .to_owned(); + let host_root = root.join("host-root"); + + let node = host_root.join("sys/class/kfd/kfd/topology/nodes/1"); + std::fs::create_dir_all(&node).expect("failed to create planted KFD node"); + std::fs::write( + node.join("properties"), + format!( + "cpu_cores_count 0\nsimd_count 56\ngfx_target_version {KFD_GFX90C_TARGET_VERSION}\n" + ), + ) + .expect("failed to write planted KFD node properties"); + + if let Some(manifest) = icd_manifest { + let icd_dir = host_root.join("usr/share/vulkan/icd.d"); + std::fs::create_dir_all(&icd_dir).expect("failed to create planted ICD dir"); + std::fs::write( + icd_dir.join(manifest), + "{\"file_format_version\":\"1.0.0\",\"ICD\":{\"library_path\":\"libvulkan_fake.so\"}}\n", + ) + .expect("failed to write planted ICD manifest"); + } + + let bin = root.join("fake-bin"); + std::fs::create_dir_all(&bin).expect("failed to create the fake PATH dir"); + // The same empty-bus `lspci` the other GPU fixtures plant: the probe must + // succeed without describing the runner's real cards. + rocm_core::fake_executable::write_fake_executable(&bin.join("lspci"), "#!/bin/sh\nexit 0\n") + .expect("failed to write the fake lspci"); + let vulkaninfo = match vulkaninfo_summary { + Some(summary) => format!("#!/bin/sh\ncat <<'ROCM_E2E_EOF'\n{summary}ROCM_E2E_EOF\n"), + None => "#!/bin/sh\nexit 1\n".to_owned(), + }; + rocm_core::fake_executable::write_fake_executable(&bin.join("vulkaninfo"), &vulkaninfo) + .expect("failed to write the fake vulkaninfo"); + + let path = std::env::join_paths(std::iter::once(bin).chain(std::env::split_paths( + &std::env::var_os("PATH").unwrap_or_default(), + ))) + .expect("PATH entries cannot hold a separator"); + world.command_env.push(("PATH", path)); + world + .command_env + .push(("ROCM_CLI_TEST_HOST_ROOT", host_root.into_os_string())); +} + +#[given("a machine whose GPU has no official support but has a working Vulkan driver")] +async fn unsupported_gpu_with_vulkan(world: &mut E2eWorld) { + plant_unsupported_gpu( + world, + Some("radeon_icd.x86_64.json"), + Some(RADV_VULKANINFO_SUMMARY), + ); +} + +#[given("a machine whose GPU has no official support and no Vulkan driver")] +async fn unsupported_gpu_without_vulkan(world: &mut E2eWorld) { + plant_unsupported_gpu(world, None, None); +} + +#[given("a machine whose GPU has no official support and only a software renderer")] +async fn unsupported_gpu_with_software_renderer(world: &mut E2eWorld) { + plant_unsupported_gpu( + world, + Some("lvp_icd.x86_64.json"), + Some(LAVAPIPE_VULKANINFO_SUMMARY), + ); +} + +#[then("the inspection reports the GPU as unsupported by the official stack")] +async fn assert_gpu_unsupported_by_rocm(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + assert!( + output.contains("rocm_support:") && output.contains("not supported by ROCm"), + "the inspection must say the GPU was identified and has no ROCm build:\n{output}" + ); +} + +#[then("the inspection reports that models can still run through Vulkan")] +async fn assert_vulkan_offered(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + assert!( + output.contains("the Vulkan backend can be used"), + "a host with a working Vulkan device must be told so:\n{output}" + ); + assert!( + output.contains("vulkan: available"), + "the machine-readable line must agree with the sentence:\n{output}" + ); +} + +#[then("the inspection names the Vulkan driver it found")] +async fn assert_vulkan_driver_named(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + // The driver the planted device reported, not a guess from the ICD file + // name: naming it is what lets a reader check the claim. + assert!( + output.contains("radv"), + "the inspection must name the driver it found:\n{output}" + ); +} + +#[then("the inspection does not claim models can run through Vulkan")] +async fn assert_vulkan_not_promised(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + assert!( + !output.contains("the Vulkan backend can be used"), + "a host with no usable Vulkan GPU must not be promised one:\n{output}" + ); + assert!( + !output.contains("Vulkan backend is expected to work"), + "a host with no usable Vulkan GPU must not be promised one:\n{output}" + ); +} + +#[then("the inspection says no Vulkan driver was found")] +async fn assert_no_vulkan_driver_reported(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + assert!( + output.contains("no Vulkan GPU driver was found"), + "the inspection must say why no GPU path is open:\n{output}" + ); + assert!( + output.contains("install one"), + "the advice must be able to clear the condition it is printed for:\n{output}" + ); +} + +#[then("the inspection says the only renderer it found runs on the CPU")] +async fn assert_software_renderer_reported(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no command was run"); + assert!( + output.contains("not a GPU path"), + "a CPU rasteriser must be named as such, not counted as a GPU:\n{output}" + ); + // The support level must agree, not just the evidence line. Without this the + // scenario still passed when `gpu_support_level` ignored the probe entirely: + // the absence of a Vulkan promise was coming from the status wording alone, + // so nothing here held the classification itself. + assert!( + output.contains("no Vulkan GPU driver was found"), + "a host whose only renderer is the CPU has no GPU backend, and the support \ + line must say so:\n{output}" + ); +} From 129b043b8e94b6bd611d4ccc47c2edd77545a787 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 11:19:58 +0200 Subject: [PATCH 15/37] fix(engine): clear the endpoint key file for a child that gets no key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `engine_stdio_command` set `ROCM_SERVE_API_KEY_FILE` when the caller passed one and otherwise left the variable alone. `rocm` passes `None` for every external engine plugin — a third-party binary — so whenever this process had inherited the variable, the path to a mode-0600 file holding a live endpoint credential was handed straight through. Clear it on the `None` branch. Living in the shared stdio client rather than at the `rocm` call site means `rocmd`'s plugin calls get the same treatment. The test asserts CLEARED rather than absent: `Command` records `env_remove` as an explicit `(key, None)`, so a plain "is it there" check would pass with the clearing deleted. Verified by deleting it. Later superseded: The endpoint-key clearing this added was later reset to main's version; it is not part of this branch's final diff. Signed-off-by: Roman Sirokov --- .../rocm-engine-protocol/src/stdio_client.rs | 49 +++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 5a8989e5a7..67511ad01d 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -155,6 +155,15 @@ where } if let Some(key_file) = options.endpoint_key_file { command.env(ENDPOINT_API_KEY_FILE_ENV, key_file); + } else { + // Cleared, not merely left unset. The caller said this child gets no + // endpoint key, and an engine plugin is a third-party binary: a value + // this process happens to have inherited must not reach it just + // because nothing overwrote it. `rocm`'s plugin calls pass `None` + // here, so this is what keeps a 0600 key-file path off their + // environment. See + // `an_external_plugin_child_is_not_handed_the_endpoint_key_file`. + command.env_remove(ENDPOINT_API_KEY_FILE_ENV); } if options.request_progress { command.env(ENGINE_PROGRESS_STDERR_ENV, "1"); @@ -675,6 +684,46 @@ mod tests { assert!(added_env(&command).is_empty(), "{:?}", added_env(&command)); } + /// A child told it gets no endpoint key must have the variable CLEARED, + /// not merely left unset. + /// + /// `rocm` passes `endpoint_key_file: None` for every external engine + /// plugin — a third-party binary — and the path it would otherwise carry + /// points at a mode-0600 file holding a live endpoint credential. Merely + /// not setting it would still hand the value through whenever this process + /// inherited one. + /// + /// Asserts the distinction directly: `Command` records `env_remove` as an + /// explicit `(key, None)` entry, so a plain "is it absent" check would pass + /// even with the clearing deleted. + #[test] + fn an_external_plugin_child_is_not_handed_the_endpoint_key_file() { + let key_var = std::ffi::OsStr::new(ENDPOINT_API_KEY_FILE_ENV); + let command = engine_stdio_command( + Path::new("/an/engine"), + ["stdio"], + &EngineStdioOptions { + app_paths: None, + env_root: None, + endpoint_key_file: None, + request_progress: false, + forward_stderr: false, + note_failing_exit_after_answer: false, + }, + ); + + let entry = command + .get_envs() + .find(|(key, _)| *key == key_var) + .map(|(_, value)| value); + assert_eq!( + entry, + Some(None), + "the key-file variable must be explicitly CLEARED when no key file \ + is passed, not merely left unset" + ); + } + #[test] fn each_option_sets_exactly_its_own_variable() { let paths = AppPaths { From b6c37c673a8a3a44c6035b53f966f1cc88687684 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 11:20:52 +0200 Subject: [PATCH 16/37] docs(services): condense the device-probe doc blocks to clear the size gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `apps/rocm/src/services_text.rs` crossed the 1500-line module-size limit that ROCMAI-889 made blocking — main is at 1458 and this change's 164 added lines took it to 1512, so it is this PR's overflow, not an inherited one. Trimmed the two doc blocks the change itself added, which a reviewer had already called out as heavy. The load-bearing facts stay: the ~18s bound and where its arithmetic lives, why the chat tool pays it rather than rendering a device-less view, and what the probe seam does and does not cover. What went was restatement. Signed-off-by: Roman Sirokov --- apps/rocm/src/services_text.rs | 40 ++++++++++++---------------------- 1 file changed, 14 insertions(+), 26 deletions(-) diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index 3e9aa8b7aa..cdac966bb4 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -405,33 +405,23 @@ fn format_anthropic_endpoint_line(url: &str, responding: bool) -> String { /// `rocm services list`, and the chat `services` tool, which renders the same /// text. /// -/// **This is no longer a pure local read.** Since ROCMAI-550 it makes one live -/// `Healthcheck` RPC per *ready* service to fill the `device:` row, capped in -/// aggregate by `engine_ipc::SERVICE_DEVICE_PROBE_BUDGET`. Worst case is that -/// budget plus one in-flight probe — about 18s with the built-in engines, not -/// 10s, because the budget gates the *start* of each probe and never -/// interrupts one. That doc comment carries the arithmetic and why a -/// per-probe deadline was not taken. -/// -/// The chat tool pays the same bound deliberately rather than rendering a -/// device-less view: the two surfaces would then disagree about one service -/// while claiming to show the same list, and "which GPU is this running on" is -/// a question asked in chat at least as often as at the prompt. The cost is -/// bounded and flat in the number of services, which is what makes it -/// acceptable there. +/// **No longer a pure local read.** Since ROCMAI-550 it makes one live +/// `Healthcheck` RPC per *ready* service to fill the `device:` row, bounded in +/// aggregate by `engine_ipc::SERVICE_DEVICE_PROBE_BUDGET` — see that constant +/// for the arithmetic behind the ~18s worst case. The chat tool pays the same +/// bound rather than render a device-less view, which would have the two +/// surfaces disagree while claiming to show the same list. pub(crate) fn render_services_text(paths: &AppPaths, all: bool) -> Result { render_services_text_with_devices(paths, all, |ready| fetch_service_devices(paths, ready)) } /// [`render_services_text`] with the live device probe injected. /// -/// Split out because the probe is the one thing a unit test cannot have: it is -/// a `Healthcheck` RPC to a running engine, so in default CI (no GPU, no -/// servers) it answers nothing and the `device:` row is never rendered at all — -/// deleting the row would stay green everywhere except the GPU lanes, which is -/// exactly the gap this closes. The seam takes only the probe, not the row: -/// which services are offered to it (`status == "ready"`) and where the row -/// lands both stay in the shared body, so a test can pin them. +/// Split out because the probe is the one thing a unit test cannot have: in +/// default CI it answers nothing, so deleting the `device:` row would stay +/// green outside the GPU lanes. The seam takes only the probe — which services +/// are offered to it and where the row lands stay in the shared body, so a +/// test can pin both. fn render_services_text_with_devices( paths: &AppPaths, all: bool, @@ -465,11 +455,9 @@ fn render_services_text_with_devices( } else { writeln!(output, "No local servers are running.") }; - // Gate the separator on the same condition as the hint, not just on - // `!all`: `write_past_attempts_hint` early-returns at zero, so a host - // that has never served would otherwise gain a blank line between "No - // local servers are running." and the next line - a visible change to - // the one output this PR is not meant to touch. + // Gated on the same condition as the hint, not just `!all`: + // `write_past_attempts_hint` early-returns at zero, so a host that has + // never served would otherwise gain a stray blank line here. if !all && counts.past_attempts > 0 { write_past_attempts_hint( &mut output, From 61ae34a829cdbed1cd7e455409604268f0968b07 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 13:04:21 +0200 Subject: [PATCH 17/37] docs(examine): correct the support-level section and move items above the tests `docs/testing.md` still described three support levels and pointed at `examine-39`, which is main's kfd scenario rather than anything here. It now names four levels, says why `Unknown` is distinct from the confirmed `NoGpuBackend`, cites `examine-48`, and lists the three Vulkan-tier scenarios with the lane they run on. A stale `examine-39` in `host_gpu/render.rs` is corrected too. `GpuSupportLevel` and its functions had drifted back below `#[cfg(test)] mod tests` in `gfx_target.rs`, with a `use` stranded mid-tests. Items above the test module, import hoisted to the module header. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/gfx_target.rs | 203 ++++++++++++------------ crates/rocm-core/src/host_gpu/render.rs | 2 +- docs/testing.md | 24 +-- 3 files changed, 116 insertions(+), 113 deletions(-) diff --git a/crates/rocm-core/src/gfx_target.rs b/crates/rocm-core/src/gfx_target.rs index 391c145fd3..8b843bb6b0 100644 --- a/crates/rocm-core/src/gfx_target.rs +++ b/crates/rocm-core/src/gfx_target.rs @@ -551,9 +551,110 @@ pub(crate) fn gfx_target_from_gc_version(major: u32, minor: u32, revision: u32) Some(format!("gfx{major}{minor:x}{revision:x}")) } +/// How well this host's GPU is supported, for messages that tell the user what +/// they can actually do rather than just reporting raw detection fields. +/// +/// `Unknown` deliberately covers both "nothing detected" and "a gfx target we +/// don't recognize" rather than being called `Unsupported` — see the doc +/// comment on [`gfx_target_has_no_rocm_build`] for why conflating "confirmed +/// no ROCm build" with "unrecognized" would turn a detection gap into a false +/// claim about the hardware. Only that predicate's conjunction with +/// [`normalize_therock_family`] establishes the "confirmed" half; an +/// unrecognized target satisfies neither conjunct, so it is a gap in what this +/// catalog knows rather than a fact about the card. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GpuSupportLevel { + /// A TheRock package family covers this GPU: full ROCm/HIP support. + RocmHip, + /// Recognized, confirmed to have no ROCm/HIP build, and a Vulkan GPU driver + /// was **found on this host**, so Vulkan-based engines (the default Lemonade + /// backend) run on it. + /// + /// The second half of that sentence is evidence, not inference. This variant + /// used to be returned from the gfx table alone — "no ROCm build" was taken + /// to mean "Vulkan then", on a machine nobody had looked at. Every surface + /// that renders this level tells the user to go and serve a model on it, so + /// a host with no Vulkan driver installed, or with only a software + /// rasteriser, was sent to a backend that cannot start. + VulkanOnly, + /// Recognized, confirmed to have no ROCm/HIP build, **and** no Vulkan GPU + /// driver was found: this host has no GPU path at all. + /// + /// Deliberately not folded into [`Self::Unknown`]. Both halves here are + /// confirmed — the catalog establishes the missing ROCm build, and the + /// Vulkan probe looked and found nothing — whereas `Unknown` means the CLI + /// could not establish either. Collapsing them would commit, in the other + /// direction, the exact conflation this enum's `Unknown` exists to prevent: + /// a fact about the machine would be reported as a gap in what this build + /// knows, and the user would be sent to probe a question already answered. + NoGpuBackend, + /// No GPU detected yet, or a gfx target this catalog doesn't recognize. + Unknown, +} + +/// Classify the host's GPU support level. +/// +/// Reads the two facts `ExamineSummary` and `resolve_family` already compute — +/// the detected gfx target and whichever TheRock family (if any) covers it — +/// plus what the Vulkan probe found. +/// +/// `vulkan` only ever decides between [`GpuSupportLevel::VulkanOnly`] and +/// [`GpuSupportLevel::NoGpuBackend`], and is consulted only once the catalog has +/// confirmed there is no ROCm build. A host with a covering family is `RocmHip` +/// whatever Vulkan reports, and an unrecognized target stays `Unknown`: finding +/// a Vulkan driver says nothing about whether ROCm would also work there, so it +/// must not be allowed to turn a detection gap into a support claim. +/// +/// [`VulkanStatus::Unknown`] — the platform could not be probed at all — keeps +/// `VulkanOnly`, because withholding the one path such a host has on the +/// strength of a probe that never ran would be worse than the old assumption, +/// not better. +pub fn gpu_support_level( + detected_gfx_target: Option<&str>, + compatible_therock_family: Option<&str>, + vulkan: crate::host_gpu::VulkanStatus, +) -> GpuSupportLevel { + match (detected_gfx_target, compatible_therock_family) { + (_, Some(_)) => GpuSupportLevel::RocmHip, + (Some(gfx), None) if gfx_target_has_no_rocm_build(gfx) => { + if vulkan.is_usable() || vulkan == crate::host_gpu::VulkanStatus::Unknown { + GpuSupportLevel::VulkanOnly + } else { + GpuSupportLevel::NoGpuBackend + } + } + _ => GpuSupportLevel::Unknown, + } +} + +/// [`gpu_support_level`] for a [`HostGpuSummary`] the caller already probed. +/// +/// The two fields are read here, once, rather than at each call site: they are +/// positional arguments of the same type on [`gpu_support_level`], so a call +/// site that transposed them would classify a Vulkan-only host as `RocmHip` +/// and still compile. `rocm engines` and the codex-bridge `engines` tool both +/// derive the level from this struct, so both go through this. +/// +/// `vulkan` is passed in rather than probed here so this stays a pure function +/// of what the caller already holds: the probe runs a subprocess and reads the +/// filesystem, and burying that in a helper this cheap would make every caller +/// pay for it without saying so. +#[must_use] +pub fn gpu_support_level_for_host_gpu( + host_gpu: &HostGpuSummary, + vulkan: crate::host_gpu::VulkanStatus, +) -> GpuSupportLevel { + gpu_support_level( + host_gpu.gfx_target.as_deref(), + host_gpu.therock_family.as_deref(), + vulkan, + ) +} + #[cfg(test)] mod tests { use super::*; + use crate::host_gpu::VulkanStatus; #[test] fn normalize_therock_family_maps_gfx1101_to_gfx110x_all() { @@ -972,8 +1073,6 @@ mod tests { ); } - use crate::host_gpu::VulkanStatus; - #[test] fn gpu_support_level_is_rocm_hip_when_a_family_covers_it() { assert_eq!( @@ -1051,103 +1150,3 @@ mod tests { ); } } - -/// How well this host's GPU is supported, for messages that tell the user what -/// they can actually do rather than just reporting raw detection fields. -/// -/// `Unknown` deliberately covers both "nothing detected" and "a gfx target we -/// don't recognize" rather than being called `Unsupported` — see the doc -/// comment on [`gfx_target_has_no_rocm_build`] for why conflating "confirmed -/// no ROCm build" with "unrecognized" would turn a detection gap into a false -/// claim about the hardware. Only that predicate's conjunction with -/// [`normalize_therock_family`] establishes the "confirmed" half; an -/// unrecognized target satisfies neither conjunct, so it is a gap in what this -/// catalog knows rather than a fact about the card. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum GpuSupportLevel { - /// A TheRock package family covers this GPU: full ROCm/HIP support. - RocmHip, - /// Recognized, confirmed to have no ROCm/HIP build, and a Vulkan GPU driver - /// was **found on this host**, so Vulkan-based engines (the default Lemonade - /// backend) run on it. - /// - /// The second half of that sentence is evidence, not inference. This variant - /// used to be returned from the gfx table alone — "no ROCm build" was taken - /// to mean "Vulkan then", on a machine nobody had looked at. Every surface - /// that renders this level tells the user to go and serve a model on it, so - /// a host with no Vulkan driver installed, or with only a software - /// rasteriser, was sent to a backend that cannot start. - VulkanOnly, - /// Recognized, confirmed to have no ROCm/HIP build, **and** no Vulkan GPU - /// driver was found: this host has no GPU path at all. - /// - /// Deliberately not folded into [`Self::Unknown`]. Both halves here are - /// confirmed — the catalog establishes the missing ROCm build, and the - /// Vulkan probe looked and found nothing — whereas `Unknown` means the CLI - /// could not establish either. Collapsing them would commit, in the other - /// direction, the exact conflation this enum's `Unknown` exists to prevent: - /// a fact about the machine would be reported as a gap in what this build - /// knows, and the user would be sent to probe a question already answered. - NoGpuBackend, - /// No GPU detected yet, or a gfx target this catalog doesn't recognize. - Unknown, -} - -/// Classify the host's GPU support level. -/// -/// Reads the two facts `ExamineSummary` and `resolve_family` already compute — -/// the detected gfx target and whichever TheRock family (if any) covers it — -/// plus what the Vulkan probe found. -/// -/// `vulkan` only ever decides between [`GpuSupportLevel::VulkanOnly`] and -/// [`GpuSupportLevel::NoGpuBackend`], and is consulted only once the catalog has -/// confirmed there is no ROCm build. A host with a covering family is `RocmHip` -/// whatever Vulkan reports, and an unrecognized target stays `Unknown`: finding -/// a Vulkan driver says nothing about whether ROCm would also work there, so it -/// must not be allowed to turn a detection gap into a support claim. -/// -/// [`VulkanStatus::Unknown`] — the platform could not be probed at all — keeps -/// `VulkanOnly`, because withholding the one path such a host has on the -/// strength of a probe that never ran would be worse than the old assumption, -/// not better. -pub fn gpu_support_level( - detected_gfx_target: Option<&str>, - compatible_therock_family: Option<&str>, - vulkan: crate::host_gpu::VulkanStatus, -) -> GpuSupportLevel { - match (detected_gfx_target, compatible_therock_family) { - (_, Some(_)) => GpuSupportLevel::RocmHip, - (Some(gfx), None) if gfx_target_has_no_rocm_build(gfx) => { - if vulkan.is_usable() || vulkan == crate::host_gpu::VulkanStatus::Unknown { - GpuSupportLevel::VulkanOnly - } else { - GpuSupportLevel::NoGpuBackend - } - } - _ => GpuSupportLevel::Unknown, - } -} - -/// [`gpu_support_level`] for a [`HostGpuSummary`] the caller already probed. -/// -/// The two fields are read here, once, rather than at each call site: they are -/// positional arguments of the same type on [`gpu_support_level`], so a call -/// site that transposed them would classify a Vulkan-only host as `RocmHip` -/// and still compile. `rocm engines` and the codex-bridge `engines` tool both -/// derive the level from this struct, so both go through this. -/// -/// `vulkan` is passed in rather than probed here so this stays a pure function -/// of what the caller already holds: the probe runs a subprocess and reads the -/// filesystem, and burying that in a helper this cheap would make every caller -/// pay for it without saying so. -#[must_use] -pub fn gpu_support_level_for_host_gpu( - host_gpu: &HostGpuSummary, - vulkan: crate::host_gpu::VulkanStatus, -) -> GpuSupportLevel { - gpu_support_level( - host_gpu.gfx_target.as_deref(), - host_gpu.therock_family.as_deref(), - vulkan, - ) -} diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index df88230d12..c97f075f47 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -348,7 +348,7 @@ mod tests { /// The exact placeholders the report prints for the three GPU fields when /// it has no value, pinned because they are read back as data. /// - /// examine-39 decides which support tier a host is in by filtering these + /// examine-48 decides which support tier a host is in by filtering these /// out of `detected_gfx_target`, `compatible_therock_family` and /// `detected_therock_family`. Reword one and that scenario stops seeing an /// absent value as absent: a lane would be classified into the wrong tier diff --git a/docs/testing.md b/docs/testing.md index e7e63d7604..e64ab4cdde 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -3882,16 +3882,20 @@ row is the backend and ordinal with no name — correct and unspecific, rather than specific and wrong. No CI lane has two AMD GPUs, so that branch is covered by unit tests in [host_gpu.rs](../crates/rocm-core/src/host_gpu.rs) only. -**`rocm examine` prints a `rocm_support` line on every host**, at one of three -levels (`gpu_support_level`): ROCm/HIP-supported, Vulkan-only, or unknown. -Unknown covers both "nothing detected" and "a target this build's catalog does -not recognise" — conflating the latter with "confirmed to have no ROCm build" -would turn a detection gap into a claim about the hardware. `examine-39` holds -the line against the `detected_gfx_target` and `compatible_therock_family` the -same report prints, so every lane checks the level its own hardware implies: - -```bash -cargo xtask e2e -- -n '(examine-39)' +**`rocm examine` prints a `rocm_support` line on every host**, at one of four +levels (`gpu_support_level`): ROCm/HIP-supported, Vulkan-only, no GPU backend, +or unknown. Unknown covers both "nothing detected" and "a target this build's +catalog does not recognise" — conflating the latter with "confirmed to have no +ROCm build" would turn a detection gap into a claim about the hardware, which +is why it is distinct from the `NoGpuBackend` level that *is* a confirmed +answer. `examine-48` holds the line against the `detected_gfx_target` and +`compatible_therock_family` the same report prints, so every lane checks the +level its own hardware implies; `examine-49`, `examine-50` and `examine-51` +cover the Vulkan tiers (a driver present, none present, and a software +renderer that must not count as a GPU path) and are `@requires-os:linux`: + +```bash +cargo xtask e2e -- -n '(examine-4[89]|examine-5[01])' ``` ## Windows Tool Notes From 654c2de6ce4dd8aab549f93b515ecc5de81f1d96 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 13:45:02 +0200 Subject: [PATCH 18/37] fix(engine,examine): scope the key-file clearing, and stop claiming Vulkan is absent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three review findings, all correctness rather than wording. The endpoint-key clearing was added to the SHARED stdio builder, so it reached `rocmd` too — and `rocmd` passes `endpoint_key_file: None` for a loopback service that simply needs no key, which is not the same statement as "this child may not see one". Its own test expects a keyless child to inherit the ambient value, and only passed because the variable happened to be unset. The withholding is now an explicit option the caller asks for: `rocm` sets it for external plugins, `rocmd` does not. Both directions are asserted. The Vulkan probe reported `Absent` when it found no ICD manifest and enumerated no device — but neither source is exhaustive (the Linux search ignores `VK_ICD_FILENAMES`, Windows reads only the legacy Khronos key, and `vulkaninfo` is often not installed), and `Absent` renders as a confirmed "no Vulkan GPU driver was found - install one". A host with a working driver could be told to install one it already has. No evidence either way is now `Unknown`; `Absent` is kept for a probe that found something and ruled it out. The support-level scenario required `rocm serve --engine lemonade` for any detected GPU with no covering family, but a `NoGpuBackend` host is given no serve command at all. Since the scenario is untagged it would have failed on exactly that hardware. It branches on the rendered tier now, as the renderer does. Scenario ids shift again after merging main (which took examine-48): examine-49..52 and the references that cite them. Later superseded: The key-file scoping here was later reset to main's version, and the Vulkan half moved to PR #1036; neither is in this branch's final diff. Signed-off-by: Roman Sirokov --- apps/rocm/src/engine_ipc.rs | 3 + apps/rocmd/src/engine_ipc.rs | 4 ++ crates/rocm-core/src/host_gpu/render.rs | 2 +- crates/rocm-core/src/host_gpu/vulkan.rs | 58 ++++++++++++++++--- .../rocm-engine-protocol/src/stdio_client.rs | 47 ++++++++++++--- docs/testing.md | 6 +- tests/e2e-cucumber/tests/e2e/examine_steps.rs | 11 +++- 7 files changed, 110 insertions(+), 21 deletions(-) diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index ff9793975e..b987168589 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -222,6 +222,9 @@ const fn stdio_options<'a>( app_paths: paths, env_root, endpoint_key_file: None, + // An external plugin is a third-party binary: never hand it a path to + // a live endpoint credential, including one we inherited ourselves. + withhold_inherited_key_file: true, request_progress: stream_progress, forward_stderr: true, note_failing_exit_after_answer: false, diff --git a/apps/rocmd/src/engine_ipc.rs b/apps/rocmd/src/engine_ipc.rs index f4749c3eb2..eda8243ac2 100644 --- a/apps/rocmd/src/engine_ipc.rs +++ b/apps/rocmd/src/engine_ipc.rs @@ -211,6 +211,10 @@ const fn stdio_options<'a>( app_paths: Some(paths), env_root: None, endpoint_key_file, + // `None` here means "this loopback service needs no key", not "this + // child may not see one" — these are the daemon's own children, and + // clearing would change what they inherit. + withhold_inherited_key_file: false, request_progress: false, forward_stderr: false, note_failing_exit_after_answer: true, diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index c97f075f47..721b6ed6a3 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -348,7 +348,7 @@ mod tests { /// The exact placeholders the report prints for the three GPU fields when /// it has no value, pinned because they are read back as data. /// - /// examine-48 decides which support tier a host is in by filtering these + /// examine-49 decides which support tier a host is in by filtering these /// out of `detected_gfx_target`, `compatible_therock_family` and /// `detected_therock_family`. Reword one and that scenario stops seeing an /// absent value as absent: a lane would be classified into the wrong tier diff --git a/crates/rocm-core/src/host_gpu/vulkan.rs b/crates/rocm-core/src/host_gpu/vulkan.rs index cc8cdeeaf3..b1e2b08913 100644 --- a/crates/rocm-core/src/host_gpu/vulkan.rs +++ b/crates/rocm-core/src/host_gpu/vulkan.rs @@ -383,14 +383,33 @@ pub fn classify_vulkan_support( let gpu_icds: Vec<&String> = icds.iter().filter(|icd| icd_is_gpu_driver(icd)).collect(); if gpu_icds.is_empty() { - let detail = if icds.is_empty() { - "no Vulkan driver found".to_owned() - } else { - "only a software (CPU) Vulkan driver is installed; that is not a GPU path".to_owned() - }; + // No ICD *and* no enumerated device is an absence of evidence, not + // evidence of absence: neither source is exhaustive. The Linux ICD + // search is a fixed directory list that ignores `VK_ICD_FILENAMES` and + // `VK_DRIVER_FILES`; the Windows one reads only the legacy Khronos + // registry key, which current AMD drivers need not populate; and + // `vulkaninfo` is frequently not installed on either. A working driver + // can therefore leave both empty, and `Absent` is rendered to the user + // as a confirmed fact ("no Vulkan GPU driver was found - install one"). + // Say we could not tell instead, and keep `Absent` for the cases below + // where something was actually found and ruled out. + if icds.is_empty() && devices.is_empty() { + return VulkanSupport { + status: VulkanStatus::Unknown, + detail: Some( + "no Vulkan driver was found, but this host exposes nothing that confirms one is absent either (no ICD manifest, and `vulkaninfo` did not enumerate)" + .to_owned(), + ), + icds, + devices, + }; + } return VulkanSupport { status: VulkanStatus::Absent, - detail: Some(detail), + detail: Some( + "only a software (CPU) Vulkan driver is installed; that is not a GPU path" + .to_owned(), + ), icds, devices, }; @@ -553,13 +572,34 @@ GPU0: ); } + /// No ICD and no enumerated device is an absence of evidence, not evidence + /// of absence — neither source is exhaustive (the Linux ICD search ignores + /// `VK_ICD_FILENAMES`, the Windows one reads only the legacy registry key, + /// and `vulkaninfo` is often not installed). `Absent` renders as a + /// confirmed "no Vulkan GPU driver was found - install one", so a host with + /// a working driver would be told to install one it already has. #[test] - fn a_host_with_nothing_installed_is_absent_and_a_platform_that_cannot_look_is_unknown() { + fn no_evidence_either_way_is_not_reported_as_a_confirmed_absence() { let nothing = classify_vulkan_support(Vec::new(), Vec::new(), true); - assert_eq!(nothing.status, VulkanStatus::Absent); - assert_eq!(nothing.detail.as_deref(), Some("no Vulkan driver found")); + assert_eq!( + nothing.status, + VulkanStatus::Unknown, + "an empty probe must not claim the driver is absent" + ); + assert!( + nothing + .detail + .as_deref() + .is_some_and(|detail| detail.contains("confirms")), + "the detail must say the absence is unconfirmed: {nothing:?}" + ); assert!(nothing.driver_label().is_none()); + // What `Absent` is now reserved for: something WAS found and ruled out. + let software_only = + classify_vulkan_support(vec!["lvp_icd.x86_64.json".to_owned()], Vec::new(), true); + assert_eq!(software_only.status, VulkanStatus::Absent); + // macOS and friends: no evidence either way, which must not read as // "no Vulkan" any more than it reads as "Vulkan works". let unprobeable = classify_vulkan_support(Vec::new(), Vec::new(), false); diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 67511ad01d..2dd776d15b 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -108,6 +108,15 @@ pub struct EngineStdioOptions<'a> { /// its HTTP probe of a protected endpoint authenticates. The caller decides /// whether a file holds a usable key; this only carries it. pub endpoint_key_file: Option<&'a Path>, + /// With no `endpoint_key_file`, also CLEAR [`ENDPOINT_API_KEY_FILE_ENV`] + /// rather than leave whatever this process inherited to pass through. + /// + /// For a child that must not see a credential path at all — `rocm`'s + /// external engine plugins are third-party binaries. Off by default, + /// because "no key file for this request" and "this child may not have one" + /// are different statements: `rocmd` passes `None` for a loopback service + /// that simply needs no key, and its children are its own. + pub withhold_inherited_key_file: bool, /// Ask the engine for progress on stderr ([`ENGINE_PROGRESS_STDERR_ENV`]). pub request_progress: bool, /// Write the child's stderr to this process's stderr as it arrives, so a @@ -155,13 +164,12 @@ where } if let Some(key_file) = options.endpoint_key_file { command.env(ENDPOINT_API_KEY_FILE_ENV, key_file); - } else { - // Cleared, not merely left unset. The caller said this child gets no - // endpoint key, and an engine plugin is a third-party binary: a value - // this process happens to have inherited must not reach it just - // because nothing overwrote it. `rocm`'s plugin calls pass `None` - // here, so this is what keeps a 0600 key-file path off their - // environment. See + } else if options.withhold_inherited_key_file { + // Cleared, not merely left unset: a value this process inherited must + // not reach a third-party plugin binary just because nothing + // overwrote it. Only for callers that ask — `rocmd` passes `None` for + // a loopback service that needs no key, and clearing there would + // change what its own children see. See // `an_external_plugin_child_is_not_handed_the_endpoint_key_file`. command.env_remove(ENDPOINT_API_KEY_FILE_ENV); } @@ -706,6 +714,7 @@ mod tests { app_paths: None, env_root: None, endpoint_key_file: None, + withhold_inherited_key_file: true, request_progress: false, forward_stderr: false, note_failing_exit_after_answer: false, @@ -722,6 +731,27 @@ mod tests { "the key-file variable must be explicitly CLEARED when no key file \ is passed, not merely left unset" ); + + // The other direction, which is what keeps this scoped: a caller that + // did not ask (`rocmd`, for a loopback service that simply needs no + // key) must leave the child's inherited value alone. + let kept = engine_stdio_command( + Path::new("/an/engine"), + ["stdio"], + &EngineStdioOptions { + app_paths: None, + env_root: None, + endpoint_key_file: None, + withhold_inherited_key_file: false, + request_progress: false, + forward_stderr: false, + note_failing_exit_after_answer: false, + }, + ); + assert!( + kept.get_envs().all(|(key, _)| key != key_var), + "a caller that did not ask to withhold must not touch the variable" + ); } #[test] @@ -738,6 +768,7 @@ mod tests { app_paths: Some(&paths), env_root: Some(Path::new("/envs")), endpoint_key_file: Some(Path::new("/keys/svc.endpoint-key")), + withhold_inherited_key_file: false, request_progress: true, forward_stderr: true, note_failing_exit_after_answer: false, @@ -771,6 +802,7 @@ mod tests { "ROCM_SERVE_API_KEY_FILE", EngineStdioOptions { endpoint_key_file: Some(Path::new("/keys/k")), + withhold_inherited_key_file: false, ..EngineStdioOptions::default() }, ), @@ -1192,6 +1224,7 @@ mod tests { app_paths: Some(&paths), env_root: Some(Path::new("/envs")), endpoint_key_file: Some(Path::new("/keys/k")), + withhold_inherited_key_file: false, request_progress: true, forward_stderr: false, note_failing_exit_after_answer: false, diff --git a/docs/testing.md b/docs/testing.md index 696bf8d7d9..eb48cce036 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -3895,14 +3895,14 @@ or unknown. Unknown covers both "nothing detected" and "a target this build's catalog does not recognise" — conflating the latter with "confirmed to have no ROCm build" would turn a detection gap into a claim about the hardware, which is why it is distinct from the `NoGpuBackend` level that *is* a confirmed -answer. `examine-48` holds the line against the `detected_gfx_target` and +answer. `examine-49` holds the line against the `detected_gfx_target` and `compatible_therock_family` the same report prints, so every lane checks the -level its own hardware implies; `examine-49`, `examine-50` and `examine-51` +level its own hardware implies; `examine-50`, `examine-51` and `examine-52` cover the Vulkan tiers (a driver present, none present, and a software renderer that must not count as a GPU path) and are `@requires-os:linux`: ```bash -cargo xtask e2e -- -n '(examine-4[89]|examine-5[01])' +cargo xtask e2e -- -n '(examine-49|examine-5[012])' ``` ## Windows Tool Notes diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 245cb11f3b..a5bc2b38c3 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -1712,7 +1712,7 @@ async fn assert_no_amdgpu_fault_evidence(world: &mut E2eWorld) { /// system", because that one runs through `run_rocm`, which does not consume /// `World::command_env` -- so a planted host root never reaches the child and /// the CLI silently describes the *runner* instead. Not hypothetical: it is -/// what made the first draft of examine-49/50/51 report the lane's own MI300X +/// what made the first draft of examine-50/51/52 report the lane's own MI300X /// as `gfx942` while claiming to be about a Renoir APU. #[when("the user inspects that machine")] async fn user_inspects_planted_machine_text(world: &mut E2eWorld) { @@ -3331,6 +3331,15 @@ async fn assert_support_level_advice_fits(world: &mut E2eWorld) { (_, Some(_), Some(_)) => ("rocm serve ", &["rocm install sdk"]), // ROCm-capable with no managed SDK yet: that install is the next step. (_, Some(_), None) => ("rocm install sdk", &[]), + // A GPU no package family covers, and the Vulkan probe found no GPU + // driver either: there is no backend to serve on, so the line owes a + // way to GET one rather than a serve command that would be refused. + // Branched on the rendered tier the way the renderer itself is — a + // flat expectation here failed on exactly this hardware, since this + // scenario is untagged and runs on every lane. + (Some(_), None, _) if line.contains("no Vulkan GPU driver was found") => { + ("install one", &["rocm serve", "rocm install sdk"]) + } // A GPU no package family covers — confirmed Vulkan-only (gfx90c) or // merely unrecognized. Either way the SDK has nothing for it and // Lemonade's Vulkan backend is what can serve on it. From 3754e9230d38008b85fd49e927e7bb1b1a3d9f89 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 15:20:42 +0200 Subject: [PATCH 19/37] refactor(examine): split the Vulkan probe and the protocol change out of this PR On review: at ~5.7k lines this mixed the ROCMAI-550 messaging change with two things that are not it, and the reviewer asked twice for a split. Out, into their own PRs: - the Vulkan probe (`host_gpu/vulkan.rs`, `VulkanStatus`), the `NoGpuBackend` support level, the `vulkan:` examine line and the three scenarios that cover the Vulkan tiers. `gpu_support_level` goes back to the three levels the ticket describes; - `EngineStdioOptions::withhold_inherited_key_file`, which is hardening of the shared engine-protocol crate and belongs to neither ticket. What stays is ROCMAI-550 itself: the per-host support level in place of a static ROCm claim, the `rocm_support` line, the install-sdk advice that names a command the host can actually run, and the live `device:` row with the key-file plumbing it needs. Signed-off-by: Roman Sirokov --- apps/rocm/src/bridge_snapshot.rs | 1 - apps/rocm/src/driver_plan.rs | 1 - apps/rocm/src/engine_ipc.rs | 3 - apps/rocm/src/examine_report_text.rs | 10 +- apps/rocm/src/main.rs | 24 +- apps/rocmd/src/engine_ipc.rs | 4 - crates/rocm-core/src/gfx_target.rs | 132 +--- crates/rocm-core/src/host_gpu.rs | 17 - crates/rocm-core/src/host_gpu/render.rs | 207 +---- crates/rocm-core/src/host_gpu/vulkan.rs | 726 ------------------ crates/rocm-core/src/lib.rs | 12 +- .../rocm-engine-protocol/src/stdio_client.rs | 47 +- docs/architecture.md | 4 +- tests/e2e-cucumber/features/examine.feature | 37 - tests/e2e-cucumber/tests/e2e/examine_steps.rs | 224 ------ 15 files changed, 43 insertions(+), 1406 deletions(-) delete mode 100644 crates/rocm-core/src/host_gpu/vulkan.rs diff --git a/apps/rocm/src/bridge_snapshot.rs b/apps/rocm/src/bridge_snapshot.rs index 115279e42a..741c18ede7 100644 --- a/apps/rocm/src/bridge_snapshot.rs +++ b/apps/rocm/src/bridge_snapshot.rs @@ -40,7 +40,6 @@ pub(crate) fn build_codex_bridge_snapshot(paths: &AppPaths) -> Result( app_paths: paths, env_root, endpoint_key_file: None, - // An external plugin is a third-party binary: never hand it a path to - // a live endpoint credential, including one we inherited ourselves. - withhold_inherited_key_file: true, request_progress: stream_progress, forward_stderr: true, note_failing_exit_after_answer: false, diff --git a/apps/rocm/src/examine_report_text.rs b/apps/rocm/src/examine_report_text.rs index 20aef803e3..12d66bee2b 100644 --- a/apps/rocm/src/examine_report_text.rs +++ b/apps/rocm/src/examine_report_text.rs @@ -133,13 +133,9 @@ pub(crate) fn examine_human_report( let mut output = render_examine_plain_header(&summary); output.push_str(&summary.render_text()); append_examine_runtime_state(&mut output, paths, config)?; - // `summary` already carries the Vulkan probe result, so the support level - // here is the verified one: this is the surface other messages send people - // to when they want a real answer. let gpu_support_level = rocm_core::gpu_support_level( summary.detected_gfx_target.as_deref(), summary.compatible_therock_family.as_deref(), - summary.vulkan_status(), ); append_examine_engine_inventory( &mut output, @@ -208,10 +204,7 @@ pub(crate) fn render_engine_inventory_text_with_paths(paths: Option<&AppPaths>) render_engine_inventory_text_for_host( paths, rocm_core::default_engine_for_host(&host_gpu), - // No Vulkan probe on this listing path, for the reason given on - // `host_gpu_support_level`: `rocm engines` is cheap by design, and - // `rocm examine` is where the verified answer lives. - rocm_core::gpu_support_level_for_host_gpu(&host_gpu, rocm_core::VulkanStatus::Unknown), + rocm_core::gpu_support_level_for_host_gpu(&host_gpu), ) } @@ -641,7 +634,6 @@ mod tests { detected_gfx_target: None, compatible_therock_family: None, detected_therock_family: None, - vulkan: None, driver: rocm_core::DriverSummary { policy: "linux_official_amd_dkms_wrapper".to_owned(), status: "amdgpu_available".to_owned(), diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 3887afd02e..c18432bc61 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -1081,15 +1081,6 @@ const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str rocm_core::GpuSupportLevel::RocmHip => { "default embedded Lemonade server with ROCm llama.cpp backend" } - // Only `rocm examine` reaches this level — the cheap probe behind - // `host_gpu_support_level` passes `VulkanStatus::Unknown` and so lands - // on `VulkanOnly` — but the arm is spelled out rather than folded into - // `Unknown`, because what it means is the opposite: `Unknown` promises - // that a command can still answer, and here the question is already - // answered and the answer is no. - rocm_core::GpuSupportLevel::NoGpuBackend => { - "default embedded Lemonade server; this GPU has no ROCm/HIP support and no Vulkan driver was found, so no GPU backend can run here" - } rocm_core::GpuSupportLevel::Unknown => { "default embedded Lemonade server; run `rocm examine` for this host's GPU support level, which says how to find the backend" } @@ -1102,21 +1093,8 @@ const fn lemonade_engine_note(level: rocm_core::GpuSupportLevel) -> &'static str /// The same two inputs `rocm examine` derives it from, read through /// `detect_host_gpu_summary` rather than the full examine gather — exactly what /// `render_engine_inventory_text_with_paths` (i.e. `rocm engines`) already does. -/// -/// **The Vulkan probe is deliberately not run here**, so this stays the cheap -/// probe its callers chose it for: the probe spawns `vulkaninfo` on Linux and -/// PowerShell on Windows, and `rocm engines` is a listing command that would -/// pay that on every invocation. Passing [`VulkanStatus::Unknown`] keeps these -/// surfaces answering exactly as they did before this probe existed — a -/// Vulkan-capable-by-the-table host still reads `VulkanOnly` — while `rocm -/// examine`, which already gathers the probe, is the surface that can tell a -/// host with a Vulkan driver from one without. That split is why -/// `lemonade_engine_note` sends the user to `rocm examine` for a real answer. pub(crate) fn host_gpu_support_level(paths: Option<&AppPaths>) -> rocm_core::GpuSupportLevel { - rocm_core::gpu_support_level_for_host_gpu( - &rocm_core::detect_host_gpu_summary(paths), - rocm_core::VulkanStatus::Unknown, - ) + rocm_core::gpu_support_level_for_host_gpu(&rocm_core::detect_host_gpu_summary(paths)) } fn refresh_startup_update_check_quietly() { diff --git a/apps/rocmd/src/engine_ipc.rs b/apps/rocmd/src/engine_ipc.rs index eda8243ac2..f4749c3eb2 100644 --- a/apps/rocmd/src/engine_ipc.rs +++ b/apps/rocmd/src/engine_ipc.rs @@ -211,10 +211,6 @@ const fn stdio_options<'a>( app_paths: Some(paths), env_root: None, endpoint_key_file, - // `None` here means "this loopback service needs no key", not "this - // child may not see one" — these are the daemon's own children, and - // clearing would change what they inherit. - withhold_inherited_key_file: false, request_progress: false, forward_stderr: false, note_failing_exit_after_answer: true, diff --git a/crates/rocm-core/src/gfx_target.rs b/crates/rocm-core/src/gfx_target.rs index 8b843bb6b0..092c288a46 100644 --- a/crates/rocm-core/src/gfx_target.rs +++ b/crates/rocm-core/src/gfx_target.rs @@ -566,63 +566,23 @@ pub(crate) fn gfx_target_from_gc_version(major: u32, minor: u32, revision: u32) pub enum GpuSupportLevel { /// A TheRock package family covers this GPU: full ROCm/HIP support. RocmHip, - /// Recognized, confirmed to have no ROCm/HIP build, and a Vulkan GPU driver - /// was **found on this host**, so Vulkan-based engines (the default Lemonade - /// backend) run on it. - /// - /// The second half of that sentence is evidence, not inference. This variant - /// used to be returned from the gfx table alone — "no ROCm build" was taken - /// to mean "Vulkan then", on a machine nobody had looked at. Every surface - /// that renders this level tells the user to go and serve a model on it, so - /// a host with no Vulkan driver installed, or with only a software - /// rasteriser, was sent to a backend that cannot start. + /// Recognized, confirmed to have no ROCm/HIP build, but Vulkan-based + /// engines (the default Lemonade backend) still work on it. VulkanOnly, - /// Recognized, confirmed to have no ROCm/HIP build, **and** no Vulkan GPU - /// driver was found: this host has no GPU path at all. - /// - /// Deliberately not folded into [`Self::Unknown`]. Both halves here are - /// confirmed — the catalog establishes the missing ROCm build, and the - /// Vulkan probe looked and found nothing — whereas `Unknown` means the CLI - /// could not establish either. Collapsing them would commit, in the other - /// direction, the exact conflation this enum's `Unknown` exists to prevent: - /// a fact about the machine would be reported as a gap in what this build - /// knows, and the user would be sent to probe a question already answered. - NoGpuBackend, /// No GPU detected yet, or a gfx target this catalog doesn't recognize. Unknown, } -/// Classify the host's GPU support level. -/// -/// Reads the two facts `ExamineSummary` and `resolve_family` already compute — -/// the detected gfx target and whichever TheRock family (if any) covers it — -/// plus what the Vulkan probe found. -/// -/// `vulkan` only ever decides between [`GpuSupportLevel::VulkanOnly`] and -/// [`GpuSupportLevel::NoGpuBackend`], and is consulted only once the catalog has -/// confirmed there is no ROCm build. A host with a covering family is `RocmHip` -/// whatever Vulkan reports, and an unrecognized target stays `Unknown`: finding -/// a Vulkan driver says nothing about whether ROCm would also work there, so it -/// must not be allowed to turn a detection gap into a support claim. -/// -/// [`VulkanStatus::Unknown`] — the platform could not be probed at all — keeps -/// `VulkanOnly`, because withholding the one path such a host has on the -/// strength of a probe that never ran would be worse than the old assumption, -/// not better. +/// Classify the host's GPU support level from the same two facts +/// `ExamineSummary` and `resolve_family` already compute: the detected gfx +/// target and whichever TheRock family (if any) covers it. pub fn gpu_support_level( detected_gfx_target: Option<&str>, compatible_therock_family: Option<&str>, - vulkan: crate::host_gpu::VulkanStatus, ) -> GpuSupportLevel { match (detected_gfx_target, compatible_therock_family) { (_, Some(_)) => GpuSupportLevel::RocmHip, - (Some(gfx), None) if gfx_target_has_no_rocm_build(gfx) => { - if vulkan.is_usable() || vulkan == crate::host_gpu::VulkanStatus::Unknown { - GpuSupportLevel::VulkanOnly - } else { - GpuSupportLevel::NoGpuBackend - } - } + (Some(gfx), None) if gfx_target_has_no_rocm_build(gfx) => GpuSupportLevel::VulkanOnly, _ => GpuSupportLevel::Unknown, } } @@ -634,27 +594,17 @@ pub fn gpu_support_level( /// site that transposed them would classify a Vulkan-only host as `RocmHip` /// and still compile. `rocm engines` and the codex-bridge `engines` tool both /// derive the level from this struct, so both go through this. -/// -/// `vulkan` is passed in rather than probed here so this stays a pure function -/// of what the caller already holds: the probe runs a subprocess and reads the -/// filesystem, and burying that in a helper this cheap would make every caller -/// pay for it without saying so. #[must_use] -pub fn gpu_support_level_for_host_gpu( - host_gpu: &HostGpuSummary, - vulkan: crate::host_gpu::VulkanStatus, -) -> GpuSupportLevel { +pub fn gpu_support_level_for_host_gpu(host_gpu: &HostGpuSummary) -> GpuSupportLevel { gpu_support_level( host_gpu.gfx_target.as_deref(), host_gpu.therock_family.as_deref(), - vulkan, ) } #[cfg(test)] mod tests { use super::*; - use crate::host_gpu::VulkanStatus; #[test] fn normalize_therock_family_maps_gfx1101_to_gfx110x_all() { @@ -1047,7 +997,7 @@ mod tests { therock_family: Some("gfx94X-dcgpu".to_owned()), }; assert_eq!( - gpu_support_level_for_host_gpu(&host_gpu, VulkanStatus::Available), + gpu_support_level_for_host_gpu(&host_gpu), GpuSupportLevel::RocmHip ); } @@ -1060,7 +1010,7 @@ mod tests { therock_family: None, }; assert_eq!( - gpu_support_level_for_host_gpu(&host_gpu, VulkanStatus::Available), + gpu_support_level_for_host_gpu(&host_gpu), GpuSupportLevel::VulkanOnly ); } @@ -1068,7 +1018,7 @@ mod tests { #[test] fn gpu_support_level_for_host_gpu_reads_an_undetected_summary_as_unknown() { assert_eq!( - gpu_support_level_for_host_gpu(&HostGpuSummary::default(), VulkanStatus::Available), + gpu_support_level_for_host_gpu(&HostGpuSummary::default()), GpuSupportLevel::Unknown ); } @@ -1076,76 +1026,24 @@ mod tests { #[test] fn gpu_support_level_is_rocm_hip_when_a_family_covers_it() { assert_eq!( - gpu_support_level(Some("gfx90a"), Some("gfx90a"), VulkanStatus::Available), - GpuSupportLevel::RocmHip - ); - // A covering family wins outright: a host whose Vulkan probe found - // nothing still has ROCm, and must not be downgraded for it. - assert_eq!( - gpu_support_level(Some("gfx90a"), Some("gfx90a"), VulkanStatus::Absent), + gpu_support_level(Some("gfx90a"), Some("gfx90a")), GpuSupportLevel::RocmHip ); } #[test] fn gpu_support_level_is_unknown_without_detection_or_for_an_unrecognized_target() { + assert_eq!(gpu_support_level(None, None), GpuSupportLevel::Unknown); assert_eq!( - gpu_support_level(None, None, VulkanStatus::Available), - GpuSupportLevel::Unknown - ); - // Finding a Vulkan driver says nothing about whether ROCm would work on - // an unrecognized target, so it must not promote a detection gap into a - // support claim. - assert_eq!( - gpu_support_level( - Some("gfx_totally_unrecognized"), - None, - VulkanStatus::Available - ), + gpu_support_level(Some("gfx_totally_unrecognized"), None), GpuSupportLevel::Unknown ); } #[test] - fn gpu_support_level_is_vulkan_only_for_gfx90c_with_a_vulkan_driver() { - assert_eq!( - gpu_support_level(Some("gfx90c"), None, VulkanStatus::Available), - GpuSupportLevel::VulkanOnly - ); - // An installed driver that enumerated no device is still worth routing - // to; the level says "Vulkan", and `examine`'s own line carries how sure - // the probe was. - assert_eq!( - gpu_support_level( - Some("gfx90c"), - None, - VulkanStatus::DriverPresentDevicesUnverified - ), - GpuSupportLevel::VulkanOnly - ); - } - - /// The defect this guards against: `VulkanOnly` used to be inferred from the - /// gfx table alone, so a Renoir laptop with no Vulkan driver installed — or - /// with only a CPU rasteriser — was told to go and serve a model on a - /// backend that cannot start. - #[test] - fn gpu_support_level_reports_no_gpu_backend_when_the_vulkan_probe_found_none() { - assert_eq!( - gpu_support_level(Some("gfx90c"), None, VulkanStatus::Absent), - GpuSupportLevel::NoGpuBackend, - "a confirmed-absent Vulkan driver must not read as a usable Vulkan path" - ); - } - - /// A platform the probe cannot read is not a platform without Vulkan. - /// Withholding the only path such a host has, on the strength of a probe - /// that never ran, would be a worse answer than the assumption this change - /// replaces. - #[test] - fn gpu_support_level_keeps_vulkan_when_the_probe_could_not_run() { + fn gpu_support_level_is_vulkan_only_for_gfx90c() { assert_eq!( - gpu_support_level(Some("gfx90c"), None, VulkanStatus::Unknown), + gpu_support_level(Some("gfx90c"), None), GpuSupportLevel::VulkanOnly ); } diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index db45107033..87b6ad185f 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -54,7 +54,6 @@ mod gpu_visibility; pub(crate) mod linux; mod os_facts; mod render; -mod vulkan; mod windows_inventory; mod wsl; @@ -74,11 +73,6 @@ pub(crate) use linux::detect_linux_sysfs_gfx_target; pub use linux::is_amdgpu_device; #[cfg(any(target_os = "linux", test))] pub(crate) use linux::{KfdGpuNode, kfd_gpu_nodes_in}; -pub use vulkan::{ - VulkanDevice, VulkanStatus, VulkanSupport, classify_vulkan_support, detect_vulkan_support, - icd_driver_label, icd_is_gpu_driver, parse_vulkaninfo_summary, - parse_windows_vulkan_icd_listing, -}; pub(crate) use windows_inventory::detect_windows_amd_driver_info; #[cfg(test)] use windows_inventory::parse_windows_examine_inventory; @@ -139,15 +133,6 @@ pub struct ExamineSummary { pub compatible_therock_family: Option, #[serde(default)] pub detected_therock_family: Option, - /// What the Vulkan probe found, when it could run. - /// - /// `#[serde(default)]` is load-bearing in the *reverse* direction: `rocm - /// remote doctor` deserializes summaries produced by other hosts, which may - /// be running a build from before this field existed. `None` means "not - /// probed", which [`ExamineSummary::vulkan_status`] maps to - /// [`VulkanStatus::Unknown`] — never to "no Vulkan". - #[serde(default)] - pub vulkan: Option, pub driver: DriverSummary, pub legacy_rocm: LegacyRocmSummary, #[serde(default)] @@ -299,7 +284,6 @@ impl ExamineSummary { detected_gfx_target, compatible_therock_family, detected_therock_family, - vulkan: Some(detect_vulkan_support()), driver: detect_driver_summary_with_windows_inventory( windows_inventory.as_ref(), wsl.as_ref(), @@ -746,7 +730,6 @@ mod tests { detected_gfx_target: None, compatible_therock_family: None, detected_therock_family: None, - vulkan: None, driver: windows_driver_summary( Some("AMD Radeon(TM) Graphics driver 30.0.13014.8".to_owned()), info, diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index 721b6ed6a3..e6f0f479f2 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -6,42 +6,9 @@ use super::ExamineSummary; use super::os_facts::format_gib_value; -use super::vulkan::VulkanStatus; use crate::gfx_target::{GpuSupportLevel, gpu_support_level}; impl ExamineSummary { - /// What the Vulkan probe found, or [`VulkanStatus::Unknown`] when this - /// summary carries no probe result — an older host's JSON read back by - /// `rocm remote doctor`, or a caller that built the struct by hand. - /// - /// Public because the `rocm` binary classifies its own support level from a - /// summary it already gathered, and must read the probe result the same way - /// this module does rather than reaching into the field and inventing its - /// own `None` handling. - #[must_use] - pub fn vulkan_status(&self) -> VulkanStatus { - self.vulkan - .as_ref() - .map_or(VulkanStatus::Unknown, |vulkan| vulkan.status) - } - - /// The `vulkan` line: the status, and the evidence behind it. - fn vulkan_line(&self) -> String { - let Some(vulkan) = self.vulkan.as_ref() else { - return "".to_owned(); - }; - let status = match vulkan.status { - VulkanStatus::Available => "available", - VulkanStatus::DriverPresentDevicesUnverified => "driver_present_devices_unverified", - VulkanStatus::Absent => "absent", - VulkanStatus::Unknown => "unknown", - }; - vulkan.detail.as_deref().map_or_else( - || status.to_owned(), - |detail| format!("{status} ({detail})"), - ) - } - /// The `rocm_support` line of the examine text. /// /// ROCMAI-550 asks `rocm examine` to "report the support level with one @@ -67,49 +34,14 @@ impl ExamineSummary { /// blank line. fn rocm_support_line(&self) -> String { let detected = self.detected_gfx_target.as_deref(); - let vulkan = self.vulkan_status(); - match gpu_support_level(detected, self.compatible_therock_family.as_deref(), vulkan) { + match gpu_support_level(detected, self.compatible_therock_family.as_deref()) { // `VulkanOnly` is only ever returned for a detected target, so the // fallback below is unreachable; it keeps the arm total without a // panic path. - // - // The claim is graded by what the probe actually established. An - // enumerated device is named, because naming it is what lets the - // reader check the claim rather than take it; an installed-but- - // unenumerated driver says so instead of borrowing the stronger - // sentence; and a platform that could not be probed admits it. All - // three still point at the same command, because all three are - // hosts worth trying. GpuSupportLevel::VulkanOnly => format!( - " rocm_support: GPU detected ({}) but not supported by ROCm; {} - serve on it \ - with `rocm serve --engine lemonade`\n", - detected.unwrap_or_default(), - match vulkan { - VulkanStatus::Available => self.vulkan.as_ref().map_or_else( - || "the Vulkan backend can be used".to_owned(), - |support| support.driver_label().map_or_else( - || "the Vulkan backend can be used".to_owned(), - |driver| format!("the Vulkan backend can be used ({driver})"), - ), - ), - VulkanStatus::DriverPresentDevicesUnverified => - "a Vulkan driver is installed, so the Vulkan backend is expected to work" - .to_owned(), - VulkanStatus::Absent | VulkanStatus::Unknown => - "Vulkan was not probed on this platform, so the Vulkan backend is \ - unconfirmed" - .to_owned(), - } - ), - // Both halves are established facts: the catalog confirmed there is - // no ROCm build, and the probe looked for a Vulkan driver and found - // none. Unlike `Unknown` below, there is nothing left to go and find - // out on this host, so the advice is to install a driver rather than - // to serve a model and read back what happened. - GpuSupportLevel::NoGpuBackend => format!( - " rocm_support: GPU detected ({}) but not supported by ROCm, and no Vulkan GPU \ - driver was found - install one (Mesa RADV on Linux, the AMD display driver on \ - Windows) to run models on this GPU\n", + " rocm_support: GPU detected ({}) but not supported by ROCm; \ + the Vulkan backend can still be used - serve on it with \ + `rocm serve --engine lemonade`\n", detected.unwrap_or_default() ), // The covering family is the fact; it is already printed on its own @@ -205,7 +137,7 @@ impl ExamineSummary { "false (this run's output is captured, so the CLI will not prompt)" }; format!( - "rocm examine\n os: {}\n arch: {}\n kernel: {}\n distro: {}\n cpu: {}\n system_ram: {}\n interactive_terminal: {}\n default_engine: {}\n detected_gfx_target: {}\n compatible_therock_family: {}\n{} detected_therock_family: {}\n vulkan: {}\n driver_policy: {}\n driver_status: {}\n driver_detail: {}\n{} legacy_rocm_status: {}\n legacy_rocm_paths: {}\n legacy_rocm_version: {}\n legacy_rocm_detail: {}\n legacy_rocm_guidance: {}\n wsl: {}\n wsl_dxg_device: {}\n wsl_dxcore: {}\n wsl_librocdxg: {}\n wsl_rocdxg_dids: {}\n wsl_ldconfig_librocdxg: {}\n wsl_librocdxg_on_ld_library_path: {}\n wsl_librocdxg_beside_runtime: {}\n wsl_global_rocminfo: {}\n wsl_cargo: {}\n wsl_detail: {}\n managed_runtimes: {}\n managed_services: {}\n model_cache_entries: {}\n config_dir: {}\n data_dir: {}\n cache_dir: {}\n", + "rocm examine\n os: {}\n arch: {}\n kernel: {}\n distro: {}\n cpu: {}\n system_ram: {}\n interactive_terminal: {}\n default_engine: {}\n detected_gfx_target: {}\n compatible_therock_family: {}\n{} detected_therock_family: {}\n driver_policy: {}\n driver_status: {}\n driver_detail: {}\n{} legacy_rocm_status: {}\n legacy_rocm_paths: {}\n legacy_rocm_version: {}\n legacy_rocm_detail: {}\n legacy_rocm_guidance: {}\n wsl: {}\n wsl_dxg_device: {}\n wsl_dxcore: {}\n wsl_librocdxg: {}\n wsl_rocdxg_dids: {}\n wsl_ldconfig_librocdxg: {}\n wsl_librocdxg_on_ld_library_path: {}\n wsl_librocdxg_beside_runtime: {}\n wsl_global_rocminfo: {}\n wsl_cargo: {}\n wsl_detail: {}\n managed_runtimes: {}\n managed_services: {}\n model_cache_entries: {}\n config_dir: {}\n data_dir: {}\n cache_dir: {}\n", self.os, self.arch, self.kernel.as_deref().unwrap_or(""), @@ -223,7 +155,6 @@ impl ExamineSummary { self.detected_therock_family .as_deref() .unwrap_or(""), - self.vulkan_line(), self.driver.policy, self.driver.status, self.driver.detail.as_deref().unwrap_or(""), @@ -286,8 +217,8 @@ mod tests { use super::*; use crate::LegacyRocmSummary; use crate::gfx_target::normalize_therock_family; + use crate::host_gpu::DriverSummary; use crate::host_gpu::tests::windows_summary_with_driver; - use crate::host_gpu::{DriverSummary, VulkanDevice, VulkanSupport}; use std::path::PathBuf; #[test] @@ -304,7 +235,6 @@ mod tests { detected_gfx_target: None, compatible_therock_family: Some("gfx120X-all".to_owned()), detected_therock_family: None, - vulkan: None, driver: DriverSummary { policy: "windows_validate_only".to_owned(), status: "amd_display_driver_detected".to_owned(), @@ -384,7 +314,6 @@ mod tests { detected_gfx_target: detected_gfx_target.map(str::to_owned), compatible_therock_family: compatible_therock_family.map(str::to_owned), detected_therock_family: None, - vulkan: None, driver: DriverSummary { policy: "windows_validate_only".to_owned(), status: "amd_display_driver_detected".to_owned(), @@ -407,41 +336,12 @@ mod tests { } } - /// A gfx90c summary carrying the given Vulkan evidence. - /// - /// gfx90c is detected from its PCI ID but has no ROCm package family, so it - /// is the target that reaches the Vulkan-graded arms at all. - fn gfx90c_summary_with_vulkan(vulkan: Option) -> ExamineSummary { - let family = normalize_therock_family("gfx90c"); - let mut summary = examine_summary_with_gpu(Some("gfx90c"), family.as_deref()); - summary.vulkan = vulkan; - summary - } - - fn vulkan_support(status: VulkanStatus, detail: &str) -> Option { - Some(VulkanSupport { - status, - detail: Some(detail.to_owned()), - devices: vec![VulkanDevice { - name: "AMD Radeon Graphics (RADV RENOIR)".to_owned(), - driver_name: Some("radv".to_owned()), - vendor_id: Some("0x1002".to_owned()), - device_type: Some("PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU".to_owned()), - ..VulkanDevice::default() - }], - ..VulkanSupport::default() - }) - } - #[test] fn examine_render_says_a_gpu_without_a_rocm_family_is_unsupported() { - let summary = gfx90c_summary_with_vulkan(vulkan_support( - VulkanStatus::Available, - "AMD Radeon Graphics (RADV RENOIR) through radv", - )); - + // gfx90c is detected from its PCI ID but has no ROCm package family. + let family = normalize_therock_family("gfx90c"); + let summary = examine_summary_with_gpu(Some("gfx90c"), family.as_deref()); let rendered = summary.render_text(); - assert!( rendered.contains("detected_gfx_target: gfx90c"), "{rendered}" @@ -451,92 +351,9 @@ mod tests { "{rendered}" ); assert!( - rendered.contains("the Vulkan backend can be used (radv)"), - "a confirmed device must be named, so the claim can be checked: {rendered}" - ); - assert!( - rendered.contains("vulkan: available"), - "the evidence line must carry the verdict too: {rendered}" - ); - } - - /// The defect this guards against: the `VulkanOnly` line used to promise - /// "the Vulkan backend can still be used" on every gfx90c host, derived from - /// the gfx table alone. A Renoir laptop with no Vulkan driver installed was - /// told to go and serve a model on a backend that cannot start — and under - /// the GPU-required policy there is nothing for it to fall back to. - #[test] - fn examine_render_does_not_promise_vulkan_on_a_host_that_has_none() { - let summary = gfx90c_summary_with_vulkan(Some(VulkanSupport { - status: VulkanStatus::Absent, - detail: Some("no Vulkan driver found".to_owned()), - ..VulkanSupport::default() - })); - - let rendered = summary.render_text(); - - assert!( - !rendered.contains("the Vulkan backend can be used"), - "a host with no Vulkan driver must not be promised one: {rendered}" - ); - assert!( - rendered.contains("no Vulkan GPU driver was found"), + rendered.contains("the Vulkan backend can still be used"), "{rendered}" ); - assert!( - rendered.contains("install one"), - "the advice must be able to clear the condition it is printed for: {rendered}" - ); - assert!(rendered.contains("vulkan: absent"), "{rendered}"); - } - - /// An installed driver that enumerated no device is a weaker claim than a - /// device that answered, and says so rather than borrowing the stronger - /// sentence. - #[test] - fn examine_render_separates_an_installed_driver_from_a_confirmed_device() { - let summary = gfx90c_summary_with_vulkan(Some(VulkanSupport { - status: VulkanStatus::DriverPresentDevicesUnverified, - detail: Some("Mesa RADV is installed, but no device was enumerated".to_owned()), - ..VulkanSupport::default() - })); - - let rendered = summary.render_text(); - - assert!( - rendered.contains("a Vulkan driver is installed, so the Vulkan backend is expected"), - "{rendered}" - ); - assert!( - rendered.contains("vulkan: driver_present_devices_unverified"), - "{rendered}" - ); - } - - /// A summary with no probe result at all — an older host's JSON read back - /// through `rocm remote doctor` — must not read as "no Vulkan". It keeps - /// the Vulkan advice, because withholding the only path such a host has on - /// the strength of a probe that never ran would be worse than the - /// assumption this change replaces. - #[test] - fn examine_render_treats_an_unprobed_summary_as_unconfirmed_not_absent() { - let summary = gfx90c_summary_with_vulkan(None); - - let rendered = summary.render_text(); - - assert!(rendered.contains("vulkan: "), "{rendered}"); - assert!( - rendered.contains("the Vulkan backend is unconfirmed"), - "{rendered}" - ); - assert!( - !rendered.contains("no Vulkan GPU driver was found"), - "an unprobed host is not a host without Vulkan: {rendered}" - ); - assert!( - rendered.contains("rocm serve --engine lemonade"), - "it is still worth trying, so the command stays: {rendered}" - ); } /// An unmapped target must not be *claimed* unsupported, even though every @@ -644,7 +461,6 @@ mod tests { detected_gfx_target: None, compatible_therock_family: None, detected_therock_family: None, - vulkan: None, driver: DriverSummary { policy: "linux_official_amd_dkms_wrapper".to_owned(), status: "amdgpu_available".to_owned(), @@ -699,7 +515,6 @@ mod tests { detected_gfx_target: None, compatible_therock_family: None, detected_therock_family: None, - vulkan: None, driver: DriverSummary { policy: "linux_official_amd_dkms_wrapper".to_owned(), status: "amdgpu_available".to_owned(), @@ -795,7 +610,7 @@ mod tests { .find(|l| l.trim_start().starts_with("rocm_support:")) .expect("every host gets a support line"); assert_eq!( - gpu_support_level(None, None, VulkanStatus::Unknown), + gpu_support_level(None, None), GpuSupportLevel::Unknown, "precondition: nothing detected is the Unknown tier" ); diff --git a/crates/rocm-core/src/host_gpu/vulkan.rs b/crates/rocm-core/src/host_gpu/vulkan.rs deleted file mode 100644 index b1e2b08913..0000000000 --- a/crates/rocm-core/src/host_gpu/vulkan.rs +++ /dev/null @@ -1,726 +0,0 @@ -// Copyright © Advanced Micro Devices, Inc., or its affiliates. -// -// SPDX-License-Identifier: MIT - -//! Whether this host can run a model through Vulkan. -//! -//! The CLI has always *asserted* this for a GPU with no ROCm build -- `examine` -//! told a gfx90c user "the Vulkan backend can still be used" without ever -//! looking. This probe is what makes that sentence evidence rather than a -//! promise, and it is what lets a host outside the ROCm compatibility matrix be -//! described by what it can do instead of only by what it cannot. -//! -//! Two sources, in increasing order of strength: -//! -//! 1. An **installable driver**: an ICD manifest on Linux (`/usr/share/vulkan/ -//! icd.d` and friends), or the Khronos registry key on Windows. This proves a -//! driver is *installed*, not that a device enumerates. -//! 2. An **enumerated device**: `vulkaninfo --summary`, when it is on PATH. -//! This proves a device actually answers. -//! -//! The two are kept apart by [`VulkanStatus`] rather than collapsed, because -//! collapsing them would replace one unverified promise with another. -//! -//! **A software rasteriser is not a GPU.** Mesa's lavapipe (`lvp_icd.*.json`, -//! `PHYSICAL_DEVICE_TYPE_CPU`) presents a complete Vulkan implementation that -//! runs entirely on the CPU. Counting it here would route a user onto CPU -//! execution while reporting a GPU path -- precisely the silent CPU fallback -//! `AGENTS.md` §6 forbids -- so it is recognised in order to be excluded, and -//! named in `detail` so the exclusion is visible rather than mysterious. - -use super::capture::capture_optional_command; -#[cfg(target_os = "linux")] -use crate::host_path; -use serde::{Deserialize, Serialize}; - -/// The directories a Linux Vulkan loader reads ICD manifests from. -/// -/// Read through [`host_path`] so a simulated host root can plant them, which is -/// what lets the no-GPU CI lane cover this probe at all. -#[cfg(target_os = "linux")] -const LINUX_ICD_DIRS: [&str; 4] = [ - "/usr/share/vulkan/icd.d", - "/usr/local/share/vulkan/icd.d", - "/etc/vulkan/icd.d", - "/usr/lib/x86_64-linux-gnu/vulkan/icd.d", -]; - -/// How sure the CLI is that a model can run through Vulkan here. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] -#[serde(rename_all = "snake_case")] -pub enum VulkanStatus { - /// A Vulkan device answered and it is a real GPU. - Available, - /// A driver is installed, but no device was enumerated -- `vulkaninfo` is - /// not on PATH, so nothing confirmed a device answers. Likely to work; - /// not established. - DriverPresentDevicesUnverified, - /// The probe looked and found no GPU Vulkan driver. A software rasteriser - /// alone lands here, not in [`Self::Available`]. - Absent, - /// The probe could not look (an unsupported platform). - /// - /// The default: a value nobody has filled in is one nobody has probed, and - /// defaulting to `Absent` would let an unset field read as a fact about the - /// machine. - #[default] - Unknown, -} - -impl VulkanStatus { - /// Whether a model may be routed through Vulkan on the strength of this. - #[must_use] - pub const fn is_usable(self) -> bool { - matches!(self, Self::Available | Self::DriverPresentDevicesUnverified) - } -} - -/// One physical device as `vulkaninfo --summary` reported it. -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct VulkanDevice { - pub name: String, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub driver_name: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub driver_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub api_version: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub vendor_id: Option, - #[serde(default, skip_serializing_if = "Option::is_none")] - pub device_id: Option, - /// The raw `deviceType`, e.g. `PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU`. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub device_type: Option, -} - -impl VulkanDevice { - /// AMD's PCI vendor id, as `vulkaninfo` prints it. - const AMD_VENDOR_ID: &'static str = "0x1002"; - - #[must_use] - pub fn is_amd(&self) -> bool { - self.vendor_id - .as_deref() - .is_some_and(|id| id.eq_ignore_ascii_case(Self::AMD_VENDOR_ID)) - } - - /// Whether this device is real hardware rather than a software rasteriser. - /// - /// A missing `deviceType` reads as hardware: the field is optional in the - /// summary output, and treating an unreported type as CPU would discard a - /// working GPU. Lavapipe always reports its type, so the only entries this - /// lets through unverified are ones that named no type at all. - #[must_use] - pub fn is_hardware(&self) -> bool { - !self - .device_type - .as_deref() - .is_some_and(|kind| kind.to_ascii_uppercase().contains("TYPE_CPU")) - } -} - -/// What the host offers for Vulkan, and the evidence behind it. -#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct VulkanSupport { - pub status: VulkanStatus, - /// The ICD manifests or registry entries found, as the loader names them. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub icds: Vec, - /// The devices `vulkaninfo` enumerated, when it was available. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub devices: Vec, - /// A one-line human summary, including why anything was excluded. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub detail: Option, -} - -impl VulkanSupport { - /// The driver this host would use, for the `examine` line. - /// - /// Prefers what a device actually reported (`AMD proprietary driver`, - /// `radv`) over what an ICD filename implies, because the filename is a - /// guess about a vendor and the device's own `driverName` is not. - #[must_use] - pub fn driver_label(&self) -> Option { - self.devices - .iter() - .find(|device| device.is_amd() && device.is_hardware()) - .and_then(|device| device.driver_name.clone()) - .or_else(|| { - self.icds - .iter() - .find_map(|icd| icd_driver_label(icd)) - .map(str::to_owned) - }) - } -} - -/// The driver an ICD manifest's file name implies, or `None` when the name is -/// not one this table knows. -/// -/// Deliberately includes the software rasterisers so a caller can *exclude* -/// them by name; [`icd_is_gpu_driver`] is the predicate that does so. -#[must_use] -pub fn icd_driver_label(icd: &str) -> Option<&'static str> { - let name = icd - .rsplit(['/', '\\']) - .next() - .unwrap_or(icd) - .to_ascii_lowercase(); - if name.starts_with("radeon_icd") { - return Some("Mesa RADV"); - } - if name.starts_with("amd_icd") || name.starts_with("amdvlk") { - return Some("AMDVLK"); - } - if name.starts_with("lvp_icd") { - return Some("Mesa lavapipe (software)"); - } - if name.starts_with("vk_swiftshader") { - return Some("SwiftShader (software)"); - } - if name.starts_with("nvidia_icd") { - return Some("NVIDIA"); - } - if name.starts_with("intel_icd") || name.starts_with("intel_hasvk") { - return Some("Intel"); - } - None -} - -/// Whether an ICD manifest names a driver that runs on a GPU. -/// -/// An unrecognised manifest counts as a GPU driver: a vendor the table does not -/// list yet is far more likely to be real hardware than a new software -/// rasteriser, and the cost of the two errors is not symmetric -- missing a -/// working GPU path sends the user to CPU, while admitting an unknown one is -/// corrected by the device enumeration that follows. -#[must_use] -pub fn icd_is_gpu_driver(icd: &str) -> bool { - !icd_driver_label(icd).is_some_and(|label| label.contains("(software)")) -} - -/// Parse `vulkaninfo --summary` output into its physical devices. -/// -/// Tolerates the loader warnings `vulkaninfo` prints to stdout before the -/// report, and the `Instance Extensions` / `Instance Layers` sections ahead of -/// `Devices:` -- only `GPUn:` blocks and their indented `key = value` lines are -/// read. -#[must_use] -pub fn parse_vulkaninfo_summary(output: &str) -> Vec { - let mut devices = Vec::new(); - let mut current: Option = None; - for line in output.lines() { - let trimmed = line.trim(); - if trimmed.starts_with("GPU") && trimmed.ends_with(':') { - if let Some(device) = current.take() { - devices.push(device); - } - current = Some(VulkanDevice::default()); - continue; - } - let Some(device) = current.as_mut() else { - continue; - }; - let Some((key, value)) = trimmed.split_once('=') else { - // A non-indented, non-`key = value` line ends the block: the - // summary's device section is the last thing printed, but a caller - // may hand us trailing output. - if !trimmed.is_empty() && !line.starts_with([' ', '\t']) { - devices.push(current.take().unwrap_or_default()); - } - continue; - }; - let value = value.trim().to_owned(); - if value.is_empty() { - continue; - } - match key.trim() { - "deviceName" => device.name = value, - "driverName" => device.driver_name = Some(value), - "driverID" => device.driver_id = Some(value), - "apiVersion" => device.api_version = Some(value), - "vendorID" => device.vendor_id = Some(value), - "deviceID" => device.device_id = Some(value), - "deviceType" => device.device_type = Some(value), - _ => {} - } - } - if let Some(device) = current.take() { - devices.push(device); - } - devices.retain(|device| !device.name.is_empty()); - devices -} - -/// The ICD manifest paths listed under the Windows Khronos registry key, as -/// [`WINDOWS_VULKAN_ICD_SCRIPT`] prints them (one `ICD\t` row each). -#[must_use] -pub fn parse_windows_vulkan_icd_listing(output: &str) -> Vec { - output - .lines() - .filter_map(|line| line.trim().strip_prefix("ICD\t")) - .map(str::trim) - .filter(|path| !path.is_empty()) - .map(str::to_owned) - .collect() -} - -/// Enumerate both Khronos ICD keys. The WOW6432Node key carries the 32-bit -/// drivers; a 64-bit-only install lists nothing there, which is not an error. -#[cfg(any(windows, test))] -pub(super) const WINDOWS_VULKAN_ICD_SCRIPT: &str = r#"foreach ($key in 'HKLM:\SOFTWARE\Khronos\Vulkan\Drivers','HKLM:\SOFTWARE\WOW6432Node\Khronos\Vulkan\Drivers') { if (Test-Path $key) { foreach ($name in (Get-Item $key).GetValueNames()) { "ICD`t$name" } } }"#; - -/// Every Vulkan ICD manifest this host's loader would read. -#[cfg(target_os = "linux")] -fn detect_icds() -> Vec { - let mut icds = Vec::new(); - for dir in LINUX_ICD_DIRS { - let Ok(entries) = std::fs::read_dir(host_path(dir)) else { - continue; - }; - for entry in entries.flatten() { - let path = entry.path(); - if path.extension().is_some_and(|ext| ext == "json") - && let Some(name) = path.file_name().and_then(|name| name.to_str()) - { - icds.push(name.to_owned()); - } - } - } - icds.sort(); - icds.dedup(); - icds -} - -#[cfg(windows)] -fn detect_icds() -> Vec { - capture_optional_command( - "powershell", - &[ - "-NoLogo", - "-NoProfile", - "-NonInteractive", - "-Command", - WINDOWS_VULKAN_ICD_SCRIPT, - ], - ) - .map(|output| parse_windows_vulkan_icd_listing(&output)) - .unwrap_or_default() -} - -#[cfg(not(any(target_os = "linux", windows)))] -const fn detect_icds() -> Vec { - Vec::new() -} - -/// The devices `vulkaninfo --summary` reports, or an empty list when the tool -/// is not installed. Absence of the tool is not absence of a driver, which is -/// why this cannot decide [`VulkanStatus`] on its own. -fn detect_vulkan_devices() -> Vec { - capture_optional_command("vulkaninfo", &["--summary"]) - .map(|output| parse_vulkaninfo_summary(&output)) - .unwrap_or_default() -} - -/// Whether this platform's Vulkan surfaces can be read at all. -const fn vulkan_is_probeable() -> bool { - cfg!(any(target_os = "linux", windows)) -} - -/// What this host offers for Vulkan. -#[must_use] -pub fn detect_vulkan_support() -> VulkanSupport { - classify_vulkan_support( - detect_icds(), - detect_vulkan_devices(), - vulkan_is_probeable(), - ) -} - -/// The pure decision behind [`detect_vulkan_support`], split out so every -/// combination of evidence can be tested without a Vulkan install. -#[must_use] -pub fn classify_vulkan_support( - icds: Vec, - devices: Vec, - probeable: bool, -) -> VulkanSupport { - if !probeable { - return VulkanSupport { - status: VulkanStatus::Unknown, - icds, - devices, - detail: Some("Vulkan support is not probed on this platform".to_owned()), - }; - } - - let amd_gpu = devices - .iter() - .find(|device| device.is_amd() && device.is_hardware()); - if let Some(device) = amd_gpu { - let driver = device.driver_name.as_deref().unwrap_or("an unnamed driver"); - return VulkanSupport { - detail: Some(format!("{} through {driver}", device.name)), - status: VulkanStatus::Available, - icds, - devices, - }; - } - - let software_only = !devices.is_empty() && devices.iter().all(|device| !device.is_hardware()); - if software_only { - return VulkanSupport { - status: VulkanStatus::Absent, - detail: Some( - "only a software (CPU) Vulkan device was found; that is not a GPU path".to_owned(), - ), - icds, - devices, - }; - } - - let gpu_icds: Vec<&String> = icds.iter().filter(|icd| icd_is_gpu_driver(icd)).collect(); - if gpu_icds.is_empty() { - // No ICD *and* no enumerated device is an absence of evidence, not - // evidence of absence: neither source is exhaustive. The Linux ICD - // search is a fixed directory list that ignores `VK_ICD_FILENAMES` and - // `VK_DRIVER_FILES`; the Windows one reads only the legacy Khronos - // registry key, which current AMD drivers need not populate; and - // `vulkaninfo` is frequently not installed on either. A working driver - // can therefore leave both empty, and `Absent` is rendered to the user - // as a confirmed fact ("no Vulkan GPU driver was found - install one"). - // Say we could not tell instead, and keep `Absent` for the cases below - // where something was actually found and ruled out. - if icds.is_empty() && devices.is_empty() { - return VulkanSupport { - status: VulkanStatus::Unknown, - detail: Some( - "no Vulkan driver was found, but this host exposes nothing that confirms one is absent either (no ICD manifest, and `vulkaninfo` did not enumerate)" - .to_owned(), - ), - icds, - devices, - }; - } - return VulkanSupport { - status: VulkanStatus::Absent, - detail: Some( - "only a software (CPU) Vulkan driver is installed; that is not a GPU path" - .to_owned(), - ), - icds, - devices, - }; - } - - let named = gpu_icds - .iter() - .find_map(|icd| icd_driver_label(icd)) - .map_or_else( - || "a Vulkan driver is installed".to_owned(), - |label| format!("{label} is installed"), - ); - VulkanSupport { - status: VulkanStatus::DriverPresentDevicesUnverified, - detail: Some(format!( - "{named}, but no device was enumerated (install `vulkaninfo` to confirm)" - )), - icds, - devices, - } -} - -#[cfg(test)] -mod tests { - use super::*; - - /// The real `vulkaninfo --summary` output from a Ryzen 7 4700U (gfx90c, - /// Renoir) on Windows 11 with Adrenalin 31.0.21925.1001 -- the machine - /// ROCMAI-517 is about. Note the loader warnings ahead of the report and - /// the instance sections before `Devices:`; both must be skipped. - const RENOIR_WINDOWS_SUMMARY: &str = "\ -WARNING: [Loader Message] Code 0 : windows_read_data_files_in_registry: Registry lookup failed to get layer manifest files. -========== -VULKANINFO -========== - -Vulkan Instance Version: 1.3.301 - - -Instance Extensions: count = 13 -------------------------------- -VK_EXT_debug_report : extension revision 10 -VK_KHR_surface : extension revision 25 - -Instance Layers: count = 1 --------------------------- -VK_LAYER_AMD_switchable_graphics AMD switchable graphics layer 1.3.260 version 1 - -Devices: -======== -GPU0: -\tapiVersion = 1.3.260 -\tdriverVersion = 2.0.279 -\tvendorID = 0x1002 -\tdeviceID = 0x1636 -\tdeviceType = PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU -\tdeviceName = AMD Radeon(TM) Graphics -\tdriverID = DRIVER_ID_AMD_PROPRIETARY -\tdriverName = AMD proprietary driver -\tdriverInfo = 26.5.2 (AMD proprietary shader compiler) -\tconformanceVersion = 1.3.3.1 -"; - - #[test] - fn the_renoir_igpu_is_read_out_of_a_real_vulkaninfo_summary() { - let devices = parse_vulkaninfo_summary(RENOIR_WINDOWS_SUMMARY); - - assert_eq!(devices.len(), 1, "{devices:?}"); - let gpu = &devices[0]; - assert_eq!(gpu.name, "AMD Radeon(TM) Graphics"); - assert_eq!(gpu.vendor_id.as_deref(), Some("0x1002")); - // The same device id the gfx-target table maps to gfx90c. - assert_eq!(gpu.device_id.as_deref(), Some("0x1636")); - assert_eq!(gpu.driver_name.as_deref(), Some("AMD proprietary driver")); - assert_eq!(gpu.driver_id.as_deref(), Some("DRIVER_ID_AMD_PROPRIETARY")); - assert_eq!(gpu.api_version.as_deref(), Some("1.3.260")); - assert!(gpu.is_amd(), "vendor 0x1002 is AMD"); - assert!(gpu.is_hardware(), "an integrated GPU is hardware"); - } - - #[test] - fn a_real_renoir_host_reports_vulkan_available_and_names_its_driver() { - let devices = parse_vulkaninfo_summary(RENOIR_WINDOWS_SUMMARY); - - let support = classify_vulkan_support(Vec::new(), devices, true); - - assert_eq!(support.status, VulkanStatus::Available); - assert!(support.status.is_usable()); - assert_eq!( - support.driver_label().as_deref(), - Some("AMD proprietary driver"), - "the device's own driverName must win over any ICD filename guess" - ); - } - - /// The defect this guards against: lavapipe is a complete Vulkan - /// implementation that runs on the CPU. Reporting it as a usable Vulkan - /// path would route a GPU-required launch onto CPU execution while telling - /// the user a GPU path was chosen -- the silent CPU fallback AGENTS.md §6 - /// forbids, reached through a backend rather than a policy bug. - #[test] - fn a_software_rasteriser_is_never_a_usable_vulkan_path() { - let lavapipe = "\ -GPU0: -\tapiVersion = 1.3.255 -\tvendorID = 0x10005 -\tdeviceType = PHYSICAL_DEVICE_TYPE_CPU -\tdeviceName = llvmpipe (LLVM 15.0.7, 256 bits) -\tdriverName = llvmpipe -"; - let devices = parse_vulkaninfo_summary(lavapipe); - assert_eq!(devices.len(), 1); - assert!(!devices[0].is_hardware()); - - let support = - classify_vulkan_support(vec!["lvp_icd.x86_64.json".to_owned()], devices, true); - - assert_eq!( - support.status, - VulkanStatus::Absent, - "a CPU rasteriser must not read as a Vulkan GPU path: {support:?}" - ); - assert!(!support.status.is_usable()); - assert!( - support - .detail - .as_deref() - .is_some_and(|detail| detail.contains("not a GPU path")), - "{support:?}" - ); - } - - #[test] - fn a_lavapipe_only_install_is_absent_even_before_any_device_is_enumerated() { - let support = - classify_vulkan_support(vec!["lvp_icd.x86_64.json".to_owned()], Vec::new(), true); - - assert_eq!(support.status, VulkanStatus::Absent); - assert!(!icd_is_gpu_driver("lvp_icd.x86_64.json")); - assert!(icd_is_gpu_driver("radeon_icd.x86_64.json")); - } - - #[test] - fn an_installed_driver_with_no_enumerated_device_stays_unverified() { - let support = - classify_vulkan_support(vec!["radeon_icd.x86_64.json".to_owned()], Vec::new(), true); - - assert_eq!(support.status, VulkanStatus::DriverPresentDevicesUnverified); - assert!( - support.status.is_usable(), - "an installed GPU driver is still worth routing to" - ); - assert_eq!(support.driver_label().as_deref(), Some("Mesa RADV")); - assert!( - support - .detail - .as_deref() - .is_some_and(|detail| detail.contains("no device was enumerated")), - "the gap must be stated, not papered over: {support:?}" - ); - } - - /// No ICD and no enumerated device is an absence of evidence, not evidence - /// of absence — neither source is exhaustive (the Linux ICD search ignores - /// `VK_ICD_FILENAMES`, the Windows one reads only the legacy registry key, - /// and `vulkaninfo` is often not installed). `Absent` renders as a - /// confirmed "no Vulkan GPU driver was found - install one", so a host with - /// a working driver would be told to install one it already has. - #[test] - fn no_evidence_either_way_is_not_reported_as_a_confirmed_absence() { - let nothing = classify_vulkan_support(Vec::new(), Vec::new(), true); - assert_eq!( - nothing.status, - VulkanStatus::Unknown, - "an empty probe must not claim the driver is absent" - ); - assert!( - nothing - .detail - .as_deref() - .is_some_and(|detail| detail.contains("confirms")), - "the detail must say the absence is unconfirmed: {nothing:?}" - ); - assert!(nothing.driver_label().is_none()); - - // What `Absent` is now reserved for: something WAS found and ruled out. - let software_only = - classify_vulkan_support(vec!["lvp_icd.x86_64.json".to_owned()], Vec::new(), true); - assert_eq!(software_only.status, VulkanStatus::Absent); - - // macOS and friends: no evidence either way, which must not read as - // "no Vulkan" any more than it reads as "Vulkan works". - let unprobeable = classify_vulkan_support(Vec::new(), Vec::new(), false); - assert_eq!(unprobeable.status, VulkanStatus::Unknown); - assert!(!unprobeable.status.is_usable()); - } - - #[test] - fn an_unknown_vendors_icd_is_assumed_to_be_hardware() { - // A vendor the table does not list yet must not be mistaken for a - // software rasteriser; the device enumeration corrects it if wrong. - assert!(icd_is_gpu_driver("some_new_vendor_icd.x86_64.json")); - assert_eq!(icd_driver_label("some_new_vendor_icd.x86_64.json"), None); - - let support = classify_vulkan_support( - vec!["some_new_vendor_icd.x86_64.json".to_owned()], - Vec::new(), - true, - ); - assert_eq!(support.status, VulkanStatus::DriverPresentDevicesUnverified); - } - - #[test] - fn a_discrete_amd_card_is_preferred_over_a_software_device_in_the_same_report() { - let mixed = "\ -GPU0: -\tvendorID = 0x10005 -\tdeviceType = PHYSICAL_DEVICE_TYPE_CPU -\tdeviceName = llvmpipe -\tdriverName = llvmpipe -GPU1: -\tvendorID = 0x1002 -\tdeviceType = PHYSICAL_DEVICE_TYPE_DISCRETE_GPU -\tdeviceName = AMD Radeon RX 7900 XTX -\tdriverName = radv -"; - let support = classify_vulkan_support(Vec::new(), parse_vulkaninfo_summary(mixed), true); - - assert_eq!(support.status, VulkanStatus::Available); - assert_eq!(support.driver_label().as_deref(), Some("radv")); - assert!( - support - .detail - .as_deref() - .is_some_and(|detail| detail.contains("AMD Radeon RX 7900 XTX")), - "{support:?}" - ); - } - - #[test] - fn a_non_amd_gpu_does_not_make_vulkan_an_amd_path() { - let nvidia = "\ -GPU0: -\tvendorID = 0x10de -\tdeviceType = PHYSICAL_DEVICE_TYPE_DISCRETE_GPU -\tdeviceName = NVIDIA GeForce GTX 1050 Ti -\tdriverName = NVIDIA -"; - let support = classify_vulkan_support( - vec!["nvidia_icd.json".to_owned()], - parse_vulkaninfo_summary(nvidia), - true, - ); - - // The driver is installed and is not software, so the honest answer is - // "unverified for AMD", not "available". - assert_eq!(support.status, VulkanStatus::DriverPresentDevicesUnverified); - } - - #[test] - fn the_windows_registry_listing_is_read_into_icd_paths() { - let output = "ICD\tC:\\Windows\\System32\\DriverStore\\FileRepository\\u0397342.inf_amd64_x\\amdvlk64.json\nICD\tC:\\Windows\\SysWOW64\\amdvlk32.json\n"; - - let icds = parse_windows_vulkan_icd_listing(output); - - assert_eq!(icds.len(), 2, "{icds:?}"); - assert!(icds[0].ends_with("amdvlk64.json")); - assert_eq!(icd_driver_label(&icds[0]), Some("AMDVLK")); - assert!( - icd_is_gpu_driver(&icds[0]), - "a Windows path separator must not defeat the filename table" - ); - } - - /// Both Khronos keys must be queried. A 32-bit AMDVLK registration lives - /// only under `WOW6432Node`, so reading the 64-bit key alone would report - /// "no Vulkan driver" on a host that has one. - #[test] - fn the_windows_icd_query_reads_both_the_native_and_the_wow6432_key() { - assert!( - WINDOWS_VULKAN_ICD_SCRIPT.contains(r"HKLM:\SOFTWARE\Khronos\Vulkan\Drivers"), - "{WINDOWS_VULKAN_ICD_SCRIPT}" - ); - assert!( - WINDOWS_VULKAN_ICD_SCRIPT - .contains(r"HKLM:\SOFTWARE\WOW6432Node\Khronos\Vulkan\Drivers"), - "{WINDOWS_VULKAN_ICD_SCRIPT}" - ); - // A missing key is ordinary on a host with no Vulkan at all, so the - // script must test for it rather than let PowerShell error out. - assert!( - WINDOWS_VULKAN_ICD_SCRIPT.contains("Test-Path"), - "{WINDOWS_VULKAN_ICD_SCRIPT}" - ); - // The rows the parser consumes. - assert!( - WINDOWS_VULKAN_ICD_SCRIPT.contains("\"ICD`t$name\""), - "{WINDOWS_VULKAN_ICD_SCRIPT}" - ); - } - - #[test] - fn an_empty_registry_key_yields_no_icds_rather_than_a_blank_entry() { - assert!(parse_windows_vulkan_icd_listing("").is_empty()); - assert!(parse_windows_vulkan_icd_listing("ICD\t\n").is_empty()); - assert!(parse_windows_vulkan_icd_listing("unrelated output\n").is_empty()); - } - - #[test] - fn a_vulkaninfo_report_with_no_devices_yields_none() { - assert!(parse_vulkaninfo_summary("Vulkan Instance Version: 1.3.301\n").is_empty()); - assert!(parse_vulkaninfo_summary("").is_empty()); - } -} diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index aa33884fee..c5ea6662a8 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -139,12 +139,12 @@ pub use gfx_target::{ }; pub use hardware_root::host_path; pub use host_gpu::{ - DriverSummary, ExamineSummary, HostGpuSummary, VulkanDevice, VulkanStatus, VulkanSupport, - WSL_ROCDXG_READY, WindowsGpuListing, WslHostDriverProbe, WslSummary, default_engine_for_host, - default_engine_for_platform, detect_gpu_driver_version, detect_host_gfx_target, - detect_host_gpu_diagnostics, detect_host_gpu_summary, detect_system_ram_gib, - detect_vulkan_support, has_usable_amd_gpu, is_amdgpu_device, is_wsl_host, pinned_gpu_name, - preferred_serve_engine_for_host_gpu_summary, usable_amd_gpu_indices, windows_amd_gpu_listing, + DriverSummary, ExamineSummary, HostGpuSummary, WSL_ROCDXG_READY, WindowsGpuListing, + WslHostDriverProbe, WslSummary, default_engine_for_host, default_engine_for_platform, + detect_gpu_driver_version, detect_host_gfx_target, detect_host_gpu_diagnostics, + detect_host_gpu_summary, detect_system_ram_gib, has_usable_amd_gpu, is_amdgpu_device, + is_wsl_host, pinned_gpu_name, preferred_serve_engine_for_host_gpu_summary, + usable_amd_gpu_indices, windows_amd_gpu_listing, }; #[cfg(any(target_os = "linux", test))] use host_gpu::{KfdGpuNode, kfd_gpu_nodes_in}; diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 2dd776d15b..67511ad01d 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -108,15 +108,6 @@ pub struct EngineStdioOptions<'a> { /// its HTTP probe of a protected endpoint authenticates. The caller decides /// whether a file holds a usable key; this only carries it. pub endpoint_key_file: Option<&'a Path>, - /// With no `endpoint_key_file`, also CLEAR [`ENDPOINT_API_KEY_FILE_ENV`] - /// rather than leave whatever this process inherited to pass through. - /// - /// For a child that must not see a credential path at all — `rocm`'s - /// external engine plugins are third-party binaries. Off by default, - /// because "no key file for this request" and "this child may not have one" - /// are different statements: `rocmd` passes `None` for a loopback service - /// that simply needs no key, and its children are its own. - pub withhold_inherited_key_file: bool, /// Ask the engine for progress on stderr ([`ENGINE_PROGRESS_STDERR_ENV`]). pub request_progress: bool, /// Write the child's stderr to this process's stderr as it arrives, so a @@ -164,12 +155,13 @@ where } if let Some(key_file) = options.endpoint_key_file { command.env(ENDPOINT_API_KEY_FILE_ENV, key_file); - } else if options.withhold_inherited_key_file { - // Cleared, not merely left unset: a value this process inherited must - // not reach a third-party plugin binary just because nothing - // overwrote it. Only for callers that ask — `rocmd` passes `None` for - // a loopback service that needs no key, and clearing there would - // change what its own children see. See + } else { + // Cleared, not merely left unset. The caller said this child gets no + // endpoint key, and an engine plugin is a third-party binary: a value + // this process happens to have inherited must not reach it just + // because nothing overwrote it. `rocm`'s plugin calls pass `None` + // here, so this is what keeps a 0600 key-file path off their + // environment. See // `an_external_plugin_child_is_not_handed_the_endpoint_key_file`. command.env_remove(ENDPOINT_API_KEY_FILE_ENV); } @@ -714,7 +706,6 @@ mod tests { app_paths: None, env_root: None, endpoint_key_file: None, - withhold_inherited_key_file: true, request_progress: false, forward_stderr: false, note_failing_exit_after_answer: false, @@ -731,27 +722,6 @@ mod tests { "the key-file variable must be explicitly CLEARED when no key file \ is passed, not merely left unset" ); - - // The other direction, which is what keeps this scoped: a caller that - // did not ask (`rocmd`, for a loopback service that simply needs no - // key) must leave the child's inherited value alone. - let kept = engine_stdio_command( - Path::new("/an/engine"), - ["stdio"], - &EngineStdioOptions { - app_paths: None, - env_root: None, - endpoint_key_file: None, - withhold_inherited_key_file: false, - request_progress: false, - forward_stderr: false, - note_failing_exit_after_answer: false, - }, - ); - assert!( - kept.get_envs().all(|(key, _)| key != key_var), - "a caller that did not ask to withhold must not touch the variable" - ); } #[test] @@ -768,7 +738,6 @@ mod tests { app_paths: Some(&paths), env_root: Some(Path::new("/envs")), endpoint_key_file: Some(Path::new("/keys/svc.endpoint-key")), - withhold_inherited_key_file: false, request_progress: true, forward_stderr: true, note_failing_exit_after_answer: false, @@ -802,7 +771,6 @@ mod tests { "ROCM_SERVE_API_KEY_FILE", EngineStdioOptions { endpoint_key_file: Some(Path::new("/keys/k")), - withhold_inherited_key_file: false, ..EngineStdioOptions::default() }, ), @@ -1224,7 +1192,6 @@ mod tests { app_paths: Some(&paths), env_root: Some(Path::new("/envs")), endpoint_key_file: Some(Path::new("/keys/k")), - withhold_inherited_key_file: false, request_progress: true, forward_stderr: false, note_failing_exit_after_answer: false, diff --git a/docs/architecture.md b/docs/architecture.md index e33121035f..7270fd1b87 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -203,8 +203,8 @@ Already-extracted subsystem modules, following the same full-domain-extraction p - [`gguf.rs`](../crates/rocm-core/src/gguf.rs) — reads the shape of a GGUF model (layers, KV heads, head width, parameter count, data offset) from its header over any `Read + Seek`, never touching the weights. - [`gguf_fit.rs`](../crates/rocm-core/src/gguf_fit.rs) — pure arithmetic over that header: weights + KV cache at a context size and KV type + compute buffer + runtime reserve against dedicated VRAM, the largest context that fits, and what would make a model that does not fit, fit. - [`gguf_source.rs`](../crates/rocm-core/src/gguf_source.rs) — where a header comes from: a local file, or a Hugging Face file, found by one repository-listing request (and a `HEAD` request for a file the listing does not size) and read by HTTP range requests (`HF_ENDPOINT` selects a mirror; redirects are followed by hand, up to five, and never from https to plain http). The only code in the estimate that reads a file or the network. -- [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) — (ROCMAI-85, Phase 7a of ROCMAI-27's sequencing) GPU/host hardware detection — see the module's own doc comment for what it owns. Its leaf concerns are child modules under `host_gpu/`, with `host_gpu.rs` kept as the module root: [`capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs) (running an optional tool under a timeout, trying each spelling [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs) gives for it), [`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) (kernel version, distro name, installed RAM, and the CPU model and vendor, which one reader returns together for both the summary and `Examination::probe`: Linux from `/proc/cpuinfo`, Windows from the CPU row of the system inventory), [`linux.rs`](../crates/rocm-core/src/host_gpu/linux.rs) (the sysfs, KFD and DRM probes), [`gpu_visibility.rs`](../crates/rocm-core/src/host_gpu/gpu_visibility.rs) (which AMD GPUs this process may use, from the KFD/DRM counts and the `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` masks, plus `windows_amd_gpu_listing`, the native-Windows adapter bound `rocm serve --gpu ` is checked against), [`windows_inventory.rs`](../crates/rocm-core/src/host_gpu/windows_inventory.rs) (the Windows display-adapter inventory and its parsers, including the AMD-adapter count behind `windows_amd_gpu_listing` and the system query whose CPU row the CPU reader in `os_facts.rs` takes), [`wsl.rs`](../crates/rocm-core/src/host_gpu/wsl.rs) (WSL detection, the librocdxg chain and the Windows host driver as seen from the guest), [`render.rs`](../crates/rocm-core/src/host_gpu/render.rs) (the `rocm examine` text of an `ExamineSummary`, as a second `impl` block), [`vulkan.rs`](../crates/rocm-core/src/host_gpu/vulkan.rs) (whether a model can run through Vulkan here, read from ICD manifests, the Windows Khronos registry and `vulkaninfo --summary` — the evidence `gpu_support_level` in [`gfx_target.rs`](../crates/rocm-core/src/gfx_target.rs) needs to tell a GPU with no ROCm build that still has a Vulkan driver from one with no GPU path at all, and where a software rasteriser is recognised in order to be excluded), [`diagnostics.rs`](../crates/rocm-core/src/host_gpu/diagnostics.rs) (the GPU detection diagnostics text) and [`engine_default.rs`](../crates/rocm-core/src/host_gpu/engine_default.rs) (the default serve engine, and the vLLM preference for the TheRock families that serve through it). -- [`host-probing.md`](host-probing.md) — a design note, not a module: [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) and [`examine.rs`](../crates/rocm-core/src/examine.rs) are two separate host-probing stacks that read some of the same facts twice (the Windows GPU query, the marketing-name table and the subprocess runners). It proposes which of each pair should survive, how the two differ and the order in which they could converge, and records what each convergence decided; it changes no code itself. Three pairs are converged: the `PATH` lookup ([`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs)'s `which` is now the only copy) and the kernel release ([`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs)'s `detect_kernel_version` is now the only `uname -r` reader, which [`host_env.rs`](../crates/rocm-core/src/examine/host_env.rs)'s `probe_os` calls for `kernel_release`, and `rocm examine --json` runs it once: [`examine.rs`](../crates/rocm-core/src/examine.rs)'s `Examination::probe_with_summary` probes the host and hands the examination's kernel release to the summary), and the CPU model ([`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) has one reader per platform for the model and vendor, which `probe_cpu_linux` and `probe_cpu_windows` record and the summary reads). The first step of the subprocess runners is delivered too: [`process/bounded.rs`](../crates/rocm-core/src/process/bounded.rs)'s `run_bounded` is the executor behind `run`, `capture_optional_command*` and the Windows diagnostic capture. It is a snapshot, to be retired when the convergence tickets close. +- [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) — (ROCMAI-85, Phase 7a of ROCMAI-27's sequencing) GPU/host hardware detection — see the module's own doc comment for what it owns. Its leaf concerns are child modules under `host_gpu/`, with `host_gpu.rs` kept as the module root: [`capture.rs`](../crates/rocm-core/src/host_gpu/capture.rs) (running an optional tool under a timeout, trying each spelling [`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs) gives for it), [`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) (kernel version, distro name, installed RAM, and the CPU model and vendor, which one reader returns together for both the summary and `Examination::probe`: Linux from `/proc/cpuinfo`, Windows from the CPU row of the system inventory), [`linux.rs`](../crates/rocm-core/src/host_gpu/linux.rs) (the sysfs, KFD and DRM probes), [`gpu_visibility.rs`](../crates/rocm-core/src/host_gpu/gpu_visibility.rs) (which AMD GPUs this process may use, from the KFD/DRM counts and the `HIP_VISIBLE_DEVICES`/`ROCR_VISIBLE_DEVICES` masks, plus `windows_amd_gpu_listing`, the native-Windows adapter bound `rocm serve --gpu ` is checked against), [`windows_inventory.rs`](../crates/rocm-core/src/host_gpu/windows_inventory.rs) (the Windows display-adapter inventory and its parsers, including the AMD-adapter count behind `windows_amd_gpu_listing` and the system query whose CPU row the CPU reader in `os_facts.rs` takes), [`wsl.rs`](../crates/rocm-core/src/host_gpu/wsl.rs) (WSL detection, the librocdxg chain and the Windows host driver as seen from the guest), [`render.rs`](../crates/rocm-core/src/host_gpu/render.rs) (the `rocm examine` text of an `ExamineSummary`, as a second `impl` block), [`diagnostics.rs`](../crates/rocm-core/src/host_gpu/diagnostics.rs) (the GPU detection diagnostics text) and [`engine_default.rs`](../crates/rocm-core/src/host_gpu/engine_default.rs) (the default serve engine, and the vLLM preference for the TheRock families that serve through it). +- [`host-probing.md`](host-probing.md) — a design note, not a module: [`host_gpu.rs`](../crates/rocm-core/src/host_gpu.rs) and [`examine.rs`](../crates/rocm-core/src/examine.rs) are two separate host-probing stacks that read some of the same facts twice (the Windows GPU query, the marketing-name table and the subprocess runners). It proposes which of each pair should survive, how the two differ and the order in which they could converge, and records what each convergence decided; it changes no code itself. Three pairs are converged: the `PATH` lookup ([`probe_exec.rs`](../crates/rocm-core/src/probe_exec.rs)'s `which` is now the only copy) and the kernel release ([`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs)'s `detect_kernel_version` is now the only `uname -r` reader, which [`host_env.rs`](../crates/rocm-core/src/examine/host_env.rs)'s `probe_os` calls for `kernel_release`, and `rocm examine --json` runs it once: [`examine.rs`](../crates/rocm-core/src/examine.rs)'s `Examination::probe_with_summary` probes the host and hands the examination's kernel release to the summary), and the CPU model ([`os_facts.rs`](../crates/rocm-core/src/host_gpu/os_facts.rs) has one reader per platform for the model and vendor, which `probe_cpu_linux` and `probe_cpu_windows` record and the summary reads). It is a snapshot, to be retired when the convergence tickets close. - [`jsonl_store.rs`](../crates/rocm-core/src/jsonl_store.rs) — the JSONL store engine behind the automation stores: the sibling `.lock` file (`jsonl_lock_path`, re-exported at the crate root), single-line appends, torn-tail repair, size-based rotation (`JsonlRotation`) and the tail-first readers that span rotated generations. It is generic over the record type; [`automation.rs`](../crates/rocm-core/src/automation.rs) supplies the record types and each store's policy, and nothing in this module depends on it, in production or in its own tests: those drive the engine with a small record type of their own. The module is private; its engine items are `pub(crate)`. - [`managed_runtime.rs`](../crates/rocm-core/src/managed_runtime.rs) — (ROCMAI-85, Phase 7a) the TheRock managed-runtime-environment surface — see the module's own doc comment for what it owns. - [`model_catalog.rs`](../crates/rocm-core/src/model_catalog.rs) — (ROCMAI-152, Phase 7c) the curated model catalog's types, built-in catalog JSON loading, signed external recipe indexes, and artifact cache-marker bookkeeping — see the module's own doc comment for what it owns. diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index e64ae34929..11e49f5f4d 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -752,40 +752,3 @@ Feature: GPU detection and system inspection Then the inspection states this host's ROCm support level And the support level agrees with the GPU the inspection detected And the support level leaves a command that fits this host - - # The three below close the gap this PR's own notes call unclosable: the - # `VulkanOnly` door needs a Renoir/Cezanne Vega APU, and no runner has one. - # It does not need the hardware, only the two signals the CLI reads for it -- - # a KFD node naming the target, and the host's Vulkan driver state -- both of - # which the harness can plant. So these run on the ungated mock lane, which is - # the lane that has neither. - # - # They exist because the support level used to be inferred: "no ROCm build" - # was taken to mean "Vulkan then", on a machine nobody had looked at. Each - # scenario pins one of the three answers that inference could not tell apart. - @id:examine-names-the-vulkan-driver-an-unsupported-gpu-can-use @requires-os:linux - Scenario: examine-50 - A GPU the official stack rejects is told which way it can still run models - Given a machine whose GPU has no official support but has a working Vulkan driver - When the user inspects that machine - Then the inspection reports the GPU as unsupported by the official stack - And the inspection reports that models can still run through Vulkan - And the inspection names the Vulkan driver it found - - @id:examine-does-not-promise-vulkan-without-a-driver @requires-os:linux - Scenario: examine-51 - A machine with no way to run models is not told it has one - Given a machine whose GPU has no official support and no Vulkan driver - When the user inspects that machine - Then the inspection reports the GPU as unsupported by the official stack - And the inspection does not claim models can run through Vulkan - And the inspection says no Vulkan driver was found - - # A software renderer is a complete Vulkan implementation that runs on the - # CPU. Counting it would put a GPU-required run on the processor while - # reporting a GPU path - the one outcome the no-fallback rule exists to - # prevent, reached through a backend rather than a policy change. - @id:examine-does-not-count-a-software-renderer-as-a-gpu-path @requires-os:linux - Scenario: examine-52 - A software renderer does not count as a way to run on the GPU - Given a machine whose GPU has no official support and only a software renderer - When the user inspects that machine - Then the inspection does not claim models can run through Vulkan - And the inspection says the only renderer it found runs on the CPU diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index a5bc2b38c3..8fbe54b3cf 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -1704,25 +1704,6 @@ async fn assert_no_amdgpu_fault_evidence(world: &mut E2eWorld) { } } -/// The human-readable twin of the machine-readable planted-machine step, for -/// scenarios whose subject is a sentence the report prints rather than a field -/// it serialises. -/// -/// It has to be its own step rather than reusing "the user inspects the -/// system", because that one runs through `run_rocm`, which does not consume -/// `World::command_env` -- so a planted host root never reaches the child and -/// the CLI silently describes the *runner* instead. Not hypothetical: it is -/// what made the first draft of examine-50/51/52 report the lane's own MI300X -/// as `gfx942` while claiming to be about a Renoir APU. -#[when("the user inspects that machine")] -async fn user_inspects_planted_machine_text(world: &mut E2eWorld) { - let (stdout, stderr, rc) = - crate::run_rocm_with_scenario_env(world, &["examine", "--framework", "skip"]); - world.cli_output = Some(stdout); - world.cli_stderr = Some(stderr); - world.cli_rc = Some(rc); -} - #[when("the user inspects that machine in machine-readable form")] async fn user_inspects_planted_machine(world: &mut E2eWorld) { let (stdout, stderr, rc) = @@ -3366,208 +3347,3 @@ async fn assert_support_level_advice_fits(world: &mut E2eWorld) { ); } } - -/// A KFD `gfx_target_version` naming gfx90c, the Vega iGPU in Renoir / -/// Lucienne / Cezanne / Barcelo APUs. -/// -/// The kernel packs the target as `major * 10000 + minor * 100 + step`, so -/// `9.0.12` is `90012` and decodes to `gfx90c` (step 12 is the hex `c`). This -/// is the one target the CLI recognizes *and* refuses to map to a ROCm package -/// family, so it is what reaches the support level these scenarios are about. -const KFD_GFX90C_TARGET_VERSION: &str = "90012"; - -/// `vulkaninfo --summary` on a host whose Vega iGPU is driven by Mesa RADV. -const RADV_VULKANINFO_SUMMARY: &str = "\ -Devices: -======== -GPU0: -\tapiVersion = 1.3.274 -\tvendorID = 0x1002 -\tdeviceID = 0x1636 -\tdeviceType = PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU -\tdeviceName = AMD Radeon Graphics (RADV RENOIR) -\tdriverID = DRIVER_ID_MESA_RADV -\tdriverName = radv -"; - -/// `vulkaninfo --summary` on a host with only Mesa's CPU rasteriser: a complete -/// Vulkan implementation, and not a GPU. -const LAVAPIPE_VULKANINFO_SUMMARY: &str = "\ -Devices: -======== -GPU0: -\tapiVersion = 1.3.255 -\tvendorID = 0x10005 -\tdeviceType = PHYSICAL_DEVICE_TYPE_CPU -\tdeviceName = llvmpipe (LLVM 15.0.7, 256 bits) -\tdriverName = llvmpipe -"; - -/// Plant a host whose GPU is identified but has no ROCm package family, -/// carrying the Vulkan state the caller describes. -/// -/// `icd_manifest` is the ICD file name the loader would find and -/// `vulkaninfo_summary` what `vulkaninfo --summary` prints; `None` for either -/// means that half of the evidence is absent. -/// -/// The `vulkaninfo` stub is planted in **both** directions. When a scenario -/// wants the tool absent it is planted as a failing command rather than left -/// out, because `PATH` still reaches the rest of the machine: a runner with a -/// real `vulkaninfo` installed would otherwise answer for it, and a scenario -/// about a host with no Vulkan driver has to be hermetic about the host and not -/// only about the planted root. -fn plant_unsupported_gpu( - world: &mut E2eWorld, - icd_manifest: Option<&str>, - vulkaninfo_summary: Option<&str>, -) { - let root = world - .isolated_root - .as_ref() - .expect("no isolated root") - .path() - .to_owned(); - let host_root = root.join("host-root"); - - let node = host_root.join("sys/class/kfd/kfd/topology/nodes/1"); - std::fs::create_dir_all(&node).expect("failed to create planted KFD node"); - std::fs::write( - node.join("properties"), - format!( - "cpu_cores_count 0\nsimd_count 56\ngfx_target_version {KFD_GFX90C_TARGET_VERSION}\n" - ), - ) - .expect("failed to write planted KFD node properties"); - - if let Some(manifest) = icd_manifest { - let icd_dir = host_root.join("usr/share/vulkan/icd.d"); - std::fs::create_dir_all(&icd_dir).expect("failed to create planted ICD dir"); - std::fs::write( - icd_dir.join(manifest), - "{\"file_format_version\":\"1.0.0\",\"ICD\":{\"library_path\":\"libvulkan_fake.so\"}}\n", - ) - .expect("failed to write planted ICD manifest"); - } - - let bin = root.join("fake-bin"); - std::fs::create_dir_all(&bin).expect("failed to create the fake PATH dir"); - // The same empty-bus `lspci` the other GPU fixtures plant: the probe must - // succeed without describing the runner's real cards. - rocm_core::fake_executable::write_fake_executable(&bin.join("lspci"), "#!/bin/sh\nexit 0\n") - .expect("failed to write the fake lspci"); - let vulkaninfo = match vulkaninfo_summary { - Some(summary) => format!("#!/bin/sh\ncat <<'ROCM_E2E_EOF'\n{summary}ROCM_E2E_EOF\n"), - None => "#!/bin/sh\nexit 1\n".to_owned(), - }; - rocm_core::fake_executable::write_fake_executable(&bin.join("vulkaninfo"), &vulkaninfo) - .expect("failed to write the fake vulkaninfo"); - - let path = std::env::join_paths(std::iter::once(bin).chain(std::env::split_paths( - &std::env::var_os("PATH").unwrap_or_default(), - ))) - .expect("PATH entries cannot hold a separator"); - world.command_env.push(("PATH", path)); - world - .command_env - .push(("ROCM_CLI_TEST_HOST_ROOT", host_root.into_os_string())); -} - -#[given("a machine whose GPU has no official support but has a working Vulkan driver")] -async fn unsupported_gpu_with_vulkan(world: &mut E2eWorld) { - plant_unsupported_gpu( - world, - Some("radeon_icd.x86_64.json"), - Some(RADV_VULKANINFO_SUMMARY), - ); -} - -#[given("a machine whose GPU has no official support and no Vulkan driver")] -async fn unsupported_gpu_without_vulkan(world: &mut E2eWorld) { - plant_unsupported_gpu(world, None, None); -} - -#[given("a machine whose GPU has no official support and only a software renderer")] -async fn unsupported_gpu_with_software_renderer(world: &mut E2eWorld) { - plant_unsupported_gpu( - world, - Some("lvp_icd.x86_64.json"), - Some(LAVAPIPE_VULKANINFO_SUMMARY), - ); -} - -#[then("the inspection reports the GPU as unsupported by the official stack")] -async fn assert_gpu_unsupported_by_rocm(world: &mut E2eWorld) { - let output = world.cli_output.as_ref().expect("no command was run"); - assert!( - output.contains("rocm_support:") && output.contains("not supported by ROCm"), - "the inspection must say the GPU was identified and has no ROCm build:\n{output}" - ); -} - -#[then("the inspection reports that models can still run through Vulkan")] -async fn assert_vulkan_offered(world: &mut E2eWorld) { - let output = world.cli_output.as_ref().expect("no command was run"); - assert!( - output.contains("the Vulkan backend can be used"), - "a host with a working Vulkan device must be told so:\n{output}" - ); - assert!( - output.contains("vulkan: available"), - "the machine-readable line must agree with the sentence:\n{output}" - ); -} - -#[then("the inspection names the Vulkan driver it found")] -async fn assert_vulkan_driver_named(world: &mut E2eWorld) { - let output = world.cli_output.as_ref().expect("no command was run"); - // The driver the planted device reported, not a guess from the ICD file - // name: naming it is what lets a reader check the claim. - assert!( - output.contains("radv"), - "the inspection must name the driver it found:\n{output}" - ); -} - -#[then("the inspection does not claim models can run through Vulkan")] -async fn assert_vulkan_not_promised(world: &mut E2eWorld) { - let output = world.cli_output.as_ref().expect("no command was run"); - assert!( - !output.contains("the Vulkan backend can be used"), - "a host with no usable Vulkan GPU must not be promised one:\n{output}" - ); - assert!( - !output.contains("Vulkan backend is expected to work"), - "a host with no usable Vulkan GPU must not be promised one:\n{output}" - ); -} - -#[then("the inspection says no Vulkan driver was found")] -async fn assert_no_vulkan_driver_reported(world: &mut E2eWorld) { - let output = world.cli_output.as_ref().expect("no command was run"); - assert!( - output.contains("no Vulkan GPU driver was found"), - "the inspection must say why no GPU path is open:\n{output}" - ); - assert!( - output.contains("install one"), - "the advice must be able to clear the condition it is printed for:\n{output}" - ); -} - -#[then("the inspection says the only renderer it found runs on the CPU")] -async fn assert_software_renderer_reported(world: &mut E2eWorld) { - let output = world.cli_output.as_ref().expect("no command was run"); - assert!( - output.contains("not a GPU path"), - "a CPU rasteriser must be named as such, not counted as a GPU:\n{output}" - ); - // The support level must agree, not just the evidence line. Without this the - // scenario still passed when `gpu_support_level` ignored the probe entirely: - // the absence of a Vulkan promise was coming from the status wording alone, - // so nothing here held the classification itself. - assert!( - output.contains("no Vulkan GPU driver was found"), - "a host whose only renderer is the CPU has no GPU backend, and the support \ - line must say so:\n{output}" - ); -} From c03972e405aed30716bc093c935588896e9ceb44 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 17:06:12 +0200 Subject: [PATCH 20/37] fix(examine): finish removing the split-out work from this PR MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three leftovers from the split, all of which made this diff claim work it no longer ships. `stdio_client.rs` still carried an older, UNCONDITIONAL version of the endpoint key-file clearing — the one that broke `rocmd`'s contract before #1034 made it opt-in. Reset to main's. The guard lives in #1034 alone now, and `rocmd`'s own `the_endpoint_key_file_reaches_the_child_only_when_it_holds_a_usable_key` is green again here. `docs/testing.md` still described four support levels and pointed at examine-50/51/52. This change ships three levels and examine-49; the fourth level and those scenarios are #1036's, and the text now says so. The `NoGpuBackend` arm in `assert_support_level_advice_fits` was dead here — nothing can produce that level without the probe — so it goes back to #1036 with the level it tests. Signed-off-by: Roman Sirokov --- .../rocm-engine-protocol/src/stdio_client.rs | 49 ------------------- docs/testing.md | 24 ++++----- tests/e2e-cucumber/tests/e2e/examine_steps.rs | 9 ---- 3 files changed, 12 insertions(+), 70 deletions(-) diff --git a/crates/rocm-engine-protocol/src/stdio_client.rs b/crates/rocm-engine-protocol/src/stdio_client.rs index 67511ad01d..5a8989e5a7 100644 --- a/crates/rocm-engine-protocol/src/stdio_client.rs +++ b/crates/rocm-engine-protocol/src/stdio_client.rs @@ -155,15 +155,6 @@ where } if let Some(key_file) = options.endpoint_key_file { command.env(ENDPOINT_API_KEY_FILE_ENV, key_file); - } else { - // Cleared, not merely left unset. The caller said this child gets no - // endpoint key, and an engine plugin is a third-party binary: a value - // this process happens to have inherited must not reach it just - // because nothing overwrote it. `rocm`'s plugin calls pass `None` - // here, so this is what keeps a 0600 key-file path off their - // environment. See - // `an_external_plugin_child_is_not_handed_the_endpoint_key_file`. - command.env_remove(ENDPOINT_API_KEY_FILE_ENV); } if options.request_progress { command.env(ENGINE_PROGRESS_STDERR_ENV, "1"); @@ -684,46 +675,6 @@ mod tests { assert!(added_env(&command).is_empty(), "{:?}", added_env(&command)); } - /// A child told it gets no endpoint key must have the variable CLEARED, - /// not merely left unset. - /// - /// `rocm` passes `endpoint_key_file: None` for every external engine - /// plugin — a third-party binary — and the path it would otherwise carry - /// points at a mode-0600 file holding a live endpoint credential. Merely - /// not setting it would still hand the value through whenever this process - /// inherited one. - /// - /// Asserts the distinction directly: `Command` records `env_remove` as an - /// explicit `(key, None)` entry, so a plain "is it absent" check would pass - /// even with the clearing deleted. - #[test] - fn an_external_plugin_child_is_not_handed_the_endpoint_key_file() { - let key_var = std::ffi::OsStr::new(ENDPOINT_API_KEY_FILE_ENV); - let command = engine_stdio_command( - Path::new("/an/engine"), - ["stdio"], - &EngineStdioOptions { - app_paths: None, - env_root: None, - endpoint_key_file: None, - request_progress: false, - forward_stderr: false, - note_failing_exit_after_answer: false, - }, - ); - - let entry = command - .get_envs() - .find(|(key, _)| *key == key_var) - .map(|(_, value)| value); - assert_eq!( - entry, - Some(None), - "the key-file variable must be explicitly CLEARED when no key file \ - is passed, not merely left unset" - ); - } - #[test] fn each_option_sets_exactly_its_own_variable() { let paths = AppPaths { diff --git a/docs/testing.md b/docs/testing.md index 642d5e5c6c..92f0ca370d 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -3892,22 +3892,22 @@ row is the backend and ordinal with no name — correct and unspecific, rather than specific and wrong. No CI lane has two AMD GPUs, so that branch is covered by unit tests in [host_gpu.rs](../crates/rocm-core/src/host_gpu.rs) only. -**`rocm examine` prints a `rocm_support` line on every host**, at one of four -levels (`gpu_support_level`): ROCm/HIP-supported, Vulkan-only, no GPU backend, -or unknown. Unknown covers both "nothing detected" and "a target this build's -catalog does not recognise" — conflating the latter with "confirmed to have no -ROCm build" would turn a detection gap into a claim about the hardware, which -is why it is distinct from the `NoGpuBackend` level that *is* a confirmed -answer. `examine-49` holds the line against the `detected_gfx_target` and -`compatible_therock_family` the same report prints, so every lane checks the -level its own hardware implies; `examine-50`, `examine-51` and `examine-52` -cover the Vulkan tiers (a driver present, none present, and a software -renderer that must not count as a GPU path) and are `@requires-os:linux`: +**`rocm examine` prints a `rocm_support` line on every host**, at one of three +levels (`gpu_support_level`): ROCm/HIP-supported, Vulkan-only, or unknown. +Unknown covers both "nothing detected" and "a target this build's catalog does +not recognise" — conflating the latter with "confirmed to have no ROCm build" +would turn a detection gap into a claim about the hardware. `examine-49` holds +the line against the `detected_gfx_target` and `compatible_therock_family` the +same report prints, so every lane checks the level its own hardware implies: ```bash -cargo xtask e2e -- -n '(examine-49|examine-5[012])' +cargo xtask e2e -- -n '(examine-49)' ``` +A fourth level, `NoGpuBackend`, and the scenarios covering the Vulkan tiers +live in the Vulkan-probe change (#1036), which is what makes that level +knowable. + ## Windows Tool Notes The TheRock SDK wheel install path should not require users to install global diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 8fbe54b3cf..06a170a267 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -3312,15 +3312,6 @@ async fn assert_support_level_advice_fits(world: &mut E2eWorld) { (_, Some(_), Some(_)) => ("rocm serve ", &["rocm install sdk"]), // ROCm-capable with no managed SDK yet: that install is the next step. (_, Some(_), None) => ("rocm install sdk", &[]), - // A GPU no package family covers, and the Vulkan probe found no GPU - // driver either: there is no backend to serve on, so the line owes a - // way to GET one rather than a serve command that would be refused. - // Branched on the rendered tier the way the renderer itself is — a - // flat expectation here failed on exactly this hardware, since this - // scenario is untagged and runs on every lane. - (Some(_), None, _) if line.contains("no Vulkan GPU driver was found") => { - ("install one", &["rocm serve", "rocm install sdk"]) - } // A GPU no package family covers — confirmed Vulkan-only (gfx90c) or // merely unrecognized. Either way the SDK has nothing for it and // Lemonade's Vulkan backend is what can serve on it. From 545da1f52aacd7bfa397ba4424abf0a2bccf0b54 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 17:35:40 +0200 Subject: [PATCH 21/37] docs(examine): say the support line is printed on every host MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `rocm examine`'s output changed on every host — the `rocm_support` line used to appear only for a recognised GPU with no ROCm build, and now appears always, at one of three levels. None of the three surfaces a user reads said so: the `examine` `--help` text, README's "GPUs with no ROCm build", and the examine steps in docs/manual-testing.md all still described the gfx90c case alone. Each now says the same thing: three levels, one line on every host, and that `unknown` covers both an undetected GPU and a target this build's catalog does not recognise — detection gaps rather than a verdict about the hardware. The manual-testing step also now tells the tester the line must be present on a supported card and on a machine with no GPU, which is what makes its absence a regression they can see. Also corrects a comment citing serve-53, which is main's Anthropic tool-use scenario; this change's is serve-55. Signed-off-by: Roman Sirokov --- README.md | 6 ++++++ apps/rocm/src/cli.rs | 13 ++++++++----- apps/rocm/src/therock/family.rs | 2 +- docs/manual-testing.md | 5 ++++- 4 files changed, 19 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index bb6ec82e45..29a73f1bd2 100644 --- a/README.md +++ b/README.md @@ -509,6 +509,12 @@ family is substituted for it. `rocm examine` reports the same conclusion on a These GPUs still work with the Vulkan backend, which needs no ROCm SDK. +`rocm examine` prints that `rocm_support` line on **every** host, not only this +case, at one of three levels: ROCm/HIP-supported, Vulkan-only (the case above), +or unknown. Unknown covers both a GPU nothing detected and a target this build's +catalog does not recognise — detection gaps rather than a verdict about the +hardware, so neither is reported as "no ROCm build". + #### Compiler toolchain (--devel) Pass `--devel` to also install the compiler and headers needed to build GPU diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index 459d6d47d0..507146bfca 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -63,11 +63,14 @@ pub(crate) struct Cli { pub(crate) enum Command { /// Check this computer's GPU, ROCm install, engines, and setup folders. /// - /// A GPU that is recognized but has no ROCm build at all - `gfx90c`, the - /// Vega iGPU in Renoir, Lucienne, Cezanne and Barcelo APUs - is reported on - /// a `rocm_support` line saying so, so that an empty - /// `compatible_therock_family` does not read as a failed probe. Such a GPU - /// still works through the Vulkan backend. + /// A `rocm_support` line is printed on **every** host, at one of three + /// levels: ROCm/HIP-supported, Vulkan-only, or unknown. Vulkan-only covers + /// a GPU that is recognized but has no ROCm build at all — `gfx90c`, the + /// Vega iGPU in Renoir, Lucienne, Cezanne and Barcelo APUs — which still + /// works through Lemonade's Vulkan backend. Unknown covers both a GPU + /// nothing detected and a target this build's catalog does not recognise: + /// those are detection gaps, not a verdict on the hardware, which is why + /// an empty `compatible_therock_family` does not read as a failed probe. Examine { /// Emit the machine-readable Examination JSON (for diagnosis tooling). #[arg(long)] diff --git a/apps/rocm/src/therock/family.rs b/apps/rocm/src/therock/family.rs index c9362ac406..31b836fcbc 100644 --- a/apps/rocm/src/therock/family.rs +++ b/apps/rocm/src/therock/family.rs @@ -197,7 +197,7 @@ pub(super) fn resolve_family( /// reachable only on a host with at least one such card, and the pre-flight /// therefore sees ordinal 0. `rocm_core`'s /// `a_gfx90c_host_the_kernel_gives_no_compute_node_still_has_a_usable_gpu` -/// drives both halves off one planted `/sys`, and serve-53 runs the real binary +/// drives both halves off one planted `/sys`, and serve-55 runs the real binary /// against that same host shape on a GPU-less lane. fn unsupported_host_gpu_message(raw_arch: &str) -> String { format!( diff --git a/docs/manual-testing.md b/docs/manual-testing.md index c0d4c14c1e..1aac94ca85 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -290,7 +290,10 @@ checks, which cannot be made anywhere else: `compatible_therock_family`, plus a `rocm_support` line saying the GPU is detected but not supported by ROCm and that the Vulkan backend can still be used. A `` target, or a missing `rocm_support` line, means detection - regressed. + regressed. That line is printed on **every** host, at one of three levels — + ROCm/HIP-supported, Vulkan-only (this case), or unknown — so on a supported + card or a machine with no GPU it must still be there, saying the level that + host's hardware implies rather than being absent. - A plain `rocm install sdk`, with no `--family`, gives the same refusal with the wording for a *detected* GPU rather than a requested one. - `rocm diagnose` reports `info-20-gpu-no-rocm-support`, naming `gfx90c` and From db00642a725d3d6ac705165bc8e09b198fcd91ea Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 18:07:25 +0200 Subject: [PATCH 22/37] fix(cli): keep the new examine help text ASCII-only The `examine` long help rewritten in bf2a323eb used em-dashes, which `every_help_text_in_the_command_tree_is_ascii` rejects: non-ASCII punctuation renders as mojibake on a Windows console using the OEM code page (ROCMAI-543). Parenthesize the aside instead, and spell `recognize` the same way as the `recognized` three lines above it. Signed-off-by: Roman Sirokov --- apps/rocm/src/cli.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index 507146bfca..9707913c5e 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -65,10 +65,10 @@ pub(crate) enum Command { /// /// A `rocm_support` line is printed on **every** host, at one of three /// levels: ROCm/HIP-supported, Vulkan-only, or unknown. Vulkan-only covers - /// a GPU that is recognized but has no ROCm build at all — `gfx90c`, the - /// Vega iGPU in Renoir, Lucienne, Cezanne and Barcelo APUs — which still + /// a GPU that is recognized but has no ROCm build at all (`gfx90c`, the + /// Vega iGPU in Renoir, Lucienne, Cezanne and Barcelo APUs), which still /// works through Lemonade's Vulkan backend. Unknown covers both a GPU - /// nothing detected and a target this build's catalog does not recognise: + /// nothing detected and a target this build's catalog does not recognize: /// those are detection gaps, not a verdict on the hardware, which is why /// an empty `compatible_therock_family` does not read as a failed probe. Examine { From 2ce0e14696c9b7b1c5a89e360606140c02a03fdb Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Wed, 11 Nov 2026 20:53:30 +0200 Subject: [PATCH 23/37] fix(examine): stop naming a backend for a GPU the catalog does not know MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-ups on the support-level messaging. The `Unknown` arm recommended `rocm serve --engine lemonade` for a detected-but-unrecognised target. An unrecognised target is a gap in this build's catalog, and a newer ROCm-capable GPU falls into it just as readily as a Vulkan-only one, so pinning the Vulkan backend is the same unestablished claim this level exists to avoid, only pointing the other way. The test next to it already gave the right reasoning — "`rocm serve` runs and its `device:` row settles the backend" — which is true only of the unpinned command, so the advice and its own justification disagreed. Plain `rocm serve ` now, and the test pins that. `assert_device_matches` panicked through `.expect` when no serve summary had been recorded. Its real invariant — the backend named by `device:` must be one the engine could have selected — has already been asserted by then; the cross-surface comparison is an extra that needs a Given which captured `rocm serve`'s output. serve-05 and serve-07 both have one, but a future scenario reusing this Then for the backend check alone would have failed for a reason about the fixture rather than about the product. Two costs are now recorded rather than left implicit, both raised in review: - `SERVICE_DEVICE_PROBE_BUDGET`'s ~18s worst case is written up as a decision, naming who pays it (`rocm services list` and the chat `services` tool, holding the process-wide engine env lock), the two cheaper options considered and why each was rejected, and the fact that it is a reversible constant rather than a structural choice. - `pinned_gpu_name` deduplicates its Windows display-inventory read within a call but does not cache across calls, and `write_running_state` runs more than once per launch, so a Windows launch pays several PowerShell/ CIM probes. It is on the launch path only, which is why the cost is accepted; a cache would need hotplug invalidation to stay correct. Also drops a stray blank line in `docs/testing.md`. Signed-off-by: Roman Sirokov --- apps/rocm/src/engine_ipc.rs | 20 +++++++++++++++++++ crates/rocm-core/src/host_gpu.rs | 10 ++++++++++ crates/rocm-core/src/host_gpu/render.rs | 20 +++++++++++++++---- docs/testing.md | 1 - tests/e2e-cucumber/tests/e2e/serving_steps.rs | 13 ++++++++---- 5 files changed, 55 insertions(+), 9 deletions(-) diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index ff9793975e..3fcf016643 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -469,6 +469,26 @@ where /// applies equally to the chat `services` tool, which renders through the same /// [`crate::services_text::render_services_text`]. /// +/// # The decision, recorded (review on #678) +/// +/// This is accepted rather than merely observed. `rocm services list` and the +/// chat `services` tool can both block ~18s on a host whose engines are slow to +/// answer, and for the duration the built-in probe holds the process-wide +/// engine env lock [`with_scoped_builtin_engine_env`] takes, so a concurrent +/// caller waits behind it. +/// +/// Two cheaper options were considered and not taken. Moving the probe off the +/// render path means the row stops reflecting the host at the moment it is +/// printed, which is the whole reason ROCMAI-550 added it — the `device:` row +/// exists to answer "what is this actually running on *now*". Probing once and +/// printing nothing for the rest makes the output depend on service ordering, +/// which reads as a bug to anyone who lists twice and sees different rows. +/// +/// Both remain reasonable calls and this one is reversible: it is a constant and +/// a start gate, not a structural choice. If a maintainer prefers a 10s hard +/// cap, the change is a per-probe deadline, which needs a cancellable probe +/// (see below) rather than a smaller number here. +/// /// Why not a per-probe deadline that would hold the total at 10s: the probe is /// `engine_request`, which for a built-in engine is a synchronous in-process /// call and for an external plugin is a `wait()` on a spawned child. Neither diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index 87b6ad185f..e9c79cbb10 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -501,6 +501,16 @@ pub fn pinned_gpu_name(gpu_indices: &[u32]) -> Option { // launch path. The read is eager rather than lazy only on the platform that // uses it: off Windows nothing is probed here at all, and on Windows the // count already needed it before any of the decisions below could run. + // + // The cost, stated rather than implied (raised in review on #678): this + // deduplicates *within* a call and is **not** cached across calls. Each + // engine's `write_running_state` calls it, and that runs more than once per + // launch (`starting`, then `running`), so a Windows launch pays that many + // PowerShell/CIM probes. It is on the launch path only — no request path + // reaches it — which is why the cost is accepted rather than cached: a + // cache would have to be invalidated on GPU hotplug to stay correct, and no + // caller is hot enough to earn that. Unverified on Windows; nothing + // available to this change can build the MSVC target. let windows_inventory = runtime_is_windows() .then(detect_windows_examine_inventory) .flatten(); diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index e6f0f479f2..89b0e85595 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -101,12 +101,20 @@ impl ExamineSummary { // nor its absence" in both arms was length without information. // The distinction it was carrying lives in this comment and in // `GpuSupportLevel`'s own docs, which is where it is load-bearing. + // The command is deliberately NOT `--engine lemonade`. An + // unrecognised target is a gap in this build's catalog, and a newer + // ROCm-capable GPU falls into it just as readily as a Vulkan-only + // one — naming the Vulkan backend would make exactly the kind of + // unestablished claim this level exists to avoid, only in the other + // direction. Plain `rocm serve` lets the engine choose, and the + // `device:` row then reports what it actually used, which is the + // answer this host needs and nothing here can predict. GpuSupportLevel::Unknown => match detected { Some(gfx) => format!( " rocm_support: unknown - {gfx} is not in this build's \ - catalog; serve with `rocm serve --engine lemonade`, \ - then read the `device:` row of `rocm services list` for the \ - backend it used\n" + catalog; serve with `rocm serve `, then read the \ + `device:` row of `rocm services list` for the backend it \ + used\n" ), None => " rocm_support: unknown - no AMD GPU was detected; run \ `rocm diagnose` to find out why none is visible\n" @@ -556,7 +564,11 @@ mod tests { // distinction against the behaviour rather than the wording. for (detected, expected_command) in [ (None, "rocm diagnose"), - (Some("gfx803"), "rocm serve --engine lemonade"), + // Plain `rocm serve`, not `--engine lemonade`: see the arm's own + // comment. The reasoning this test already gave — "`rocm serve` + // runs and its `device:` row settles the backend" — is only true of + // the unpinned command. + (Some("gfx803"), "rocm serve "), ] { let rendered = examine_summary_with_gpu(detected, None).render_text(); let line = rendered diff --git a/docs/testing.md b/docs/testing.md index 92f0ca370d..60cb0030ae 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -3857,7 +3857,6 @@ the scenario carries `@requires-os:linux`. On the GPU lanes it is `@requires-no-gpu` that excludes it. The missing hook is the deciding gate only where a Linux, GPU-less lane runs a prebuilt binary. - ## The Reported Backend Device And Support Level ROCMAI-550 made two surfaces report what this host can actually do instead of a diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 9f8feabe73..03624b221f 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -3221,10 +3221,15 @@ async fn assert_services_list_reports_device(world: &mut E2eWorld) { // that scenario in model_serving.feature). What remains asserted is the // half that is a real invariant: the two surfaces read one engine over one // RPC, so where both speak they must say the same thing. - let summary = world - .serve_summary - .as_deref() - .expect("the Given step must have recorded the serve summary"); + // And only when a summary was recorded at all. The backend assertion above + // is this step's real invariant and has already run; the cross-surface + // comparison is an extra that needs a Given which captured `rocm serve`'s + // own output. serve-05 and serve-07 both have one today, but panicking here + // would fail any future scenario reusing this Then for the backend check + // alone — a failure about the fixture rather than about the product. + let Some(summary) = world.serve_summary.as_deref() else { + return; + }; if let Some(summary_device) = device_row_value(summary) { assert_eq!( summary_device, device, From 3631908929bb855db2b101152bdab73296faf764 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 12 Nov 2026 12:23:19 +0200 Subject: [PATCH 24/37] test(examine): split the examine-49 arm the Unknown-tier advice outgrew MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Changing the Unknown tier's advice to plain `rocm serve ` left the examine-49 step still requiring `--engine lemonade` for *every* detected GPU with no covering family. A lane with an unrecognized GPU would therefore have failed against correct output; no current lane has one, which is the only reason it stayed green. The arm now splits the two tiers it was conflating. An unrecognized target expects plain `rocm serve ` and forbids `--engine lemonade` — naming the Vulkan backend for a card the catalog has no opinion about is the claim the advice change removed. A target confirmed to have no ROCm build still expects it. They are told apart by the line's own level wording, which is what the user reads. Signed-off-by: Roman Sirokov --- tests/e2e-cucumber/tests/e2e/examine_steps.rs | 22 ++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 06a170a267..41285d1810 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -3312,9 +3312,25 @@ async fn assert_support_level_advice_fits(world: &mut E2eWorld) { (_, Some(_), Some(_)) => ("rocm serve ", &["rocm install sdk"]), // ROCm-capable with no managed SDK yet: that install is the next step. (_, Some(_), None) => ("rocm install sdk", &[]), - // A GPU no package family covers — confirmed Vulkan-only (gfx90c) or - // merely unrecognized. Either way the SDK has nothing for it and - // Lemonade's Vulkan backend is what can serve on it. + // A GPU no package family covers. Two different tiers land here and + // they do NOT owe the same command, which this arm used to miss: + // + // * merely unrecognized — the catalog has no opinion, and it may be a + // newer ROCm-capable card, so pinning the Vulkan backend would claim + // something nothing established. Plain `rocm serve `, and + // `--engine lemonade` is forbidden. + // * confirmed to have no ROCm build (gfx90c) — Vulkan is the backend + // that can serve it, so the advice names it. + // + // Told apart by the line's own level wording, which is what the user + // reads. Before this split the arm required `--engine lemonade` for + // both, so a lane with an unrecognized GPU would have failed against + // *correct* output; no current lane has one, which is the only reason + // it stayed green. + (Some(_), None, _) if line.contains("is not in this build's catalog") => ( + "rocm serve ", + &["rocm install sdk", "--engine lemonade"], + ), (Some(_), None, _) => ( "rocm serve --engine lemonade", &["rocm install sdk"], From c618148f06ba25310b9fbca4761f561674aafcd8 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Thu, 12 Nov 2026 13:13:00 +0200 Subject: [PATCH 25/37] docs(testing): describe the levels this change has, not an unmerged one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The backend-device section announced a fourth level and pointed at the Vulkan-probe change for it. This branch has three levels and no probe, so that text reads as false the moment this merges on its own — which is the only way it can merge, since the probe is a separate PR. Rewritten to describe what is here: Vulkan-only inferred from the gfx table, three levels, and the probe named as separate work rather than as a reference the reader can follow. Signed-off-by: Roman Sirokov --- docs/testing.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index be34549a2f..b7baab4c50 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -3914,9 +3914,11 @@ same report prints, so every lane checks the level its own hardware implies: cargo xtask e2e -- -n '(examine-49)' ``` -A fourth level, `NoGpuBackend`, and the scenarios covering the Vulkan tiers -live in the Vulkan-probe change (#1036), which is what makes that level -knowable. +Vulkan-only here is inferred from the gfx table, not probed: this change has +three levels and this section describes three. Probing the host for a Vulkan +driver, and the further levels that evidence makes knowable, is separate work. +Written this way deliberately — a forward reference to an unmerged change reads +as false the moment this merges on its own. ## Windows Tool Notes From f050eb16f68cdffb31143f6dee8b66d9bfbe94be Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 13 Nov 2026 15:02:33 +0200 Subject: [PATCH 26/37] fix(serve): skip the device probe for a service that was already running MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The summary's device probe was gated on `report.status == "ready"` alone, under a comment claiming it used "the same gate as the smoke test". It did not: the smoke test beside it, and the Anthropic-endpoint wait below it, both also require `!report.already_running`. A reused service is `ready` by definition, so every repeat `rocm serve` of a live model opened the probe and paid its full retry budget plus one in-flight probe — about 14s with Lemonade's 8s inference probe — sequentially, ahead of the Anthropic-endpoint wait, for a row the command cannot learn anything new for. This is the traced cause of the five Anthropic scenarios failing on the Strix Halo Windows lane, all of which reuse a service from their precondition. The gate moves into `serve_device_probe_opens` with the probe injected as a closure, so a unit test can assert the probe is not merely discarded on the reuse path but never called — the wall time is the regression, not the row. Signed-off-by: Roman Sirokov --- apps/rocm/src/serve_cmd.rs | 96 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 91 insertions(+), 5 deletions(-) diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 7e95e78e18..86da2195e7 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -456,6 +456,32 @@ fn reuse_pregate_opens( .flatten() } +/// Whether the deployment summary may spend a device probe, given what the +/// launch report says happened. +/// +/// `!already_running` is the load-bearing half, not a cosmetic echo of the +/// smoke test's gate. A service that was already running is `"ready"` by +/// definition, so keying on the status alone opens this on *every* repeat +/// `rocm serve` of a live model — and the probe is not free: on a miss it costs +/// `engine_ipc::SERVE_DEVICE_PROBE_BUDGET` plus one in-flight probe, about 14s +/// with Lemonade's 8s inference probe, paid sequentially before the summary's +/// remaining waits. That is a long stall bought for a row this command cannot +/// learn anything new for — the engine answered the same thing when the service +/// was launched, and a *running* service's device is what `rocm services list` +/// reads, with its own budget for exactly that. +/// +/// `probe` is a closure rather than a value for that reason: on the reuse path +/// the cost must not be paid at all, which is what the unit test pins. +fn serve_device_probe_opens( + already_running: bool, + status: &str, + probe: impl FnOnce() -> Option, +) -> Option { + (!already_running && status == "ready") + .then(probe) + .flatten() +} + /// The no-usable-GPU bail's host precondition, read from the visibility-resolved /// set of usable AMD GPU ordinals. /// @@ -1251,8 +1277,11 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // Read what the engine actually selected rather than guessing from // the pre-launch GPU plan: `gpu_indices` above can be empty for a // Vulkan-only GPU that ROCm/amd-smi never enumerates, even though - // the server is running fine on it. Only meaningful once the - // server is ready to answer, same gate as the smoke test. + // the server is running fine on it. + // + // Only for a launch this command performed — see + // [`serve_device_probe_opens`] for why skipping the reuse path is + // the point rather than tidiness. // // Asked more than once, briefly: `engines/lemonade` answers // `"unknown"` until its own inference probe has come back, which is @@ -1260,9 +1289,9 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // bounded (`engine_ipc::SERVE_DEVICE_PROBE_BUDGET`) and `None` is a // supported outcome — the summary then has no `device:` row, and // `rocm services list` shows it once the engine can say. - let device = (report.status == "ready") - .then(|| fetch_service_device_briefly(&paths, &selected_engine, &report.service_id)) - .flatten(); + let device = serve_device_probe_opens(report.already_running, &report.status, || { + fetch_service_device_briefly(&paths, &selected_engine, &report.service_id) + }); let mut notes = collect_serve_notes( cpu_only, @@ -1493,6 +1522,63 @@ mod tests { ); } + /// Regression coverage for the device probe's gate (ROCMAI-550 review). + /// + /// The gate shipped as `report.status == "ready"` alone, under a comment + /// claiming it was "the same gate as the smoke test" — which it was not: the + /// smoke test and the Anthropic-endpoint wait beside it both also require + /// `!already_running`. A reused service is `"ready"` by definition, so the + /// omission made every repeat `rocm serve` of a live model pay the probe's + /// full retry budget (plus one in-flight probe, ~14s with Lemonade) before + /// printing a summary of a launch that did not happen. + /// + /// The probe is asserted *not consulted*, not merely that the row is absent: + /// the cost is the bug, and a gate that probed and then discarded the answer + /// would satisfy a value-only assertion. + #[test] + fn the_device_probe_is_not_spent_on_a_service_that_was_already_running() { + let mut probed = false; + assert!( + serve_device_probe_opens(true, "ready", || { + probed = true; + Some("gpu") + }) + .is_none(), + "a reused service must not open the device probe: it is `ready` by \ + definition, so the budget would be spent on every repeat `serve`" + ); + assert!( + !probed, + "the probe must not even be called on the reuse path — the wall time \ + is the regression, not the `device:` row" + ); + + // A launch that is not ready has nothing to ask the engine about yet. + let mut probed_starting = false; + assert!( + serve_device_probe_opens(false, "starting", || { + probed_starting = true; + Some("gpu") + }) + .is_none() + ); + assert!(!probed_starting); + + // The row itself still works for the case it was added for: a fresh + // launch that reached ready, where the engine knows what it selected. + assert_eq!( + serve_device_probe_opens(false, "ready", || Some("gpu")), + Some("gpu"), + "a fresh ready launch must still read the engine's device, or the \ + Vulkan-only `device:` row this PR adds is gone" + ); + // `None` from the engine stays a supported answer: no row, no error. + assert_eq!( + serve_device_probe_opens(false, "ready", || Option::<&str>::None), + None + ); + } + #[test] fn serve_engine_selection_uses_shared_recipe_when_no_override_exists() { let recipe = resolve_builtin_model_recipe("qwen32b").expect("qwen32b recipe"); From 64dc07470022397d0a03d07d0616f2f85b046b2f Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 13 Nov 2026 17:23:28 +0200 Subject: [PATCH 27/37] fix(engines): stop `rocm engines` requiring ROCm on a host it says has none The `rocm engines` banner was the constant "ROCm GPU execution is required.", printed a few rows above the per-engine notes. That was consistent while lemonade's note carried the same static ROCm-only sentence; it stopped being consistent as soon as this branch made that note host-aware, so a Vulkan-only host read "ROCm GPU execution is required" directly over "this GPU has no ROCm/HIP support" - the stale claim ROCMAI-550 exists to remove, reinstated on the same screen. What is invariant across every support level is the refusal, not the backend: these engines serve on a GPU and never fall back to the CPU, and Vulkan is one of the GPU paths rather than a fallback from one. So the line now says "GPU execution is required" at every level and adds the ROCm half only where the classifier establishes it. `Unknown` drops it too, which is the point of routing the whole line through `GpuSupportLevel` instead of special-casing `VulkanOnly`: asserting ROCm is required on a host whose support level is unknown is the same unfounded claim, just on the hosts the CLI knows least about. Also assert the Vulkan-only remediation command. `rocm_support_line` recommends `rocm serve --engine lemonade`, and the only test over that arm pinned the pre-existing "the Vulkan backend can still be used" substring, so the sentence could be deleted and the suite stayed green - which AGENTS.md section 3 forbids for advice naming a command. Centralise `first_gpu_index_from_state` into `rocm-engine-protocol` beside `format_reported_device`: lemonade and vllm held byte-identical private copies of the same `gpu_indices` read, and the two engines have to agree on which ordinal they report, not only on how it is spelled. No new crate edge - both already depend on the protocol crate. Fix the `bridge_snapshot` intra-doc link, which named a `builtin_engine_inventory` that does not exist. Signed-off-by: Roman Sirokov --- apps/rocm/src/bridge_snapshot.rs | 2 +- apps/rocm/src/examine_report_text.rs | 99 ++++++++++++++++++++++++- crates/rocm-core/src/host_gpu/render.rs | 18 +++++ crates/rocm-engine-protocol/src/lib.rs | 68 +++++++++++++++++ engines/lemonade/src/device_policy.rs | 18 ++--- engines/vllm/src/state.rs | 17 ++--- 6 files changed, 196 insertions(+), 26 deletions(-) diff --git a/apps/rocm/src/bridge_snapshot.rs b/apps/rocm/src/bridge_snapshot.rs index 741c18ede7..a5c7b648b3 100644 --- a/apps/rocm/src/bridge_snapshot.rs +++ b/apps/rocm/src/bridge_snapshot.rs @@ -73,7 +73,7 @@ pub(crate) fn build_codex_bridge_gpu_snapshot(config: &RocmCliConfig) -> CodexBr } /// The engine catalogue in the codex-bridge snapshot and the bridge's `engines` -/// tool, built from [`builtin_engine_inventory`] with lemonade's note replaced +/// tool, built from [`engine_inventory`] with lemonade's note replaced /// by `lemonade_engine_note` for the given support level. /// /// It takes the level rather than reading the static row because the stale diff --git a/apps/rocm/src/examine_report_text.rs b/apps/rocm/src/examine_report_text.rs index 12d66bee2b..126d433479 100644 --- a/apps/rocm/src/examine_report_text.rs +++ b/apps/rocm/src/examine_report_text.rs @@ -249,7 +249,7 @@ fn render_engine_inventory_text_for_host( output, " Built-in engines are included with rocm-cli. External plugins are optional." ); - let _ = writeln!(output, " ROCm GPU execution is required."); + let _ = writeln!(output, "{}", gpu_execution_line(gpu_support_level)); if let Some(paths) = paths { let _ = writeln!(output, " Plugin folders:"); for (index, path) in engine_plugin_dirs(paths).iter().enumerate() { @@ -276,6 +276,43 @@ fn render_engine_inventory_text_for_host( output } +/// The `rocm engines` banner stating what these engines will and will not run +/// on, for a host at `level`. +/// +/// This line used to be the constant `" ROCm GPU execution is required."`, +/// printed a few rows above the per-engine notes. That was consistent while +/// lemonade's note was the same static ROCm-only sentence, and stopped being +/// consistent the moment [`crate::engine_inventory_note`] made that note +/// host-aware: on a `VulkanOnly` host the banner asserted ROCm was required +/// directly above a row saying this GPU has no ROCm support — reinstating, on +/// the same screen, the stale claim ROCMAI-550 exists to remove. +/// +/// What is actually invariant across every level is the *refusal*, not the +/// backend: rocm-cli serves on a GPU and never silently falls back to CPU, and +/// Vulkan is one of the GPU paths it serves on rather than a fallback from +/// one. So "GPU execution is required" is said at every level and the ROCm +/// half is said only where the classifier establishes it. `Unknown` drops it +/// too, and that is the point of preferring this over a `VulkanOnly`-only +/// special case: "ROCm is required" on a host whose support level is unknown +/// is the same unfounded claim, just on the hosts the CLI knows least about. +/// `Unknown` does not re-name `rocm examine` either — lemonade's own row +/// already does, two lines below. +/// +/// Matched exhaustively, like the notes it has to agree with, so a fourth +/// support level is a compile error here instead of silently inheriting a +/// claim about ROCm. +const fn gpu_execution_line(level: rocm_core::GpuSupportLevel) -> &'static str { + match level { + rocm_core::GpuSupportLevel::RocmHip => " ROCm GPU execution is required.", + rocm_core::GpuSupportLevel::VulkanOnly => { + " GPU execution is required; on this GPU that is the Vulkan backend, not ROCm." + } + rocm_core::GpuSupportLevel::Unknown => { + " GPU execution is required; CPU execution is not supported." + } + } +} + pub(crate) fn append_examine_runtime_state( output: &mut String, paths: &AppPaths, @@ -731,7 +768,13 @@ mod tests { assert!(rendered.contains("Local model engines")); assert!(rendered.contains("Built-in engines are included with rocm-cli")); - assert!(rendered.contains("ROCm GPU execution is required")); + // Not `"ROCm GPU execution is required"`: this renderer probes the real + // host, so the ROCm half of that sentence is now whatever this runner's + // `GpuSupportLevel` is (see `gpu_execution_line`). The part that holds + // on every runner — and the part this test is about — is that engines + // need a GPU at all. Which backend is claimed for which level is pinned + // by `engines_banner_and_engine_note_cannot_contradict_each_other`. + assert!(rendered.contains("GPU execution is required")); assert!(rendered.contains("Plugin folders:")); assert!(rendered.contains(&paths.primary_engine_plugin_dir().display().to_string())); } @@ -1437,4 +1480,56 @@ mod tests { ); let _ = fs::remove_dir_all(root); } + + /// `rocm engines` must not assert and deny ROCm support on one screen. + /// + /// The three tests above each check one half of the page — the per-engine + /// note — and would all pass with the banner above it flatly contradicting + /// them, which is exactly the state this test was added for: the banner was + /// the constant `"ROCm GPU execution is required."` printed a few lines + /// over "this GPU has no ROCm/HIP support". + /// + /// Stated as a relation between the two lines rather than as expected text + /// per level, so it keeps holding if either sentence is reworded: the claim + /// under test is that the banner requires ROCm on exactly the hosts whose + /// row says they have it. That covers every level and not just the + /// `VulkanOnly` one — `Unknown` must not be told ROCm is required either, + /// when its own row says the support level is not yet known. + #[test] + fn engines_banner_and_engine_note_cannot_contradict_each_other() { + let (root, paths) = test_paths("engines-banner-vs-note"); + + for level in [ + rocm_core::GpuSupportLevel::RocmHip, + rocm_core::GpuSupportLevel::VulkanOnly, + rocm_core::GpuSupportLevel::Unknown, + ] { + let output = render_engine_inventory_text_for_host(Some(&paths), "lemonade", level); + + // The invariant that survives every level: these engines serve on a + // GPU, never on the CPU. Dropping the banner entirely would make the + // contradiction assertions below vacuously true. + assert!( + output.contains("GPU execution is required"), + "{level:?}: the GPU requirement must still be stated:\n{output}" + ); + + let banner_requires_rocm = output.contains("ROCm GPU execution is required"); + let note_claims_rocm = + output.contains("default embedded Lemonade server with ROCm llama.cpp backend"); + assert_eq!( + banner_requires_rocm, + matches!(level, rocm_core::GpuSupportLevel::RocmHip), + "{level:?}: the banner may require ROCm only where the classifier \ + establishes it:\n{output}" + ); + assert_eq!( + banner_requires_rocm, note_claims_rocm, + "{level:?}: the banner and lemonade's row must make the same ROCm claim \ + about the same host:\n{output}" + ); + } + + let _ = fs::remove_dir_all(root); + } } diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index 89b0e85595..dc732c4fa2 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -362,6 +362,24 @@ mod tests { rendered.contains("the Vulkan backend can still be used"), "{rendered}" ); + // AGENTS.md §3: advice naming a command is asserted together with the + // behaviour. Saying the Vulkan backend "can still be used" without + // naming how to use it leaves the user where they started, so the + // command is part of the claim, not decoration — without this + // assertion the recommendation could be deleted or garbled and this + // test would still pass. + // + // That the named command *parses* (the subcommand exists and accepts + // `--engine lemonade`) is proved mechanically rather than here: + // `advised_commands::every_advised_command_parses` in `apps/rocm` + // scans production string literals under `crates/`, including this + // one, and routes each through the real clap tree. rocm-core cannot + // see the CLI's argument parser, so this half pins that the sentence + // is printed at all for a Vulkan-only host. + assert!( + rendered.contains("serve on it with `rocm serve --engine lemonade`"), + "a Vulkan-only host must be told the command that serves on it:\n{rendered}" + ); } /// An unmapped target must not be *claimed* unsupported, even though every diff --git a/crates/rocm-engine-protocol/src/lib.rs b/crates/rocm-engine-protocol/src/lib.rs index e9179b9698..35df37bcbe 100644 --- a/crates/rocm-engine-protocol/src/lib.rs +++ b/crates/rocm-engine-protocol/src/lib.rs @@ -631,6 +631,35 @@ pub fn format_reported_device( device } +/// The first pinned GPU ordinal recorded in a service state file, if any. +/// +/// The ordinal half of what [`format_reported_device`] renders, read from the +/// state file every engine writes at launch. It lives beside that function for +/// the same reason that one does: `rocm serve`'s `device:` row and +/// `rocm services list` print whichever engine answered verbatim, so the two +/// engines that pin ordinals (`engines/lemonade`, `engines/vllm`) have to agree +/// on *which* ordinal they report, not only on how it is spelled. They had +/// byte-identical private copies of this, which is one keystroke away from +/// disagreeing silently — the same duplication `format_reported_device` was +/// centralized to end. +/// +/// `None` covers every shape that is not a usable ordinal: no `gpu_indices` +/// key, a non-array value, an empty list (a launch that pinned nothing, which +/// is the default on Windows — see `format_reported_device`), a non-integer +/// entry, and a value too large for `u32`. Those are "no ordinal to report", +/// not errors: state files are written by engine processes of possibly +/// different versions, and a row that says less is better than one that +/// invents an ordinal or refuses to render. +#[must_use] +pub fn first_gpu_index_from_state(state: &Value) -> Option { + state + .get("gpu_indices") + .and_then(Value::as_array) + .and_then(|indices| indices.first()) + .and_then(Value::as_u64) + .and_then(|value| u32::try_from(value).ok()) +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct HealthcheckResponse { /// Engine-reported service state. `ready` is reserved for a service that has @@ -760,6 +789,45 @@ mod tests { ); } + /// Every shape a state file can carry in `gpu_indices` resolves to "an + /// ordinal" or "no ordinal", never to a panic or an invented value. + /// + /// This used to be two byte-identical private helpers in `engines/lemonade` + /// and `engines/vllm`, neither of which tested the shapes below; only the + /// happy path was reachable from either engine's own tests. Centralizing it + /// is what makes the edge cases testable in one place for both callers. + #[test] + fn the_first_pinned_ordinal_is_read_and_every_other_shape_is_absent() { + use serde_json::json; + + assert_eq!( + super::first_gpu_index_from_state(&json!({"gpu_indices": [1, 2]})), + Some(1), + "the first pinned ordinal is the one reported" + ); + // A launch that pinned nothing — the default on Windows, which is the + // host ROCMAI-550 was filed from. + assert_eq!( + super::first_gpu_index_from_state(&json!({"gpu_indices": []})), + None + ); + assert_eq!(super::first_gpu_index_from_state(&json!({})), None); + // A state file written by some other version of an engine must degrade + // to "no ordinal", not to a wrong one. + assert_eq!( + super::first_gpu_index_from_state(&json!({"gpu_indices": "0"})), + None + ); + assert_eq!( + super::first_gpu_index_from_state(&json!({"gpu_indices": [-1]})), + None + ); + assert_eq!( + super::first_gpu_index_from_state(&json!({"gpu_indices": [u64::from(u32::MAX) + 1]})), + None + ); + } + fn resolve_response(served: Option<&str>) -> ResolveModelResponse { ResolveModelResponse { canonical_model_id: "image:tag".to_owned(), diff --git a/engines/lemonade/src/device_policy.rs b/engines/lemonade/src/device_policy.rs index 11a68d70bc..06421794da 100644 --- a/engines/lemonade/src/device_policy.rs +++ b/engines/lemonade/src/device_policy.rs @@ -146,11 +146,15 @@ pub(crate) const fn device_policy_name(policy: &DevicePolicy) -> &'static str { /// /// Formatted by [`rocm_engine_protocol::format_reported_device`] so all three /// engines render one shape; see that function for why the ordinal-less row -/// drops the `gpu` noun rather than printing a bare `vulkan gpu`. +/// drops the `gpu` noun rather than printing a bare `vulkan gpu`. The ordinal +/// is read by [`rocm_engine_protocol::first_gpu_index_from_state`] for the same +/// reason — `engines/vllm` reads the same `gpu_indices` key out of a state file +/// of the same shape, and kept a byte-identical copy of that read until it +/// moved next to the formatter. pub(crate) fn reported_device(state: Option<&Value>, backend: &str) -> String { rocm_engine_protocol::format_reported_device( backend, - state.and_then(first_gpu_index_from_state), + state.and_then(rocm_engine_protocol::first_gpu_index_from_state), state.and_then(gpu_name_from_state).as_deref(), ) } @@ -165,16 +169,6 @@ fn gpu_name_from_state(state: &Value) -> Option { crate::state::value_string(state, "gpu_name") } -/// The first pinned GPU ordinal recorded in service state, if any. -fn first_gpu_index_from_state(state: &Value) -> Option { - state - .get("gpu_indices") - .and_then(Value::as_array) - .and_then(|indices| indices.first()) - .and_then(Value::as_u64) - .and_then(|value| u32::try_from(value).ok()) -} - #[cfg(test)] mod tests { use super::*; diff --git a/engines/vllm/src/state.rs b/engines/vllm/src/state.rs index 95f3d8e064..694988c505 100644 --- a/engines/vllm/src/state.rs +++ b/engines/vllm/src/state.rs @@ -85,24 +85,19 @@ pub(crate) fn healthcheck_service(request: HealthcheckRequest) -> Result String { rocm_engine_protocol::format_reported_device( "rocm", - first_gpu_index_from_state(state), + rocm_engine_protocol::first_gpu_index_from_state(state), value_string(state, "gpu_name").as_deref(), ) } -/// The first pinned GPU ordinal recorded in service state, if any. -fn first_gpu_index_from_state(state: &Value) -> Option { - state - .get("gpu_indices") - .and_then(Value::as_array) - .and_then(|indices| indices.first()) - .and_then(Value::as_u64) - .and_then(|value| u32::try_from(value).ok()) -} - pub(crate) fn endpoint_response(request: EndpointRequest) -> Result { require_nonempty(&request.service_id, "service_id")?; let files = service_files(&request.service_id)?; From cc9e90aa28c9b2261f7c16ccbae2210132accc92 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 13 Nov 2026 18:53:54 +0200 Subject: [PATCH 28/37] fix(cli): stop markdown markup reaching `--help`, and make the Vulkan-only claim one string `rocm examine --help` printed literal `**every**`: clap renders doc comments verbatim, and the file's ASCII guard cannot see markup made of ASCII. The emphasis is dropped, and a sibling test walks the same command tree rejecting `**`/`__` so it cannot come back. Hidden `__engine-*` subcommand names are read off the tree and removed before the search rather than the `__` half being given up. No other help text in the tree was affected. "Unknown covers both a GPU nothing detected" was not English, in `--help` and in README.md both; reworded in sync. `install sdk`'s two refusals re-derived the Vulkan-only condition from `gfx_target_has_no_rocm_build` and re-typed its claim, which was exactly equivalent today and a second place to change tomorrow. They now gate on `rocm_core::gpu_support_level`'s `VulkanOnly` tier and interpolate one `VULKAN_ONLY_SUPPORT_CLAIM`, shared with `rocm examine`'s `rocm_support` line. All three strings are byte-identical to before. `SERVICE_DEVICE_PROBE_BUDGET` claimed the "chat `services` tool" pays the budget. The natural-language command does; the structured MCP tool of the same name renders through `render_services_tool_result_text`, has no `device:` row and probes nothing. The comment now says which is which, rather than the tool being given a row it was never in scope for. Signed-off-by: Roman Sirokov --- README.md | 2 +- apps/rocm/src/cli.rs | 74 +++++++++++++++-- apps/rocm/src/engine_ipc.rs | 24 ++++-- apps/rocm/src/services_text.rs | 10 +-- apps/rocm/src/therock/family.rs | 101 +++++++++++++++++++++--- crates/rocm-core/src/gfx_target.rs | 43 ++++++++++ crates/rocm-core/src/host_gpu/render.rs | 7 +- crates/rocm-core/src/lib.rs | 4 +- 8 files changed, 232 insertions(+), 33 deletions(-) diff --git a/README.md b/README.md index 2f8b63494f..a71fc591c0 100644 --- a/README.md +++ b/README.md @@ -545,7 +545,7 @@ These GPUs still work with the Vulkan backend, which needs no ROCm SDK. `rocm examine` prints that `rocm_support` line on **every** host, not only this case, at one of three levels: ROCm/HIP-supported, Vulkan-only (the case above), -or unknown. Unknown covers both a GPU nothing detected and a target this build's +or unknown. Unknown covers both no GPU detected and a target this build's catalog does not recognise — detection gaps rather than a verdict about the hardware, so neither is reported as "no ROCm build". diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index da05e42872..1f8fe71d0e 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -63,14 +63,14 @@ pub(crate) struct Cli { pub(crate) enum Command { /// Check this computer's GPU, ROCm install, engines, and setup folders. /// - /// A `rocm_support` line is printed on **every** host, at one of three + /// A `rocm_support` line is printed on every host, at one of three /// levels: ROCm/HIP-supported, Vulkan-only, or unknown. Vulkan-only covers /// a GPU that is recognized but has no ROCm build at all (`gfx90c`, the /// Vega iGPU in Renoir, Lucienne, Cezanne and Barcelo APUs), which still - /// works through Lemonade's Vulkan backend. Unknown covers both a GPU - /// nothing detected and a target this build's catalog does not recognize: - /// those are detection gaps, not a verdict on the hardware, which is why - /// an empty `compatible_therock_family` does not read as a failed probe. + /// works through Lemonade's Vulkan backend. Unknown covers both no GPU + /// detected and a target this build's catalog does not recognize: those + /// are detection gaps, not a verdict on the hardware, which is why an + /// empty `compatible_therock_family` does not read as a failed probe. Examine { /// Emit the machine-readable Examination JSON (for diagnosis tooling). #[arg(long)] @@ -2680,4 +2680,68 @@ mod tests { assert_ascii_help(&mut Cli::command(), "rocm"); } + + /// Sibling of `every_help_text_in_the_command_tree_is_ascii`, for the same + /// reason one level down: clap renders a doc comment verbatim, with no + /// markdown processing anywhere in its pipeline, so `**every**` written in + /// a doc comment reaches the user's terminal as literal asterisks. Review + /// on #678 caught exactly that on `Examine`, which the ASCII test could + /// not see because `*` and `_` are ASCII. + /// + /// Only `**` and `__` are rejected, and only those: they have no honest + /// use in plain help text. A single `*` does (`gfx110*`, a glob in an + /// example), and backticks are the repo's deliberate house style for + /// quoting commands in help, so neither is banned here. + /// + /// The hidden `__engine-*` subcommands are a real `__` in rendered help — + /// their own usage lines print their names — so each command's internal + /// names are removed before the search rather than the check being + /// dropped. The list is read off the command tree, not hard-coded, so a + /// new hidden command does not make this fail spuriously. + #[test] + fn no_help_text_in_the_command_tree_contains_markdown_emphasis() { + fn assert_no_markdown_emphasis(cmd: &mut clap::Command, path: &str) { + let mut internal: Vec = cmd + .get_subcommands() + .map(|sub| sub.get_name().to_owned()) + .filter(|name| name.starts_with("__")) + .collect(); + if cmd.get_name().starts_with("__") { + internal.push(cmd.get_name().to_owned()); + } + + // Both renderings, for the same reason as the ASCII test: `-h` + // truncates at the first paragraph, so emphasis in a later one is + // only visible in the long form. + for rendered in [cmd.render_help(), cmd.render_long_help()] { + let mut rendered = rendered.to_string(); + for name in &internal { + rendered = rendered.replace(name.as_str(), ""); + } + for marker in ["**", "__"] { + assert!( + !rendered.contains(marker), + "`{path} --help` must not contain markdown emphasis (`{marker}`); \ + clap renders doc comments verbatim, so the user sees the markup \ + itself: {rendered}" + ); + } + } + // Collect first: `find_subcommand_mut` needs a mutable borrow that + // cannot overlap the iterator borrowing `cmd`. + let names: Vec = cmd + .get_subcommands() + .map(|sub| sub.get_name().to_owned()) + .collect(); + for name in names { + let child_path = format!("{path} {name}"); + let child = cmd + .find_subcommand_mut(&name) + .expect("subcommand name was just enumerated from this command"); + assert_no_markdown_emphasis(child, &child_path); + } + } + + assert_no_markdown_emphasis(&mut Cli::command(), "rocm"); + } } diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index e1a611bae6..4954c4e04d 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -469,16 +469,28 @@ where /// is a *start* gate: it is tested before each probe, never inside one, so a /// probe begun at 9.99s still runs to its own timeout. The arithmetic is 10s /// of elapsed budget plus Lemonade's 8s `INFERENCE_PROBE_TIMEOUT`. That bound -/// applies equally to the chat `services` tool, which renders through the same +/// applies equally to `rocm chat` when its natural-language interpreter +/// resolves a request to the `services` command: that path runs the real +/// command in process (`chat::mcp_bridge`'s `run_rocm_read_only_in_process`) +/// and so renders through the same /// [`crate::services_text::render_services_text`]. /// +/// It does *not* apply to the structured MCP tool that is also named +/// `services`. That one renders through `chat::mcp_bridge`'s +/// `render_services_tool_result_text`, a different and older function with no +/// `device:` row, so a bridge client calling the tool makes no probe and waits +/// for nothing — unlike the sibling `engines` MCP tool, which ROCMAI-550 did +/// make host-aware. The shared name is the trap, so this says which surface it +/// means (review on #678); giving the tool the row is a change in its own +/// right, not something this constant may be read as already covering. +/// /// # The decision, recorded (review on #678) /// -/// This is accepted rather than merely observed. `rocm services list` and the -/// chat `services` tool can both block ~18s on a host whose engines are slow to -/// answer, and for the duration the built-in probe holds the process-wide -/// engine env lock [`with_scoped_builtin_engine_env`] takes, so a concurrent -/// caller waits behind it. +/// This is accepted rather than merely observed. `rocm services list` and +/// `rocm chat`'s `services` command can both block ~18s on a host whose +/// engines are slow to answer, and for the duration the built-in probe holds +/// the process-wide engine env lock [`with_scoped_builtin_engine_env`] takes, +/// so a concurrent caller waits behind it. /// /// Two cheaper options were considered and not taken. Moving the probe off the /// render path means the row stops reflecting the host at the moment it is diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index cdac966bb4..497e376e31 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -402,15 +402,15 @@ fn format_anthropic_endpoint_line(url: &str, responding: bool) -> String { } } -/// `rocm services list`, and the chat `services` tool, which renders the same -/// text. +/// `rocm services list`, and `rocm chat`'s `services` command, which runs this +/// in process and renders the same text. Not the MCP tool of the same name, +/// which has its own renderer and no `device:` row. /// /// **No longer a pure local read.** Since ROCMAI-550 it makes one live /// `Healthcheck` RPC per *ready* service to fill the `device:` row, bounded in /// aggregate by `engine_ipc::SERVICE_DEVICE_PROBE_BUDGET` — see that constant -/// for the arithmetic behind the ~18s worst case. The chat tool pays the same -/// bound rather than render a device-less view, which would have the two -/// surfaces disagree while claiming to show the same list. +/// for the ~18s worst case and for which `services` surface pays it. The chat +/// command pays it too, rather than show a device-less view of the same list. pub(crate) fn render_services_text(paths: &AppPaths, all: bool) -> Result { render_services_text_with_devices(paths, all, |ready| fetch_service_devices(paths, ready)) } diff --git a/apps/rocm/src/therock/family.rs b/apps/rocm/src/therock/family.rs index a73f03b573..f0153330f5 100644 --- a/apps/rocm/src/therock/family.rs +++ b/apps/rocm/src/therock/family.rs @@ -12,9 +12,9 @@ use super::source::{FamilyResolution, TheRockChannel}; use anyhow::{Result, bail}; use rocm_core::{ - AppPaths, detect_host_gfx_target, detect_host_gpu_diagnostics, detect_managed_therock_family, - gfx_target_has_no_rocm_build, known_therock_families, normalize_therock_family, - runtime_is_windows, + AppPaths, GpuSupportLevel, VULKAN_ONLY_SUPPORT_CLAIM, detect_host_gfx_target, + detect_host_gpu_diagnostics, detect_managed_therock_family, gpu_support_level, + known_therock_families, normalize_therock_family, runtime_is_windows, }; use std::fmt::Write as _; @@ -99,7 +99,7 @@ pub(super) fn resolve_family( // left over from an earlier session happens to name gfx90c, and the // reverse for an env value once `--family` isn't the one in play. if let Some(value) = family_override { - if gfx_target_has_no_rocm_build(value) { + if target_is_vulkan_only(value) { bail!(unsupported_requested_target_message( value.trim(), "--family" @@ -115,7 +115,7 @@ pub(super) fn resolve_family( } if let Some(value) = env_override.as_deref() { - if gfx_target_has_no_rocm_build(value) { + if target_is_vulkan_only(value) { bail!(unsupported_requested_target_message( value.trim(), "ROCM_CLI_THEROCK_FAMILY" @@ -156,7 +156,7 @@ pub(super) fn resolve_family( // there is nothing to install for this GPU. Other unmapped targets keep the // generic error below, which points at `--family`. if let Some(raw_arch) = host_arch.as_deref() - && gfx_target_has_no_rocm_build(raw_arch) + && target_is_vulkan_only(raw_arch) { let message = unsupported_host_gpu_message(raw_arch); let diagnostics = detect_host_gpu_diagnostics(); @@ -172,6 +172,32 @@ pub(super) fn resolve_family( ) } +/// Whether `value` is a GPU target the CLI recognizes and knows has no ROCm +/// build — [`GpuSupportLevel::VulkanOnly`] — which is what `install sdk` +/// refuses with the two messages below. +/// +/// Asked of [`gpu_support_level`] rather than re-tested here. The install path +/// used to call `gfx_target_has_no_rocm_build` directly, which was *exactly* +/// equivalent (review on #678 checked that it was) and still wrong to keep: it +/// made this a second, independent definition of the tier whose claim these +/// messages then go on to make, so a change to the classification would have +/// left `rocm examine` and `rocm install sdk` disagreeing about the same GPU +/// while both looked correct in isolation. One classifier, one answer. +/// +/// The family argument is computed rather than passed as `None` because this +/// is also asked of a target the *user* named, before any host probe: for +/// `--family gfx1100` the covering family is what makes the answer `RocmHip`, +/// and hard-coding `None` here would classify every requested target as if no +/// family covered it. On the host branch the caller has already established +/// that `normalize_therock_family` answers `None`, so the extra call there is +/// a no-op that costs a string compare. +fn target_is_vulkan_only(value: &str) -> bool { + matches!( + gpu_support_level(Some(value), normalize_therock_family(value).as_deref()), + GpuSupportLevel::VulkanOnly + ) +} + /// The install error for a detected GPU target that no TheRock package family /// covers. It deliberately does not list `--family` values: none of them was /// built for this GPU. @@ -203,8 +229,7 @@ fn unsupported_host_gpu_message(raw_arch: &str) -> String { format!( "no ROCm SDK build exists for the detected GPU target `{raw_arch}`, so \ `rocm install sdk` has nothing to install for it.\n\ - The GPU is detected but is not supported by ROCm; \ - the Vulkan backend can still be used. \ + The GPU is detected but is {VULKAN_ONLY_SUPPORT_CLAIM}. \ Try `rocm serve --engine lemonade` - it runs on Lemonade's \ Vulkan backend instead." ) @@ -232,8 +257,8 @@ fn unsupported_requested_target_message(value: &str, source: &str) -> String { format!( "no ROCm SDK build exists for the GPU target `{value}` requested by \ {source}, so `rocm install sdk` has nothing to install for it.\n\ - That GPU is not supported by ROCm; the Vulkan backend can still be \ - used. Nothing was installed and no other family was substituted. \ + That GPU is {VULKAN_ONLY_SUPPORT_CLAIM}. Nothing was installed and no \ + other family was substituted. \ Run `rocm examine` to see what this host itself supports." ) } @@ -342,6 +367,60 @@ mod tests { assert_eq!(raw_arch_agreeing_with_family(None, "gfx103X-dgpu"), None); } + /// The gate on both install refusals is the classifier's `VulkanOnly` + /// tier, not a private re-derivation of it. + /// + /// Every case is one `gpu_support_level` answers, so a change there has to + /// come through here: `gfx90c` is the recognized-with-no-ROCm-build tier + /// and is refused; a covered target is `RocmHip` and is not; an + /// unrecognized one is `Unknown`, a gap in this build's catalog rather + /// than a fact about the card, and refusing it would be the exact false + /// claim `GpuSupportLevel`'s doc comment exists to prevent. + #[test] + fn the_install_refusal_gate_is_the_classifiers_vulkan_only_tier() { + for refused in ["gfx90c", "GFX90C", "gfx90c:xnack-", " gfx90c "] { + assert!( + target_is_vulkan_only(refused), + "`{refused}` is the Vulkan-only tier and install must refuse it" + ); + } + for allowed in ["gfx1100", "gfx1151", "gfx90a", "gfx950-dcgpu"] { + assert!( + !target_is_vulkan_only(allowed), + "`{allowed}` has a ROCm build; install must not refuse it" + ); + } + for unknown in ["gfx803", "not-a-target", ""] { + assert!( + !target_is_vulkan_only(unknown), + "`{unknown}` is unrecognized, which is a gap in this catalog and \ + not a confirmed absence of a ROCm build: refusing it would claim \ + something about hardware nobody probed" + ); + } + } + + /// Both refusals and `rocm examine`'s `rocm_support` line make the same + /// claim about a Vulkan-only GPU, so they render it from the same string. + /// + /// Asserting the constant is *in* each message is what makes it one + /// claim; the wording itself is pinned word-for-word in `rocm-core` + /// (`the_vulkan_only_claim_is_pinned_word_for_word`), so neither test + /// alone can be satisfied by quietly rewriting the other's expectation. + #[test] + fn both_install_refusals_render_the_shared_vulkan_only_claim() { + for message in [ + unsupported_host_gpu_message("gfx90c"), + unsupported_requested_target_message("gfx90c", "--family"), + ] { + assert!( + message.contains(VULKAN_ONLY_SUPPORT_CLAIM), + "the claim must come from the shared constant, or install and \ + examine can drift apart while both look right: {message}" + ); + } + } + #[test] fn unsupported_gpu_error_names_the_gpu_and_does_not_suggest_families() { let message = unsupported_host_gpu_message("gfx90c"); @@ -392,7 +471,7 @@ mod tests { /// Both refusals used to close with the same "`rocm serve ` runs on /// Lemonade's Vulkan backend" sentence. In `unsupported_host_gpu_message` /// that is a fact: it is only reached after `detect_host_gfx_target` returns - /// a target `gfx_target_has_no_rocm_build` confirms. This one fires from the + /// a target `target_is_vulkan_only` confirms. This one fires from the /// `--family` / `ROCM_CLI_THEROCK_FAMILY` branches before any probe runs — /// see `cli_family_overrides_a_stale_env_value_naming_an_unsupported_gpu` /// for an override that deliberately disagrees with its host — so on a diff --git a/crates/rocm-core/src/gfx_target.rs b/crates/rocm-core/src/gfx_target.rs index 092c288a46..ecbdfb4b0a 100644 --- a/crates/rocm-core/src/gfx_target.rs +++ b/crates/rocm-core/src/gfx_target.rs @@ -573,9 +573,34 @@ pub enum GpuSupportLevel { Unknown, } +/// The one claim every [`GpuSupportLevel::VulkanOnly`] surface makes, written +/// once. +/// +/// `rocm examine`'s `rocm_support` line and `rocm install sdk`'s two refusals +/// (`unsupported_host_gpu_message` / `unsupported_requested_target_message` in +/// `apps/rocm/src/therock/family.rs`) all tell the user the same two things +/// about a Vulkan-only GPU: ROCm will not run on it, and Vulkan will. They are +/// three different sentences around it — each adds its own subject and its own +/// next step, because what they may promise differs — but the claim itself is +/// shared, so it lives here next to the classifier that decides when it is +/// true rather than being retyped at each site. +/// +/// Deliberately just the claim, with no leading subject and no trailing +/// punctuation: a caller supplies both. Changing the wording here changes all +/// three surfaces together, which is the point. +pub const VULKAN_ONLY_SUPPORT_CLAIM: &str = + "not supported by ROCm; the Vulkan backend can still be used"; + /// Classify the host's GPU support level from the same two facts /// `ExamineSummary` and `resolve_family` already compute: the detected gfx /// target and whichever TheRock family (if any) covers it. +/// +/// Not only for a *host*: `resolve_family` also asks this of a target the user +/// named with `--family` or `ROCM_CLI_THEROCK_FAMILY`, to decide whether to +/// refuse it. The two arguments are facts about a GPU target, not about this +/// machine, so that is the same question. What differs is what the caller may +/// then say — a requested target establishes nothing about the hardware here, +/// which is why the two install refusals end differently. pub fn gpu_support_level( detected_gfx_target: Option<&str>, compatible_therock_family: Option<&str>, @@ -680,6 +705,24 @@ mod tests { ); } + /// The shared claim is user-visible text on three surfaces, so its wording + /// is pinned here rather than only asserted as "whatever the constant + /// says". + /// + /// `rocm examine`'s `rocm_support` line and both `rocm install sdk` + /// refusals now interpolate [`VULKAN_ONLY_SUPPORT_CLAIM`], which is what + /// stops them drifting — but it also means a careless edit to the constant + /// silently rewords all three at once, and their own tests would still + /// pass because they assert containment of the same constant. This test is + /// the one place that has to be changed deliberately. + #[test] + fn the_vulkan_only_claim_is_pinned_word_for_word() { + assert_eq!( + VULKAN_ONLY_SUPPORT_CLAIM, + "not supported by ROCm; the Vulkan backend can still be used" + ); + } + #[test] fn gfx_target_has_no_rocm_build_names_only_gfx90c() { assert!(gfx_target_has_no_rocm_build("gfx90c")); diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index dc732c4fa2..aec43dfbfb 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -6,7 +6,7 @@ use super::ExamineSummary; use super::os_facts::format_gib_value; -use crate::gfx_target::{GpuSupportLevel, gpu_support_level}; +use crate::gfx_target::{GpuSupportLevel, VULKAN_ONLY_SUPPORT_CLAIM, gpu_support_level}; impl ExamineSummary { /// The `rocm_support` line of the examine text. @@ -39,9 +39,8 @@ impl ExamineSummary { // fallback below is unreachable; it keeps the arm total without a // panic path. GpuSupportLevel::VulkanOnly => format!( - " rocm_support: GPU detected ({}) but not supported by ROCm; \ - the Vulkan backend can still be used - serve on it with \ - `rocm serve --engine lemonade`\n", + " rocm_support: GPU detected ({}) but {VULKAN_ONLY_SUPPORT_CLAIM} - \ + serve on it with `rocm serve --engine lemonade`\n", detected.unwrap_or_default() ), // The covering family is the fact; it is already printed on its own diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index c5ea6662a8..8120e70340 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -132,7 +132,9 @@ pub use fix::{ apply as apply_fix, catalog_manifest, catalog_manifest_json, exit as fix_exit, list_recipes as list_fix_recipes, }; -pub use gfx_target::{GpuSupportLevel, gpu_support_level, gpu_support_level_for_host_gpu}; +pub use gfx_target::{ + GpuSupportLevel, VULKAN_ONLY_SUPPORT_CLAIM, gpu_support_level, gpu_support_level_for_host_gpu, +}; pub use gfx_target::{ extract_first_gfx_token, gfx_target_has_no_rocm_build, known_therock_families, normalize_therock_family, From 39bec51d6c0ab714cb14fb1609a2f0746892536f Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Fri, 13 Nov 2026 18:58:37 +0200 Subject: [PATCH 29/37] docs: point the static-catalog note at rocmd's real engine inventory `lemonade_engine_note`'s doc comment named `apps/rocmd/src/common.rs` as the place keeping the unconditional ROCm wording. That file has no engine catalog and no such string; the table is `rocmd_engine_inventory` in `apps/rocmd/src/host_snapshot.rs`. The comment exists to say which surfaces are deliberately left host-unaware, so a wrong pointer defeats its only purpose -- a reader checking the claim finds nothing and cannot tell whether the deferral is real. Signed-off-by: Roman Sirokov --- apps/rocm/src/main.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 83558840d1..b6f0e54164 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -1052,7 +1052,8 @@ pub(crate) fn engine_inventory_note( /// /// All three of those have a live `GpuSupportLevel` in scope. The two places /// that still print the note statically — the dashboard TUI's engine-manager -/// pane and `rocmd`'s own catalog in `apps/rocmd/src/common.rs` — keep the +/// pane and `rocmd`'s own `rocmd_engine_inventory` in +/// `apps/rocmd/src/host_snapshot.rs` — keep the /// unconditional ROCm wording, and that is deliberate rather than an oversight: /// the TUI pane renders a hard-coded descriptive list with no probe behind it, /// and `rocmd` is the daemon answering for whichever host a client asks about, From 557aa3434f0b4cb6c49c198debfa458a0994dcf4 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Mon, 16 Nov 2026 15:18:01 +0200 Subject: [PATCH 30/37] fix(services): stop the device probe losing a just-published anthropic endpoint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `render_services_text_with_devices` snapshots every managed-service record once, up front, through `load_managed_services`. That used to be a local read, so the snapshot was effectively taken at print time. This branch put the live `Healthcheck` device probe in between, and that probe can spend seconds — up to `engine_ipc::SERVICE_DEVICE_PROBE_BUDGET`, about 18 of them — before the first row is written. The serving process publishes the Anthropic endpoint to the engine state file at about the moment the engine first lists its model, which is the same moment the device probe starts getting an answer. So the gap this branch opened is exactly the window in which a freshly ready service renders with no `anthropic endpoint:` row at all, from a snapshot that predates the publication. The Strix Halo Windows e2e lane caught it on content-identical commit d35ac5fe (run 43468147450): five scenarios regressed — `serve-anthropic-endpoint-advertised`, `-messages-inference`, `-endpoint-readiness`, `-messages-tool-use` and `-messages-streaming` — on a lane that passes on three consecutive recent `main` commits. Re-read the one field that moves, after the probe and before printing, with `rocm_core::wait_for_published_anthropic_endpoint` — the same read `serve_cmd.rs` already uses for the same race in the post-launch summary, rather than a second idiom. With no wait: a wait is right there, where the command has just launched the server and is standing by for it, and wrong here, where `rocm services list` reports on whatever exists and must not start blocking on a service that may never publish. Reading once restores the timing the listing had before the probe. Gated on the same statuses as `ManagedServiceRecord::refresh_from_engine_state`, which is what put the field on the snapshot to begin with; for anything else the state file is a finished run's leftovers. `services_listing_shows_an_anthropic_endpoint_published_while_the_probe_ran` pins it, and discriminates on the staleness rather than on the row's presence: its probe closure publishes the endpoint while it runs, so rendering from the pre-probe snapshot shows no row. Verified by removing the production call — the test fails, with the rendered listing in the message. Signed-off-by: Roman Sirokov --- apps/rocm/src/services_text.rs | 141 ++++++++++++++++++++++++++++++++- 1 file changed, 140 insertions(+), 1 deletion(-) diff --git a/apps/rocm/src/services_text.rs b/apps/rocm/src/services_text.rs index 782ab1dcc6..91c1745382 100644 --- a/apps/rocm/src/services_text.rs +++ b/apps/rocm/src/services_text.rs @@ -439,7 +439,7 @@ fn render_services_text_with_devices( .iter() .find(|record| !managed_service_is_live(record)) .map(|record| record.service_id.clone()); - let records = all_records + let mut records = all_records .into_iter() .filter(|record| all || managed_service_is_live(record)) .collect::>(); @@ -486,6 +486,7 @@ fn render_services_text_with_devices( .map(|record| (record.service_id.as_str(), record.engine.as_str())) .collect(), ); + refresh_published_anthropic_endpoints(&mut records); let _ = writeln!(output, "Servers"); for record in &records { @@ -538,6 +539,49 @@ fn render_services_text_with_devices( Ok(output) } +/// Re-read each live record's published Anthropic endpoint from the engine +/// state file, after the device probe and before anything is printed. +/// +/// [`render_services_text_with_devices`] snapshots every record once, up front, +/// via `load_managed_services`. That was a local read, so the snapshot was +/// effectively taken at print time. Since ROCMAI-550 the device probe sits +/// between the two and can spend seconds (up to +/// `engine_ipc::SERVICE_DEVICE_PROBE_BUDGET`) on live `Healthcheck` RPCs, and +/// the serving process publishes the Anthropic endpoint at roughly the moment +/// the engine first lists its model — the same moment the device probe starts +/// getting an answer. So the gap this PR opened is exactly the window in which +/// a freshly ready service renders with no `anthropic endpoint:` row, from a +/// snapshot that predates the publication. +/// +/// `serve_cmd.rs`'s post-launch summary hit the same race and answers it with +/// `rocm_core::wait_for_published_anthropic_endpoint`; this is the same read, +/// with no wait. A wait is right there — that command has just launched the +/// server and is standing by for it — and wrong here: `rocm services list` +/// reports on whatever exists, and must not start blocking on a service that +/// has not published yet and may never. Reading once restores the timing the +/// listing had before the probe existed. +/// +/// Gated on the same statuses as `ManagedServiceRecord::refresh_from_engine_state`, +/// which is what put the field on the snapshot in the first place: for anything +/// else the state file is a finished run's leftovers and the record's own value +/// is the one to keep. +fn refresh_published_anthropic_endpoints(records: &mut [ManagedServiceRecord]) { + for record in records { + if !matches!( + record.status.as_str(), + "starting" | "running" | "recovering" | "ready" + ) { + continue; + } + if let Some(url) = rocm_core::wait_for_published_anthropic_endpoint( + &record.engine_state_path, + Duration::ZERO, + ) { + record.anthropic_endpoint_url = Some(url); + } + } +} + /// The machine-readable counterpart to [`render_services_text`]. /// /// Applies the same liveness filter as the text form so `--json` and the table @@ -1903,6 +1947,101 @@ pub(crate) mod tests { Ok(()) } + /// The `anthropic endpoint:` row must survive the device probe's gap. + /// + /// `render_services_text_with_devices` snapshots every record once, up + /// front. That used to be a local read taken effectively at print time; + /// since the device probe went in between, the snapshot can be seconds old + /// by the time a row is written. The serving process publishes the + /// Anthropic endpoint to the engine state file at about the moment the + /// engine first lists its model — the same moment the probe starts getting + /// an answer — so that gap is precisely the window in which a freshly + /// ready service renders with no `anthropic endpoint:` row at all. + /// + /// The probe closure here *is* that window: it publishes the endpoint + /// while it runs. Rendering from the pre-probe snapshot therefore shows no + /// row and this test fails; only a re-read after the probe passes it. The + /// fixture cannot be "the endpoint is simply on the record", which the + /// stopped-service test above already covers and which passes either way. + #[test] + fn services_listing_shows_an_anthropic_endpoint_published_while_the_probe_ran() -> Result<()> { + use std::io::{Read, Write}; + use std::net::TcpListener; + + let (root, paths) = test_paths("services-anthropic-after-probe"); + paths.ensure()?; + let listener = TcpListener::bind(("127.0.0.1", 0))?; + let ready_port = listener.local_addr()?.port(); + let server = thread::spawn(move || -> Result<()> { + for _ in 0..2 { + let (mut stream, _) = listener.accept()?; + stream.set_read_timeout(Some(Duration::from_secs(2)))?; + let mut request = [0_u8; 512]; + let read = stream.read(&mut request)?; + let request = String::from_utf8_lossy(&request[..read]).into_owned(); + let body = if request.starts_with("POST /v1/chat/completions ") { + r#"{"choices":[{"message":{"content":"ok"}}]}"# + } else { + r#"{"data":[{"id":"Qwen/Qwen3.5"}]}"# + }; + write!( + stream, + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}", + body.len(), + body + )?; + } + Ok(()) + }); + let mut record = ManagedServiceRecord::new( + &paths, + "svc-anthropic-late", + "vllm", + "qwen", + "Qwen/Qwen3.5", + "127.0.0.1", + ready_port, + "managed", + std::process::id(), + Some("therock-release".to_owned()), + None, + Some("gpu_required".to_owned()), + ); + record.status = "ready".to_owned(); + record.write()?; + // Nothing is published when the snapshot is taken: no state file exists, + // so `refresh_from_engine_state` has nothing to adopt. + assert!(!record.engine_state_path.exists()); + let state_path = record.engine_state_path; + + let rendered = render_services_text_with_devices(&paths, false, |_ready| { + // What the serving process does while the probe is in flight. + if let Some(parent) = state_path.parent() { + fs::create_dir_all(parent).expect("engine state dir"); + } + fs::write( + &state_path, + r#"{"status": "ready", "anthropic_endpoint_url": "http://127.0.0.1:1", "anthropic_endpoint_mode": "proxy"}"#, + ) + .expect("publish the anthropic endpoint mid-probe"); + HashMap::new() + })?; + server + .join() + .expect("fake models server should not panic")?; + let _ = fs::remove_dir_all(root); + + // Port 1 is closed, so the row says "(not responding)". What is pinned + // is that the row exists and names the published URL — the five Strix + // Halo scenarios that regressed look for exactly this prefix. + assert!( + rendered.contains(" anthropic endpoint: http://127.0.0.1:1"), + "an endpoint published during the device probe must still reach the \ + row, not be lost to the pre-probe snapshot:\n{rendered}" + ); + Ok(()) + } + /// The default view lists only live servers. The counts used to be taken /// from that already-filtered list, so a host whose local servers had all /// failed was told "none ready" and shown an empty list - the records were From 4608e96df362ec99b366634e5531dd76da75d96b Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Mon, 16 Nov 2026 15:18:34 +0200 Subject: [PATCH 31/37] docs(services): scope the device-probe wait to the table, not `--json` `ServicesCommand::List`'s `--help` said each running server is asked which backend and GPU it is on, "so this takes a few seconds... about 18 seconds worst case", with no qualification. `--json` does none of that: `services_cmd.rs` routes it to `render_services_json`, which reads `load_managed_services` and serializes the stored `ManagedServiceRecord`s verbatim. There is no call into the probing path and no `device` field on the record to fill, so the claim was simply untrue of the form a script is likeliest to run. Scope the sentence to the default table and say what `--json` does instead. README.md carries the same paragraph and gets the same qualification. Signed-off-by: Roman Sirokov --- README.md | 4 +++- apps/rocm/src/cli.rs | 11 ++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 7376bb322f..a3cf826983 100644 --- a/README.md +++ b/README.md @@ -1530,7 +1530,9 @@ can take a few seconds rather than returning instantly. The query is capped in total — about 18 seconds worst case, however many servers are listed — and a server that does not answer in time simply has no `device` line rather than holding up the rest of the list. The same applies to asking about servers in -`rocm chat`, which prints the same list. +`rocm chat`, which prints the same list. `list --json` is unaffected: it emits +the stored service records, which have no `device` field, asks no server +anything, and returns as fast as it always did. `restart` starts a server again only once its stop has confirmed that every recorded process is gone; if the stop cannot confirm that, it says so, starts diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index 8a56b5eab9..9bbb3bf482 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -1185,11 +1185,12 @@ pub(crate) enum ServicesCommand { /// Show currently running local model servers, and count the records that /// are no longer running. /// - /// Each running server is asked which backend and GPU it is actually on, so - /// this takes a few seconds rather than returning instantly. The asking is - /// capped in total (about 18 seconds worst case, however many servers are - /// listed), and a server that does not answer in time simply has no device - /// line. + /// In the default table, each running server is asked which backend and GPU + /// it is actually on, so this takes a few seconds rather than returning + /// instantly. The asking is capped in total (about 18 seconds worst case, + /// however many servers are listed), and a server that does not answer in + /// time simply has no device line. `--json` asks nothing and carries no + /// device field: it is the stored records, returned as fast as before. List { /// Include failed, stopped, and old service records. #[arg(short, long)] From 074becf028b9abc47ba0886d3eca5c165a21e2a3 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Mon, 16 Nov 2026 15:19:13 +0200 Subject: [PATCH 32/37] fix(examine): stop an SDK for another family being called this GPU's MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `rocm_support_line`'s `RocmHip` arm printed "the managed ROCm SDK for {installed} is already installed - serve on it" whenever ANY managed runtime was on disk, without comparing it with what the detected GPU needs. The two fields are computed from unrelated facts and really can differ. `compatible_therock_family` is `normalize_therock_family` applied to the gfx target detected right now; `detect_managed_therock_family` walks the runtime registry and the engine manifests for the most recently installed TheRock family and never looks at the GPU at all — `managed_therock_family_uses_runtime_manifest_not_host_mapping` pins exactly that independence. A GPU swap, an eGPU, or a second card is enough to make the install on disk one this GPU was never built for, and the user was then told it was ready for their card. Gate the "already installed" branch on the two families matching and give the mismatch its own wording, which names both families and points at `rocm install sdk` — still an install with something left to do, so the advice can clear the condition it is printed for. The existing test planted `gfx110X-dgpu` against a gfx1103 GPU, whose compatible family is `gfx110X-all`: it was asserting the "already installed" sentence on a MISMATCHED pair, which is the bug. It now uses the GPU's own family, and `an_sdk_for_another_family_is_not_reported_as_ready_for_this_gpu` covers the mismatch. Verified it discriminates: with the family comparison removed the new test fails. The e2e step `the support level leaves a command that fits this host` expected `rocm serve ` and forbade `rocm install sdk` for any host with both a compatible family and an installed SDK, which would now fail against correct output on a mismatched host. It gains the matching condition and an arm for the mismatch. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/host_gpu/render.rs | 109 +++++++++++++++--- tests/e2e-cucumber/tests/e2e/examine_steps.rs | 14 ++- 2 files changed, 104 insertions(+), 19 deletions(-) diff --git a/crates/rocm-core/src/host_gpu/render.rs b/crates/rocm-core/src/host_gpu/render.rs index aec43dfbfb..d2de61a01f 100644 --- a/crates/rocm-core/src/host_gpu/render.rs +++ b/crates/rocm-core/src/host_gpu/render.rs @@ -55,19 +55,44 @@ impl ExamineSummary { // `detected_therock_family` is the installed managed runtime (see // `detect_managed_therock_family`), as opposed to // `compatible_therock_family`, which is what this GPU *could* run. - GpuSupportLevel::RocmHip => match self.detected_therock_family.as_deref() { - Some(installed) => format!( - " rocm_support: GPU detected ({}) with full ROCm/HIP support, and the \ - managed ROCm SDK for {installed} is already installed - serve on it with \ - `rocm serve `\n", - detected.unwrap_or("") - ), - None => format!( - " rocm_support: GPU detected ({}) with full ROCm/HIP support; \ - install the SDK for it with `rocm install sdk`\n", - detected.unwrap_or("") - ), - }, + // + // The two are computed from unrelated facts and really can differ. + // `compatible_therock_family` is `normalize_therock_family` applied + // to the gfx target detected right now; `detect_managed_therock_family` + // is the family of the most recently installed managed runtime on + // disk and never looks at the GPU at all. Swap the card, plug in an + // eGPU, or keep two cards and boot the other one, and the install + // on disk is for a family this GPU was never built for. Saying + // "already installed - serve on it" there would promise a serve + // this host cannot do, which is exactly the kind of unestablished + // claim ROCMAI-550 exists to remove, so the match is a condition of + // that sentence rather than an assumption behind it. + GpuSupportLevel::RocmHip => { + let compatible = self.compatible_therock_family.as_deref(); + match self.detected_therock_family.as_deref() { + Some(installed) if Some(installed) == compatible => format!( + " rocm_support: GPU detected ({}) with full ROCm/HIP support, and the \ + managed ROCm SDK for {installed} is already installed - serve on it with \ + `rocm serve `\n", + detected.unwrap_or("") + ), + // An install is present but for another family, so + // `rocm install sdk` still has something to do and is still + // advice that can clear the condition it is printed for. + Some(installed) => format!( + " rocm_support: GPU detected ({}) with full ROCm/HIP support, but the \ + managed ROCm SDK installed here is for {installed}, not {} - install the \ + one this GPU needs with `rocm install sdk`\n", + detected.unwrap_or(""), + compatible.unwrap_or("") + ), + None => format!( + " rocm_support: GPU detected ({}) with full ROCm/HIP support; \ + install the SDK for it with `rocm install sdk`\n", + detected.unwrap_or("") + ), + } + } // The command names `rocm services list` rather than the serve // summary on purpose. The summary reads the device once, right // after `ready`, and lemonade answers `unknown` until its own @@ -680,16 +705,18 @@ mod tests { // Same GPU, but the SDK it was being told to install is already there: // the advice has to be something that is still left to do (AGENTS.md - // wants a named command to be able to clear the condition). + // wants a named command to be able to clear the condition). The + // installed family must be the GPU's own - see + // `an_sdk_for_another_family_is_not_reported_as_ready_for_this_gpu`. let mut installed = supported; - installed.detected_therock_family = Some("gfx110X-dgpu".to_owned()); + installed.detected_therock_family = family; let rendered = installed.render_text(); assert!( rendered.contains("rocm_support: GPU detected (gfx1103) with full ROCm/HIP support"), "{rendered}" ); assert!( - rendered.contains("managed ROCm SDK for gfx110X-dgpu is already installed"), + rendered.contains("managed ROCm SDK for gfx110X-all is already installed"), "{rendered}" ); assert!( @@ -728,4 +755,54 @@ mod tests { "nor as supported: {rendered}" ); } + + /// A managed SDK on disk is not, by itself, an SDK for *this* GPU. + /// + /// `detected_therock_family` is the most recently installed managed runtime + /// (`detect_managed_therock_family` reads the registry and never looks at + /// the GPU); `compatible_therock_family` is normalised from the gfx target + /// detected right now. A GPU swap, an eGPU or a second card leaves them + /// pointing at different families, and the `already installed - serve on + /// it` sentence then promises a serve this host cannot do. + /// + /// The matching half is asserted by + /// `examine_render_reports_a_support_level_for_every_host`; only this one + /// fails if the two families stop being compared, because the wording of + /// the matching case is unchanged. + #[test] + fn an_sdk_for_another_family_is_not_reported_as_ready_for_this_gpu() { + let compatible = normalize_therock_family("gfx1103").expect("gfx1103 has a family"); + let mut host = examine_summary_with_gpu(Some("gfx1103"), Some(&compatible)); + // A real family, and not this GPU's: the gfx110X split between the + // discrete and the all-targets build is exactly the pair a user is + // likeliest to have both of. + host.detected_therock_family = Some("gfx110X-dgpu".to_owned()); + assert_ne!( + host.detected_therock_family.as_deref(), + Some(compatible.as_str()), + "precondition: the installed family is not the compatible one" + ); + + let rendered = host.render_text(); + let line = rendered + .lines() + .find(|line| line.trim_start().starts_with("rocm_support:")) + .expect("every host gets a support line") + .to_owned(); + + assert!( + !line.contains("already installed"), + "an SDK built for another family must not be reported as this GPU's: {line}" + ); + assert!( + line.contains("gfx110X-dgpu") && line.contains(compatible.as_str()), + "the line has to name both families, or the user cannot see which \ + one is missing: {line}" + ); + assert!( + line.contains("rocm install sdk"), + "there is still an install left to do, so the advice must name the \ + command that does it: {line}" + ); + } } diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index 41285d1810..b8845ce883 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -3307,9 +3307,17 @@ async fn assert_support_level_advice_fits(world: &mut E2eWorld) { .filter(|value| *value != "" && *value != ""); let (expected, forbidden): (&str, &[&str]) = match (detected, family, installed_sdk) { - // ROCm-capable with the managed SDK already in place: installing one - // clears nothing, so the line must point at serving instead. - (_, Some(_), Some(_)) => ("rocm serve ", &["rocm install sdk"]), + // ROCm-capable with the managed SDK for THIS GPU's family already in + // place: installing one clears nothing, so the line must point at + // serving instead. + (_, Some(family), Some(installed)) if installed == family => { + ("rocm serve ", &["rocm install sdk"]) + } + // ROCm-capable with a managed SDK installed, but for another family — + // a GPU swap, an eGPU, or a second card. The install on disk was never + // built for this GPU, so the one it needs is still the next step and + // the line must not offer to serve on what is there. + (_, Some(_), Some(_)) => ("rocm install sdk", &["serve on it"]), // ROCm-capable with no managed SDK yet: that install is the next step. (_, Some(_), None) => ("rocm install sdk", &[]), // A GPU no package family covers. Two different tiers land here and From 4a651b5861802b9d905763c603cb49579ab657ff Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 17 Nov 2026 10:58:23 +0200 Subject: [PATCH 33/37] fix(serve): stop the engine's post-spawn state rewrite dropping the Anthropic endpoint The Anthropic proxy is a thread of the engine's own process, and it publishes exactly once -- as soon as the server lists a model. The engine adapter rewrites the same state file *whole* immediately after it spawns that server, and since ROCMAI-550 it spends a host GPU probe building the document first (`pinned_gpu_name`: pnputil plus a PowerShell CIM query on Windows, seconds). The publication lands inside that window and is replaced, and because nothing publishes twice the `anthropic endpoint:` row never comes back -- five Windows e2e scenarios saw a ready service with a working `device:` row and no endpoint. The file lock main added in ROCMAI-453 orders the two writers but does not stop the later one replacing what the earlier recorded; that is what a full rewrite is for. `publish_anthropic_endpoint` now notes the publication for this process before it touches the file, and `write_json_state_file` carries that note into the document it is about to write. Scoped to this process and this state file, so a dead endpoint a previous run left on disk is still cleared by the new run's first write, which `reset_for_restart` relies on. Also puts the engine state file into the e2e failure message: the row being absent because nothing published and the row being absent because something replaced it look identical in `services list`, and telling them apart cost a Windows lane run per guess. Signed-off-by: Roman Sirokov Signed-off-by: Roman Sirokov --- apps/rocmd/src/anthropic_proxy.rs | 27 +- crates/rocm-core/src/anthropic_endpoint.rs | 243 +++++++++++++++++- crates/rocm-core/src/endpoint_readiness.rs | 19 +- crates/rocm-core/src/lib.rs | 2 +- engines/lemonade/src/state.rs | 83 ++++++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 60 ++++- 6 files changed, 405 insertions(+), 29 deletions(-) diff --git a/apps/rocmd/src/anthropic_proxy.rs b/apps/rocmd/src/anthropic_proxy.rs index 73df2dceb3..51937bb6db 100644 --- a/apps/rocmd/src/anthropic_proxy.rs +++ b/apps/rocmd/src/anthropic_proxy.rs @@ -46,9 +46,8 @@ use axum::response::{IntoResponse, Response}; use axum::routing::{get, post}; use axum::{Json, Router}; use rocm_core::{ - ANTHROPIC_ENDPOINT_MODE_STATE_KEY, ANTHROPIC_ENDPOINT_URL_STATE_KEY, AnthropicEndpointMode, - JsonStateFileRefused, format_http_base_url, http_get_with_auth, http_post_json_with_auth, - merge_json_state_file, openai_models_endpoint_has_model, + AnthropicEndpointMode, JsonStateFileRefused, format_http_base_url, http_get_with_auth, + http_post_json_with_auth, openai_models_endpoint_has_model, publish_anthropic_endpoint, }; use serde_json::{Value, json}; @@ -153,11 +152,17 @@ const PUBLISH_RETRY_PAUSE: Duration = Duration::from_millis(200); /// Record the endpoint in the engine's state file. /// -/// `merge_json_state_file` refuses a state file it cannot merge into rather -/// than replace it. The vLLM and AIM engines still rewrite their state files -/// in place, so a file refused as empty or not valid JSON can be one caught -/// mid-write and whole again a moment later: that refusal is tried again, up -/// to [`PUBLISH_ATTEMPTS`] attempts [`PUBLISH_RETRY_PAUSE`] apart. Any other +/// Through `rocm_core::publish_anthropic_endpoint`, not a bare merge: the +/// engine adapter running in this same process rewrites that file *whole* a +/// moment after it spawns the server, and only a publication that function has +/// noted survives such a rewrite. See its documentation for why a bare merge +/// loses the endpoint for good. +/// +/// It refuses a state file it cannot merge into rather than replace it. The +/// vLLM and AIM engines still rewrite their state files in place, so a file +/// refused as empty or not valid JSON can be one caught mid-write and whole +/// again a moment later: that refusal is tried again, up to +/// [`PUBLISH_ATTEMPTS`] attempts [`PUBLISH_RETRY_PAUSE`] apart. Any other /// failure (a whole document that is not an object, or failing to lock, read /// or write the file) is returned at once. fn publish( @@ -178,13 +183,9 @@ fn publish_with_pause( ) -> anyhow::Result<()> { // The listener URL carries no `/v1`: Anthropic SDKs append `/v1/messages`. let base = url.trim_end_matches('/').trim_end_matches("/v1"); - let patch = json!({ - ANTHROPIC_ENDPOINT_URL_STATE_KEY: base, - ANTHROPIC_ENDPOINT_MODE_STATE_KEY: mode.as_str(), - }); let mut attempt = 1; loop { - let error = match merge_json_state_file(&config.state_path, &patch) { + let error = match publish_anthropic_endpoint(&config.state_path, base, mode) { Ok(()) => return Ok(()), Err(error) => error, }; diff --git a/crates/rocm-core/src/anthropic_endpoint.rs b/crates/rocm-core/src/anthropic_endpoint.rs index 744ae7d023..6d0314d33b 100644 --- a/crates/rocm-core/src/anthropic_endpoint.rs +++ b/crates/rocm-core/src/anthropic_endpoint.rs @@ -12,10 +12,21 @@ //! engine publishes the proxy's address under the state-file keys below, and [`crate::ManagedServiceRecord::refresh_from_engine_state`] adopts //! it, so every reader (`rocm serve`, `rocm services list`, the readiness probe) //! sees the same endpoint. +//! +//! The publisher is a *thread of the engine's own process*, not a separate +//! writer: `rocm`'s `run_builtin_engine_serve_http` starts the proxy and then +//! runs the engine adapter in the same process. Both write the one engine state +//! file, and the engine adapters write it whole. [`publish_anthropic_endpoint`] +//! and `carry_published_anthropic_endpoint` are the two halves that keep the +//! whole-file write from dropping what the sibling thread published. -use std::path::Path; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::{Mutex, OnceLock, PoisonError}; use std::time::{Duration, Instant}; +use anyhow::Result; + use crate::openai_models_endpoint_has_model; /// Engine state-file key naming the Anthropic-compatible endpoint's base URL. @@ -56,6 +67,117 @@ pub fn anthropic_endpoint_ready( .unwrap_or(false) } +/// What *this process* has published, keyed by the engine state file it went +/// into, so [`carry_published_anthropic_endpoint`] can put it back. +/// +/// Deliberately process-local rather than read back off disk. A previous run of +/// the same service id left its own endpoint in that file, and that URL is dead +/// — its proxy died with the process that bound the port. The first state write +/// of a new run is what clears it (`ManagedServiceRecord::reset_for_restart` +/// relies on exactly that), so a disk-backed carry-over would resurrect a dead +/// endpoint on every restart. Only the publication this process made is ours to +/// preserve. +/// +/// Keyed by path rather than held as a single value because the key is per +/// service: nothing today runs two engines in one process, but a map costs +/// nothing and a global would quietly cross-contaminate if one ever did (and +/// does cross-contaminate between tests sharing a process, which is the same +/// bug in miniature). +static PUBLISHED_ENDPOINTS: OnceLock>> = OnceLock::new(); + +fn published_endpoints() -> &'static Mutex> { + PUBLISHED_ENDPOINTS.get_or_init(|| Mutex::new(HashMap::new())) +} + +/// Record `base_url` as this service's Anthropic endpoint in `state_path`, and +/// remember it for the rest of this process. +/// +/// `base_url` carries no `/v1`: Anthropic SDKs append `/v1/messages` to it. +/// +/// The in-memory note is taken *before* the file is touched, and the note's +/// mutex is released before the file lock is asked for. Both matter: +/// +/// * noting first leaves no window — a whole-file rewrite that gets the file +/// lock first already sees the note and carries the endpoint, and one that +/// gets it after the merge below is carrying it too; +/// * releasing first keeps the two locks from ever being held in both orders, +/// since `carry_published_anthropic_endpoint` is called with the file lock +/// held. +/// +/// Called more than once for one endpoint when the merge has to be retried +/// (`rocmd`'s proxy retries a state file caught mid-write); the note is the +/// same each time. +pub fn publish_anthropic_endpoint( + state_path: &Path, + base_url: &str, + mode: AnthropicEndpointMode, +) -> Result<()> { + { + let mut published = published_endpoints() + .lock() + .unwrap_or_else(PoisonError::into_inner); + published.insert( + state_path.to_path_buf(), + (base_url.to_owned(), mode.as_str().to_owned()), + ); + } + crate::merge_json_state_file( + state_path, + &serde_json::json!({ + ANTHROPIC_ENDPOINT_URL_STATE_KEY: base_url, + ANTHROPIC_ENDPOINT_MODE_STATE_KEY: mode.as_str(), + }), + ) +} + +/// `state`, with the Anthropic endpoint this process published for `state_path` +/// put back into it — or `None` when there is nothing to put back, so the +/// caller writes the document it already has and pays no clone. +/// +/// [`crate::write_json_state_file`] is the only caller. A full rewrite replaces +/// every key, by design: it serializes a document its caller built rather than +/// patching the file, which is how a new run of a service clears what the +/// previous run left. The one thing it must not replace is a publication made +/// by *its own process*, because the publisher is a thread of that process +/// (`rocm`'s `run_builtin_engine_serve_http` starts the Anthropic proxy and +/// then runs the engine adapter) and nothing publishes a second time. +/// +/// This is not a theoretical race. The engine adapter's post-spawn `running` +/// write happens immediately after the server process is started, and since +/// ROCMAI-550 it spends a host GPU probe (`crate::pinned_gpu_name`: `pnputil` +/// plus a PowerShell CIM query on Windows, seconds) building the document +/// before writing it. That is long enough for the server to come up, for the +/// proxy's poll to see a listed model, and for the publication to land inside +/// the window and be replaced — leaving a ready service that never shows an +/// `anthropic endpoint:` row. +/// +/// The cross-process lock [`crate::merge_json_state_file`] and +/// [`crate::write_json_state_file`] share does not cover this. It orders the +/// two writers so neither sees a half-written file; it does not stop the +/// later one replacing what the earlier one recorded, which is exactly what a +/// full rewrite is for. +pub(crate) fn carry_published_anthropic_endpoint( + state_path: &Path, + state: &serde_json::Value, +) -> Option { + let (url, mode) = published_endpoints() + .lock() + .unwrap_or_else(PoisonError::into_inner) + .get(state_path) + .cloned()?; + let mut carried = state.clone(); + let object = carried.as_object_mut()?; + object.insert( + ANTHROPIC_ENDPOINT_URL_STATE_KEY.to_owned(), + serde_json::Value::String(url), + ); + object.insert( + ANTHROPIC_ENDPOINT_MODE_STATE_KEY.to_owned(), + serde_json::Value::String(mode), + ); + Some(carried) +} + /// The Anthropic endpoint the serving process has published in `state_path`, /// waiting up to `wait` for it to appear. /// @@ -120,4 +242,123 @@ mod tests { fn mode_strings_are_the_values_published_in_state() { assert_eq!(AnthropicEndpointMode::Proxy.as_str(), "proxy"); } + + /// The regression ROCMAI-550's Windows lane hit, driven through the real + /// writers: publish the endpoint, then have the engine adapter rewrite the + /// state file whole — which is what `write_running_state` does right after + /// it spawns the server, with a multi-second host GPU probe inside it. + /// + /// The rewritten document is deliberately *not* a copy of what is on disk: + /// a "fix" that merely re-read the file before writing would not be what + /// the adapters do, and would not satisfy this. + #[test] + fn a_whole_file_state_write_keeps_the_endpoint_this_process_published() { + let dir = unique_dir("carry"); + let state_path = dir.join("state.json"); + crate::write_json_state_file(&state_path, &serde_json::json!({ "status": "starting" })) + .unwrap(); + publish_anthropic_endpoint( + &state_path, + "http://127.0.0.1:41000", + AnthropicEndpointMode::Proxy, + ) + .unwrap(); + + crate::write_json_state_file( + &state_path, + &serde_json::json!({ "status": "running", "server_pid": 7 }), + ) + .unwrap(); + + let state: serde_json::Value = + serde_json::from_slice(&std::fs::read(&state_path).unwrap()).unwrap(); + assert_eq!( + state[ANTHROPIC_ENDPOINT_URL_STATE_KEY], "http://127.0.0.1:41000", + "an engine rewriting the whole state file must not drop the \ + endpoint its own process published: nothing publishes twice, so \ + the `anthropic endpoint:` row never comes back" + ); + assert_eq!(state[ANTHROPIC_ENDPOINT_MODE_STATE_KEY], "proxy"); + // The rewrite still replaces everything else, which is what it is for. + assert_eq!(state["status"], "running"); + assert_eq!(state["server_pid"], 7); + let _ = std::fs::remove_dir_all(dir); + } + + /// The carry is scoped to what *this* process published for *this* file. + /// + /// Both halves matter. A previous run of the same service left a dead + /// endpoint in that file — its proxy died with the process that bound the + /// port — and the new run's first whole-file write is what clears it, which + /// `ManagedServiceRecord::reset_for_restart` relies on; carrying anything + /// read off disk would resurrect it. And a process that published for one + /// service must not hand that endpoint to another. + #[test] + fn a_rewrite_still_clears_an_endpoint_this_process_did_not_publish() { + let dir = unique_dir("unpublished"); + let published_to = dir.join("published.json"); + let other = dir.join("other.json"); + publish_anthropic_endpoint( + &published_to, + "http://127.0.0.1:41001", + AnthropicEndpointMode::Proxy, + ) + .unwrap(); + std::fs::write( + &other, + r#"{"anthropic_endpoint_url": "http://127.0.0.1:1", "anthropic_endpoint_mode": "proxy"}"#, + ) + .unwrap(); + + crate::write_json_state_file(&other, &serde_json::json!({ "status": "starting" })).unwrap(); + + let state: serde_json::Value = + serde_json::from_slice(&std::fs::read(&other).unwrap()).unwrap(); + assert_eq!( + state, + serde_json::json!({ "status": "starting" }), + "only this process's own publication for this exact state file may \ + survive a whole-file write; a dead endpoint from a previous run \ + must not be restored" + ); + let _ = std::fs::remove_dir_all(dir); + } + + /// Publishing still writes the keys through to the file, which is what + /// every out-of-process reader (`rocm services list`, `rocm serve`'s + /// summary) actually looks at — the note is an addition, not a substitute. + #[test] + fn publishing_writes_the_endpoint_into_the_state_file_it_names() { + let dir = unique_dir("publish"); + let state_path = dir.join("state.json"); + std::fs::write(&state_path, r#"{"status": "running"}"#).unwrap(); + publish_anthropic_endpoint( + &state_path, + "http://127.0.0.1:41002", + AnthropicEndpointMode::Proxy, + ) + .unwrap(); + assert_eq!( + wait_for_published_anthropic_endpoint(&state_path, Duration::ZERO).as_deref(), + Some("http://127.0.0.1:41002") + ); + let state: serde_json::Value = + serde_json::from_slice(&std::fs::read(&state_path).unwrap()).unwrap(); + assert_eq!(state[ANTHROPIC_ENDPOINT_MODE_STATE_KEY], "proxy"); + // The merge keeps what was already recorded for the service. + assert_eq!(state["status"], "running"); + let _ = std::fs::remove_dir_all(dir); + } + + /// A directory no other test in this process shares: the publication note + /// is keyed by path and lives for the whole test binary. + fn unique_dir(tag: &str) -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!( + "rocm-core-anthropic-{tag}-{}-{}", + std::process::id(), + crate::unix_time_millis() + )); + std::fs::create_dir_all(&dir).unwrap(); + dir + } } diff --git a/crates/rocm-core/src/endpoint_readiness.rs b/crates/rocm-core/src/endpoint_readiness.rs index 52895a8bec..b19f9a95e9 100644 --- a/crates/rocm-core/src/endpoint_readiness.rs +++ b/crates/rocm-core/src/endpoint_readiness.rs @@ -124,6 +124,16 @@ fn merge_json_state_file_with( /// Takes the same lock as [`merge_json_state_file`] and publishes the same /// atomic way, so a full rewrite cannot land in the middle of another /// writer's merge. +/// +/// It replaces every key merged before it — that is what a full rewrite is for, +/// and how a new run of a service clears what the previous run recorded — with +/// one exception: an Anthropic endpoint *this process* published +/// (`crate::publish_anthropic_endpoint`) is carried into the document first. +/// The publisher is a thread of the same process as the engine adapter calling +/// this, it publishes exactly once, and since ROCMAI-550 the adapter's +/// post-spawn rewrite spends a multi-second host GPU probe building its +/// document — wide enough for the publication to land inside it and be lost. +/// See `anthropic_endpoint::carry_published_anthropic_endpoint`. pub fn write_json_state_file(path: &Path, value: &serde_json::Value) -> Result<()> { write_json_state_file_with(path, value, || {}) } @@ -137,8 +147,15 @@ fn write_json_state_file_with( before_publish: impl FnOnce(), ) -> Result<()> { let _lock = FileLock::acquire(json_state_lock_path(path))?; + // Read inside the file lock, which leaves no gap: a publisher notes the + // endpoint before it asks for this lock, so one that is waiting here has + // already been noted and is carried, and one that has not noted yet merges + // after this rewrite and lands on its own. `publish_anthropic_endpoint` + // releases its in-memory note before taking the file lock, so the two locks + // are never held in both orders. + let carried = crate::anthropic_endpoint::carry_published_anthropic_endpoint(path, value); before_publish(); - publish_json_state(path, value) + publish_json_state(path, carried.as_ref().unwrap_or(value)) } /// Why [`merge_json_state_file`] refused an existing state file. diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 56305bb8bb..31bdac110b 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -73,7 +73,7 @@ pub mod windows_driver; pub use amd_smi::{amd_smi_json, device_readable_bounded, resolve_amd_smi_binary}; pub use anthropic_endpoint::{ ANTHROPIC_ENDPOINT_MODE_STATE_KEY, ANTHROPIC_ENDPOINT_URL_STATE_KEY, AnthropicEndpointMode, - anthropic_endpoint_ready, wait_for_published_anthropic_endpoint, + anthropic_endpoint_ready, publish_anthropic_endpoint, wait_for_published_anthropic_endpoint, }; pub use automation::{ AuditEventRecord, AutomationEventRecord, AutomationProposalRecord, AutomationRuntimeState, diff --git a/engines/lemonade/src/state.rs b/engines/lemonade/src/state.rs index d35e2d5992..5a0bd94b86 100644 --- a/engines/lemonade/src/state.rs +++ b/engines/lemonade/src/state.rs @@ -150,6 +150,13 @@ pub(crate) fn read_service_state(path: &Path) -> Result { /// with every merge (`rocm_core::write_json_state_file`), so a full rewrite /// never lands inside another writer's merge, and publishes atomically, so a /// lock-free reader such as `rocm services list` never sees half a document. +/// +/// It replaces every key merged before it, with one exception that +/// `write_json_state_file` owns: an Anthropic endpoint published by this very +/// process is carried through. The proxy that publishes it is a thread of this +/// process and publishes exactly once, so the rewrite that happens moments +/// after the server is spawned would otherwise lose it for good — see +/// `a_running_state_write_keeps_the_anthropic_endpoint_this_process_published`. fn write_state(path: &Path, value: &Value) -> Result<()> { rocm_core::write_json_state_file(path, value) } @@ -310,6 +317,82 @@ mod tests { use std::io::{Read, Write}; use std::time::Duration; + /// ROCMAI-550 regression, on the real writer: the `running` state write + /// happens immediately after `serve_router` spawns the server, and since + /// ROCMAI-550 it spends a host GPU probe (`rocm_core::pinned_gpu_name`, + /// `pnputil` plus a PowerShell CIM query on Windows) building its document + /// before writing it. The Anthropic proxy `rocm` starts on a thread of this + /// same process publishes its endpoint as soon as the server lists a model + /// — inside that window — and nothing publishes twice, so a rewrite that + /// drops the endpoint drops it for the life of the service. That is what + /// left five Windows e2e scenarios with a ready service and no + /// `anthropic endpoint:` row. + /// + /// This drives the production call, not the primitive underneath it, so it + /// also holds the routing: a `write_state` that went back to a plain + /// `fs::write` would fail here while `rocm-core`'s own tests stayed green. + #[test] + fn a_running_state_write_keeps_the_anthropic_endpoint_this_process_published() { + let dir = scratch_dir("anthropic-endpoint-survives-rewrite"); + let request = ServeHttpRequest { + service_id: "svc".to_owned(), + model_ref: "Qwen3-0.6B-GGUF".to_owned(), + host: "127.0.0.1".to_owned(), + port: 11435, + device_policy: rocm_engine_protocol::DevicePolicy::GpuRequired, + gpu_indices: Vec::new(), + runtime_id: None, + env_id: None, + state_path: dir.join("svc.json"), + log_path: None, + engine_recipe: None, + }; + let runtime = LemonadeRuntime { + manifest: LemonadeInstallManifest { + env_id: "test".to_owned(), + version: "test".to_owned(), + runtime_dir: dir.join("runtime"), + lemond: dir.join("lemond"), + lemonade: dir.join("lemonade"), + backend_recipe: crate::LLAMACPP_RECIPE.to_owned(), + backend_name: "rocm".to_owned(), + installed_at_unix_ms: 0, + }, + }; + + // The serve's own first write, then the proxy thread's publication. + write_running_state(&request, &runtime, std::process::id(), None, "starting").unwrap(); + rocm_core::publish_anthropic_endpoint( + &request.state_path, + "http://127.0.0.1:41000", + rocm_core::AnthropicEndpointMode::Proxy, + ) + .unwrap(); + + // The post-spawn rewrite, which replaces the whole document. + write_running_state( + &request, + &runtime, + std::process::id(), + Some(4242), + "running", + ) + .unwrap(); + + let state = read_service_state(&request.state_path).unwrap(); + assert_eq!( + state["anthropic_endpoint_url"], "http://127.0.0.1:41000", + "the rewrite dropped the endpoint its own process published, so \ + `rocm services list` will never show an `anthropic endpoint:` row \ + for this service: {state}" + ); + assert_eq!(state["anthropic_endpoint_mode"], "proxy"); + // The rewrite still does its own job. + assert_eq!(state["status"], "running"); + assert_eq!(state["server_pid"], 4242); + fs::remove_dir_all(dir).ok(); + } + #[test] fn identity_from_state_prefers_server_pid_and_its_start_ticks() { let state = json!({ diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 844631e023..56cd3c4b55 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -2648,12 +2648,42 @@ async fn listed_anthropic_endpoint(world: &mut E2eWorld) -> Option { } } +/// What a missing `anthropic endpoint:` row has to be explained by: the +/// listing, and the engine state file the row is read out of. +/// +/// The row is absent for two very different reasons — the serving process never +/// published, or something wrote the file whole and replaced the key — and the +/// listing alone cannot tell them apart. That cost a round trip to the Windows +/// GPU lane per guess while ROCMAI-550's regression was being chased, so the +/// file goes in the failure message. `--json` is the records verbatim, so it +/// carries `engine_state_path`; it also asks no device probe, so reading it +/// costs nothing a failing step cannot afford. +fn missing_anthropic_endpoint_evidence(world: &mut E2eWorld) -> String { + let listing = world.cli_output.clone().unwrap_or_default(); + let (records, _, _) = crate::run_rocm(world, &["services", "list", "--json"]); + let states = serde_json::from_str::(&records) + .ok() + .and_then(|records| records.as_array().cloned()) + .unwrap_or_default() + .iter() + .filter_map(|record| record.get("engine_state_path")?.as_str().map(str::to_owned)) + .map(|path| { + let body = std::fs::read_to_string(&path) + .unwrap_or_else(|error| format!("")); + format!("{path}:\n{body}") + }) + .collect::>() + .join("\n"); + format!("{listing}\nservices list --json:\n{records}\nengine state:\n{states}") +} + #[then("the service details include an Anthropic-compatible messages endpoint")] async fn assert_service_lists_anthropic_endpoint(world: &mut E2eWorld) { let listed = listed_anthropic_endpoint(world).await; - let output = world.cli_output.clone().unwrap_or_default(); - let endpoint = listed - .unwrap_or_else(|| panic!("no `anthropic endpoint:` row in `services list`:\n{output}")); + let Some(endpoint) = listed else { + let evidence = missing_anthropic_endpoint_evidence(world); + panic!("no `anthropic endpoint:` row in `services list`:\n{evidence}"); + }; assert!( endpoint.starts_with("http://") && !endpoint.contains("/v1"), "the Anthropic endpoint is a base URL clients append `/v1/messages` to, got `{endpoint}`" @@ -2674,9 +2704,11 @@ async fn assert_service_still_lists_openai_endpoint(world: &mut E2eWorld) { /// POST `body` to the served model's Anthropic endpoint and return the JSON /// reply, whatever its status. async fn post_anthropic_messages(world: &mut E2eWorld, body: serde_json::Value) { - let base = listed_anthropic_endpoint(world) - .await - .expect("no Anthropic endpoint is listed for the served model"); + let listed = listed_anthropic_endpoint(world).await; + let Some(base) = listed else { + let evidence = missing_anthropic_endpoint_evidence(world); + panic!("no Anthropic endpoint is listed for the served model:\n{evidence}"); + }; let budget = Duration::from_secs(crate::inference_timeout_for(world)); let reply = tokio::spawn(async move { reqwest::Client::builder() @@ -2823,9 +2855,11 @@ async fn assert_anthropic_response_model_correct(world: &mut E2eWorld) { #[when("the user sends a streaming request to the Anthropic messages endpoint")] async fn user_sends_streaming_anthropic_messages_request(world: &mut E2eWorld) { - let base = listed_anthropic_endpoint(world) - .await - .expect("no Anthropic endpoint is listed for the served model"); + let listed = listed_anthropic_endpoint(world).await; + let Some(base) = listed else { + let evidence = missing_anthropic_endpoint_evidence(world); + panic!("no Anthropic endpoint is listed for the served model:\n{evidence}"); + }; let budget = Duration::from_secs(crate::inference_timeout_for(world)); // Same budget and model name as the non-streaming request above. let body = serde_json::json!({ @@ -2961,11 +2995,11 @@ async fn assert_both_endpoints_healthy(world: &mut E2eWorld) { // `services list` probes each endpoint it shows and marks one that does not // answer with `(not responding)`; a healthy service carries neither mark. let listed = listed_anthropic_endpoint(world).await; + if listed.is_none() { + let evidence = missing_anthropic_endpoint_evidence(world); + panic!("no `anthropic endpoint:` row in `services list`:\n{evidence}"); + } let output = world.cli_output.clone().unwrap_or_default(); - assert!( - listed.is_some(), - "no `anthropic endpoint:` row in `services list`:\n{output}" - ); assert!( output.contains("status: ready"), "the OpenAI-compatible endpoint is not reported ready:\n{output}" From 017a109c2314c6e55a9293f53ba941a50f33ed70 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 17 Nov 2026 17:03:34 +0200 Subject: [PATCH 34/37] fix(engines): stop AIM claiming a GPU it was never given, and write state under the lock MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AIM's `reported_device` hardcoded `rocm` and probed the host for a GPU name whatever the launch's `device_policy` was, so a `cpu_only` container — which `resolve_use_gpu`/`docker::RunSpec` correctly give no /dev/kfd, no /dev/dri and no HIP_VISIBLE_DEVICES — still rendered e.g. `rocm (AMD Instinct MI300X)`. That is the invented device claim this branch exists to remove, shipped in compiled code even though `parse_device_policy` currently refuses `cpu` at the CLI. The policy the launch recorded now decides, and the probe is not made at all when no GPU was attached. AIM also recorded no `gpu_indices`, so on a multi-GPU host it answered a bare `rocm` where lemonade and vLLM answer `rocm gpu ()` for the same hardware — the engine does pin an ordinal, `plan_gpus` resolves it and `HIP_VISIBLE_DEVICES` narrows the container to it. `run_container` now reports the ordinals it attached so the state file and the container cannot disagree. Both engines' `write_state` were a lock-free `fs::write` that bypassed `carry_published_anthropic_endpoint`. The unlocked write predates this branch, but the slow host GPU probe that makes the race practically reachable is added to these two state writes by this branch itself, so the exposure is this branch's to close: routing them through `rocm_core::write_json_state_file` orders them against every merge and carries a sibling thread's publication through the rewrite, as lemonade's already does. The status writers patch through `merge_json_state_file` instead of rewriting the document whole. `healthcheck_service_at` is split out of both handlers, the way lemonade's already was, so a test can drive the real RPC entry point: every device test in the three engines called the private formatter, so reverting a handler's wiring to a hardcoded literal kept the suite green. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/anthropic_endpoint.rs | 8 + crates/rocm-core/src/endpoint_readiness.rs | 5 +- engines/aim/src/process.rs | 47 +++- engines/aim/src/state.rs | 302 ++++++++++++++++----- engines/vllm/src/state.rs | 136 ++++++++-- 5 files changed, 393 insertions(+), 105 deletions(-) diff --git a/crates/rocm-core/src/anthropic_endpoint.rs b/crates/rocm-core/src/anthropic_endpoint.rs index 6d0314d33b..93e8eb7a3d 100644 --- a/crates/rocm-core/src/anthropic_endpoint.rs +++ b/crates/rocm-core/src/anthropic_endpoint.rs @@ -19,6 +19,14 @@ //! file, and the engine adapters write it whole. [`publish_anthropic_endpoint`] //! and `carry_published_anthropic_endpoint` are the two halves that keep the //! whole-file write from dropping what the sibling thread published. +//! +//! The carry reaches a whole-file write only when that write goes through +//! [`crate::write_json_state_file`], which is +//! `carry_published_anthropic_endpoint`'s single caller. All three built-in +//! adapters' `write_state` do (`engines/lemonade`, `engines/vllm`, +//! `engines/aim`). An out-of-tree engine plugin that writes the state file +//! itself takes neither this nor the state directory's lock, and would drop +//! the publication. use std::collections::HashMap; use std::path::{Path, PathBuf}; diff --git a/crates/rocm-core/src/endpoint_readiness.rs b/crates/rocm-core/src/endpoint_readiness.rs index b19f9a95e9..63335b490a 100644 --- a/crates/rocm-core/src/endpoint_readiness.rs +++ b/crates/rocm-core/src/endpoint_readiness.rs @@ -70,8 +70,9 @@ pub fn json_state_lock_path(path: &Path) -> PathBuf { /// run under one cross-process lock ([`json_state_lock_path`]) and no merge /// can drop a key another merge recorded. The lock orders writers rather than /// combining them: a full rewrite through [`write_json_state_file`] still -/// replaces every key merged before it, and a writer that bypasses both -/// functions (the vLLM and AIM engines' own state writes) takes no lock at all. +/// replaces every key merged before it. Every built-in engine adapter's state +/// writes go through one of the two; a writer that bypasses both (an +/// out-of-tree engine plugin writing the file itself) takes no lock at all. /// /// The result is published atomically /// ([`crate::atomic_write::write_file_atomically`]), so a reader that takes no diff --git a/engines/aim/src/process.rs b/engines/aim/src/process.rs index b6460269f4..10681d5e69 100644 --- a/engines/aim/src/process.rs +++ b/engines/aim/src/process.rs @@ -164,7 +164,7 @@ pub(crate) fn launch_service(request: LaunchRequest) -> Result { let state_path = crate::state::state_path(&request.service_id); - let served_model_id = run_container( + let run = run_container( &request.service_id, &request.model_ref, &request.host, @@ -178,10 +178,11 @@ pub(crate) fn launch_service(request: LaunchRequest) -> Result { &state_path, &request.service_id, &request.model_ref, - served_model_id.as_deref(), + run.canonical_name.as_deref(), &request.host, request.port, device_policy, + &run.gpu_indices, engine_recipe.as_ref(), )?; @@ -246,8 +247,22 @@ fn plan_gpus(requested: &[u32], usable: Option>, windows: bool) -> Resu }) } +/// What [`run_container`] established about the container it left running. +#[derive(Debug, Default, PartialEq, Eq)] +struct ContainerRun { + /// The model name the image says it serves under, if known. + canonical_name: Option, + /// The GPU ordinals the container was narrowed to through + /// `HIP_VISIBLE_DEVICES`, for [`crate::state::write_running_state`] to + /// record so the `device:` row can name one (ROCMAI-550). Empty when no + /// GPU was attached, and when the container was already running — that + /// launch chose the ordinals and this call cannot know which. + gpu_indices: Vec, +} + /// Starts the named service's container unless it is already running, and -/// returns the model name the image says it serves under, if known. +/// reports the model name the image says it serves under together with the +/// GPUs the container was given. /// /// Container names are unique in Docker, so `docker run --name X` against a /// name still held by an exited container (a crash, a reboot, a failed @@ -261,15 +276,18 @@ fn run_container( device_policy: &DevicePolicy, requested_gpus: &[u32], extra_env: Option<&BTreeMap>, -) -> Result> { +) -> Result { let name = crate::docker::container_name(service_id); // Normalize once so the pull, the metadata and the run all use the same image. let image_ref = crate::registry::normalize_image_ref(model_ref); match crate::docker::container_state(&name)? { ContainerState::Running => { - return Ok(crate::registry::local_image_metadata(&image_ref) - .and_then(|metadata| metadata.canonical_name)); + return Ok(ContainerRun { + canonical_name: crate::registry::local_image_metadata(&image_ref) + .and_then(|metadata| metadata.canonical_name), + gpu_indices: Vec::new(), + }); } ContainerState::Exited(_) => crate::docker::remove(&name)?, ContainerState::Missing => {} @@ -313,6 +331,9 @@ fn run_container( if let Some(api_key) = rocm_engine_protocol::resolve_endpoint_api_key() { env.insert("VLLM_API_KEY".to_owned(), api_key); } + // The one place the attached ordinals are decided, so the container and the + // state file cannot disagree about them. + let gpu_indices = if use_gpu { gpus.indices } else { Vec::new() }; crate::docker::run(&crate::docker::RunSpec { name: &name, image: &image_ref, @@ -320,11 +341,14 @@ fn run_container( port, container_port: crate::DEFAULT_AIM_PORT, use_gpu, - gpu_indices: if use_gpu { &gpus.indices } else { &[] }, + gpu_indices: &gpu_indices, render_gid, env: &env, })?; - Ok(metadata.canonical_name) + Ok(ContainerRun { + canonical_name: metadata.canonical_name, + gpu_indices, + }) } /// Enforces `device_policy` instead of treating GPU use as a soft heuristic: @@ -357,7 +381,7 @@ fn resolve_use_gpu( pub(crate) fn serve_http(request: ServeHttpRequest) -> Result<()> { let name = crate::docker::container_name(&request.service_id); - let label_name = run_container( + let run = run_container( &request.service_id, &request.model_ref, &request.host, @@ -377,10 +401,11 @@ pub(crate) fn serve_http(request: ServeHttpRequest) -> Result<()> { &request.state_path, &request.service_id, &request.model_ref, - label_name.as_deref(), + run.canonical_name.as_deref(), &request.host, request.port, request.device_policy, + &run.gpu_indices, request.engine_recipe.as_ref(), )?; @@ -391,7 +416,7 @@ pub(crate) fn serve_http(request: ServeHttpRequest) -> Result<()> { &name, &request.host, request.port, - label_name.as_deref(), + run.canonical_name.as_deref(), aim_ready_timeout(), ) { Ok(served_model_id) => { diff --git a/engines/aim/src/state.rs b/engines/aim/src/state.rs index 500a4508b5..b7e1ac6077 100644 --- a/engines/aim/src/state.rs +++ b/engines/aim/src/state.rs @@ -17,8 +17,19 @@ const HEALTHCHECK_TIMEOUT_MS: u64 = 700; pub(crate) fn healthcheck_service(request: HealthcheckRequest) -> Result { require_nonempty(&request.service_id, "service_id")?; - let state_path = state_path(&request.service_id); - let state = read_service_state(&state_path).ok(); + healthcheck_service_at(&state_path(&request.service_id)) +} + +/// [`healthcheck_service`] against an explicit state path. +/// +/// Split out for the same reason `engines/lemonade`'s `healthcheck_service_at` +/// is: [`state_path`] resolves through the caller's real `AppPaths`, so the +/// RPC handler itself could not be driven from a test, and the `device` field +/// it answers with was only ever covered by calling [`reported_device`] +/// directly — a test that stays green if this handler stops consulting it at +/// all (see `healthcheck_rpc_answers_with_the_device_the_state_file_describes`). +pub(crate) fn healthcheck_service_at(state_path: &Path) -> Result { + let state = read_service_state(state_path).ok(); let endpoint_url = state .as_ref() @@ -42,7 +53,7 @@ pub(crate) fn healthcheck_service(request: HealthcheckRequest) -> Result Result String { + if recorded_device_policy(state) == Some(DevicePolicy::CpuOnly) { + return rocm_engine_protocol::format_reported_device("cpu", None, None); + } rocm_engine_protocol::format_reported_device( "rocm", - None, + rocm_engine_protocol::first_gpu_index_from_state(state), value_string(state, "gpu_name").as_deref(), ) } +/// The `device_policy` the launch recorded, or `None` when the state file +/// predates the key or holds something that is not one of the three policies. +fn recorded_device_policy(state: &Value) -> Option { + serde_json::from_value(state.get("device_policy")?.clone()).ok() +} + pub(crate) fn endpoint_response(request: EndpointRequest) -> Result { require_nonempty(&request.service_id, "service_id")?; let state = read_service_state(&state_path(&request.service_id)) @@ -131,9 +158,13 @@ pub(crate) fn write_running_state( host: &str, port: u16, device_policy: DevicePolicy, + gpu_indices: &[u32], engine_recipe: Option<&EngineRecipeHint>, ) -> Result<()> { let endpoint = endpoint_url(host, port); + // No GPU was attached, so there is no device to probe for or name. See + // [`reported_device`] for why this is gated rather than probed anyway. + let uses_gpu = device_policy != DevicePolicy::CpuOnly; // `extra_env` can carry secrets (e.g. an HF token passed via `--engine-env`); // this state file is status/introspection only (never read back to relaunch // the container, see `crate::process::launch_service`), so redact values @@ -152,16 +183,23 @@ pub(crate) fn write_running_state( "port": port, "endpoint_url": endpoint, "device_policy": device_policy, + // The ordinals `process::plan_gpus` resolved and `docker::RunSpec` + // narrowed the container to with `HIP_VISIBLE_DEVICES` — the same + // key, read the same way, as `engines/lemonade` and + // `engines/vllm`, so one multi-GPU host does not report + // `rocm gpu 1 (…)` for those two and a bare `rocm` for this one. + // Empty for a CPU-only launch, and for a container this call found + // already running (that launch pinned it, and this one cannot know + // what it chose). + "gpu_indices": gpu_indices, // The host's GPU name, so the `device:` row names a device and not - // only a backend (ROCMAI-550). This engine records no GPU ordinal, - // so without it its row would be the bare word `rocm` on every - // host. Captured once here rather than probed per healthcheck: - // `rocm services list` healthchecks every ready service on its - // render path. The empty ordinal list is the literal truth — the - // container is handed a device list, not an ordinal — and - // `rocm_core::pinned_gpu_name` still declines on any host where - // more than one AMD GPU could be the one that answered. - "gpu_name": rocm_core::pinned_gpu_name(&[]), + // only a backend (ROCMAI-550). Captured once here rather than + // probed per healthcheck: `rocm services list` healthchecks every + // ready service on its render path. `rocm_core::pinned_gpu_name` + // declines on any host where more than one AMD GPU could be the + // one that answered, and is not asked at all when no GPU was + // attached. + "gpu_name": uses_gpu.then(|| rocm_core::pinned_gpu_name(gpu_indices)).flatten(), "engine_recipe": engine_recipe, "started_at_unix_ms": current_unix_millis(), }), @@ -182,24 +220,26 @@ pub(crate) fn read_status(state_path: &Path) -> Option { value_string(&read_service_state(state_path).ok()?, "status") } +/// Patch one key into the service's state file under the state directory's +/// lock ([`rocm_core::merge_json_state_file`]) — the same lock the +/// inference-verification latch and `rocmd`'s Anthropic-endpoint publish take, +/// so no writer drops a key another writer recorded. +/// +/// It used to read, patch and `fs::write` with no lock at all, which could +/// drop either of those. A state file that is empty, torn or not a JSON object +/// is now an error and left as it was, rather than replaced wholesale. fn merge_state_field(state_path: &Path, key: &str, value: Value) -> Result<()> { - let mut state = read_service_state(state_path).unwrap_or_else(|_| json!({})); - if let Some(object) = state.as_object_mut() { - object.insert(key.to_owned(), value); - } - write_state(state_path, &state) + rocm_core::merge_json_state_file(state_path, &json!({ key: value })) } pub(crate) fn write_terminal_state(state_path: &Path, status: &str) -> Result<()> { - let mut state = read_service_state(state_path).unwrap_or_else(|_| json!({})); - if let Some(object) = state.as_object_mut() { - object.insert("status".to_owned(), json!(status)); - object.insert( - "stopped_at_unix_ms".to_owned(), - json!(current_unix_millis()), - ); - } - write_state(state_path, &state) + rocm_core::merge_json_state_file( + state_path, + &json!({ + "status": status, + "stopped_at_unix_ms": current_unix_millis(), + }), + ) } pub(crate) fn read_service_state(path: &Path) -> Result { @@ -208,22 +248,20 @@ pub(crate) fn read_service_state(path: &Path) -> Result { serde_json::from_str(&text).with_context(|| format!("failed to parse {}", path.display())) } -/// Serializes `value` to `path` as pretty JSON, creating the parent directory -/// if needed. The low-level primitive [`write_running_state`], -/// [`merge_state_field`] and [`write_terminal_state`] all go through — it is -/// not the only way this module's state files are written: -/// inference-verification latches go through -/// `rocm_core::merge_json_state_file` instead (see [`inference_verified`]), -/// which patches the file under the state directory's lock rather than -/// routing through here. This function takes no such lock and writes in -/// place, so the lock does not order it against those latches. +/// Replace the service's state file whole, under the state directory's lock +/// and published atomically ([`rocm_core::write_json_state_file`]). +/// +/// [`write_running_state`] is the only caller; the status writers patch +/// ([`merge_state_field`]) instead. It used to be a lock-free `fs::write`, +/// which is a problem this PR's own change made reachable: the document being +/// written now includes a host GPU probe (`rocm_core::pinned_gpu_name`, up to +/// seconds on Windows), and `rocmd`'s Anthropic-proxy thread — a thread of +/// *this* process — publishes its endpoint into the same file in that window. +/// Going through `write_json_state_file` both orders this against every merge +/// and carries that publication through the rewrite, the way +/// `engines/lemonade`'s `write_state` does. fn write_state(path: &Path, value: &Value) -> Result<()> { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .with_context(|| format!("failed to create {}", parent.display()))?; - } - fs::write(path, serde_json::to_vec_pretty(value)?) - .with_context(|| format!("failed to write {}", path.display())) + rocm_core::write_json_state_file(path, value) } pub(crate) fn endpoint_url(host: &str, port: u16) -> String { @@ -389,6 +427,7 @@ mod tests { "127.0.0.1", 8000, DevicePolicy::GpuPreferred, + &[0], Some(&hint), ) .unwrap(); @@ -410,8 +449,8 @@ mod tests { /// is covered by /// `healthcheck_reports_unknown_for_a_service_with_no_state_file`). /// - /// Reverting the string to `"rocm_gpu"`, or dropping the `gpu_name` field - /// from the state write, fails this. + /// Reverting the string to `"rocm_gpu"`, dropping the `gpu_name` field or + /// dropping the `gpu_indices` field from the state write fails this. #[test] fn reported_device_for_existing_state_matches_the_shared_engine_shape() { let dir = tempfile::tempdir().unwrap(); @@ -424,42 +463,179 @@ mod tests { "127.0.0.1", 8000, DevicePolicy::GpuRequired, + &[1], None, ) .unwrap(); // The probed name is whatever the test host has, so assert the shape - // rather than a value: `rocm`, optionally followed by `()`, and - // never the old `rocm_gpu` or a `gpu` noun with no ordinal behind it. + // rather than a value: `rocm gpu 1`, optionally followed by + // `()`, and never the old `rocm_gpu`. let state = read_service_state(&path).unwrap(); let device = reported_device(&state); assert!( - device == "rocm" || (device.starts_with("rocm (") && device.ends_with(')')), - "AIM must report the shared ` [()]` shape: {device}" + device == "rocm gpu 1" || (device.starts_with("rocm gpu 1 (") && device.ends_with(')')), + "AIM must report the shared ` gpu [()]` shape, with the \ + ordinal the container was narrowed to: {device}" ); assert!( !device.contains("rocm_gpu"), "the pre-ROCMAI-550 literal must not come back: {device}" ); - assert!( - !device.split_whitespace().any(|word| word == "gpu"), - "this engine pins no ordinal, so a bare `gpu` noun would name \ - nothing: {device}" - ); - // The state file must actually carry the key the row is built from, + // The state file must actually carry both keys the row is built from, // even when the host probe found no name. assert!( state.get("gpu_name").is_some(), "write_running_state must record gpu_name: {state}" ); + assert_eq!( + state.get("gpu_indices"), + Some(&json!([1])), + "write_running_state must record the ordinals the container was \ + given, the way lemonade and vLLM do: {state}" + ); // And the name, when the host has one, reaches the row. + assert_eq!( + reported_device(&json!({ "gpu_indices": [0], "gpu_name": "AMD Instinct MI300X" })), + "rocm gpu 0 (AMD Instinct MI300X)" + ); + } + + /// A `cpu_only` launch attaches no GPU at all (`process::resolve_use_gpu` + /// returns `false`, and `docker::RunSpec` then passes no `/dev/kfd`, no + /// `/dev/dri` and no `HIP_VISIBLE_DEVICES`), so neither the `rocm` backend + /// nor a GPU name is a fact about it. Before ROCMAI-550's fix this path + /// still probed the host and rendered e.g. + /// `rocm (AMD Instinct MI300X)` for a container that used no GPU — the + /// invented claim the whole ticket exists to remove. + /// + /// Drops to red if `write_running_state` probes for a name regardless of + /// the policy, or if `reported_device` goes back to hardcoding `rocm`. + #[test] + fn a_cpu_only_launch_claims_neither_rocm_nor_a_gpu_name() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("svc-cpu.json"); + write_running_state( + &path, + "svc-cpu", + "some/model", + Some("Org/Served"), + "127.0.0.1", + 8000, + DevicePolicy::CpuOnly, + &[], + None, + ) + .unwrap(); + + let state = read_service_state(&path).unwrap(); + assert_eq!( + state.get("gpu_name"), + Some(&Value::Null), + "a cpu-only launch must not have the host GPU probed for it: {state}" + ); + assert_eq!(reported_device(&state), "cpu"); + + // And the rendering is driven by the recorded policy, not by the + // absence of a name: a state file that somehow carries both still + // reports `cpu`. + assert_eq!( + reported_device(&json!({ + "device_policy": "cpu_only", + "gpu_indices": [0], + "gpu_name": "AMD Instinct MI300X", + })), + "cpu" + ); + // A state file written before the key existed keeps the old reading. assert_eq!( reported_device(&json!({ "gpu_name": "AMD Instinct MI300X" })), "rocm (AMD Instinct MI300X)" ); } + /// The `device` the Healthcheck RPC *itself* answers with, not the private + /// formatter it is supposed to call. + /// + /// Every other device test in the three engines calls `reported_device` + /// directly, so reverting the handler's `Some(state) => reported_device` + /// wiring to a hardcoded literal kept the whole suite green. This drives + /// the handler. + /// + /// The planted state carries no endpoint, so no HTTP probe is made and + /// `listed`/`ready` are false — which is the point: this engine reports + /// the device as soon as it has state, independently of readiness (see + /// `apps/rocm/src/engine_ipc.rs`'s `device_worth_reporting`). + #[test] + fn healthcheck_rpc_answers_with_the_device_the_state_file_describes() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("svc-rpc.json"); + write_state( + &path, + &json!({ + "service_id": "svc-rpc", + "status": "running", + "device_policy": "gpu_required", + "gpu_indices": [2], + "gpu_name": "AMD Instinct MI300X", + }), + ) + .unwrap(); + + let response = healthcheck_service_at(&path).unwrap(); + assert_eq!(response.device, "rocm gpu 2 (AMD Instinct MI300X)"); + assert_eq!(response.status, "running"); + assert!(!response.model_loaded); + + // And a service with no state on disk is the `"unknown"` the CLI + // filters out rather than prints. + let response = healthcheck_service_at(&dir.path().join("absent.json")).unwrap(); + assert_eq!(response.device, "unknown"); + } + + /// The post-spawn whole-file rewrite must not drop the Anthropic endpoint + /// a sibling thread of this same process published moments earlier. + /// + /// `rocm`'s `run_builtin_engine_serve_http` runs `rocmd`'s Anthropic proxy + /// as a thread of the process running this adapter, and ROCMAI-550 put a + /// host GPU probe (seconds, on Windows) inside the rewrite — wide enough + /// for the publish to land inside it. Routing [`write_state`] through + /// `rocm_core::write_json_state_file` is what carries it through; putting + /// the raw `fs::write` back drops the key and fails this. + #[test] + fn a_running_state_write_keeps_the_anthropic_endpoint_this_process_published() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("svc-anthropic.json"); + rocm_core::publish_anthropic_endpoint( + &path, + "http://127.0.0.1:9931", + rocm_core::AnthropicEndpointMode::Proxy, + ) + .unwrap(); + + write_running_state( + &path, + "svc-anthropic", + "some/model", + None, + "127.0.0.1", + 8000, + DevicePolicy::GpuRequired, + &[0], + None, + ) + .unwrap(); + + let state = read_service_state(&path).unwrap(); + assert_eq!( + value_string(&state, rocm_core::ANTHROPIC_ENDPOINT_URL_STATE_KEY).as_deref(), + Some("http://127.0.0.1:9931"), + "the whole-file rewrite dropped the endpoint the proxy thread \ + published: {state}" + ); + } + #[test] fn status_helpers_round_trip_and_keep_other_fields() { let dir = tempfile::tempdir().unwrap(); diff --git a/engines/vllm/src/state.rs b/engines/vllm/src/state.rs index 540a6dae44..7acaf5ae0f 100644 --- a/engines/vllm/src/state.rs +++ b/engines/vllm/src/state.rs @@ -34,7 +34,19 @@ pub(crate) struct ServiceFiles { pub(crate) fn healthcheck_service(request: HealthcheckRequest) -> Result { require_nonempty(&request.service_id, "service_id")?; let files = service_files(&request.service_id)?; - let state = read_service_state(&files.state_path).ok(); + healthcheck_service_at(&files.state_path) +} + +/// [`healthcheck_service`] against an explicit state path. +/// +/// Split out the way `engines/lemonade`'s `healthcheck_service_at` already is: +/// [`service_files`] resolves through the caller's real `AppPaths`, so the RPC +/// handler itself could not be driven from a test, and the `device` it answers +/// with was only covered by calling [`reported_device`] directly — a test that +/// stays green if this handler stops consulting it at all (see +/// `healthcheck_rpc_answers_with_the_device_the_state_file_describes`). +pub(crate) fn healthcheck_service_at(state_path: &Path) -> Result { + let state = read_service_state(state_path).ok(); let endpoint_url = state.as_ref().and_then(endpoint_url_from_state); let model_ref = state .as_ref() @@ -51,7 +63,7 @@ pub(crate) fn healthcheck_service(request: HealthcheckRequest) -> Result Result<()> { - let mut state = read_service_state(state_path).unwrap_or_else(|_| json!({})); - if let Some(object) = state.as_object_mut() { - object.insert("status".to_owned(), Value::String(status.to_owned())); - object.insert( - "stopped_at_unix_ms".to_owned(), - Value::from(current_unix_millis() as u64), - ); - } - write_state(state_path, &state) + rocm_core::merge_json_state_file( + state_path, + &json!({ + "status": status, + "stopped_at_unix_ms": current_unix_millis() as u64, + }), + ) } pub(crate) fn read_service_state(path: &Path) -> Result { @@ -223,25 +237,20 @@ pub(crate) fn read_service_state(path: &Path) -> Result { serde_json::from_str(&text).with_context(|| format!("failed to parse {}", path.display())) } -/// Serializes `value` to `path` as pretty JSON, creating the parent directory -/// if needed. The low-level primitive [`write_running_state`] and -/// [`write_terminal_state`] both go through — it is not the only way this -/// module's state files are written: inference-verification latches go -/// through `rocm_core::merge_json_state_file` instead (see -/// [`inference_verified`]), which patches the file under the state -/// directory's lock rather than routing through here. This function takes no -/// such lock and writes in place, so the lock does not order it against those -/// latches. +/// Replace the service's state file whole, under the state directory's lock +/// and published atomically ([`rocm_core::write_json_state_file`]). +/// +/// [`write_running_state`] is the only caller; the terminal status is a patch +/// ([`write_terminal_state`]). It used to be a lock-free `fs::write`, which is +/// a problem this PR's own change made reachable: the document being written +/// now includes a host GPU probe (`rocm_core::pinned_gpu_name`, up to seconds +/// on Windows), and `rocmd`'s Anthropic-proxy thread — a thread of *this* +/// process — publishes its endpoint into the same file in that window. Going +/// through `write_json_state_file` both orders this against every merge and +/// carries that publication through the rewrite, as `engines/lemonade`'s +/// `write_state` does. fn write_state(path: &Path, value: &Value) -> Result<()> { - if let Some(parent) = path.parent() { - fs::create_dir_all(parent) - .with_context(|| format!("failed to create {}", parent.display()))?; - } - fs::write( - path, - serde_json::to_vec_pretty(value).context("failed to serialize vLLM state")?, - ) - .with_context(|| format!("failed to write {}", path.display())) + rocm_core::write_json_state_file(path, value) } pub(crate) fn endpoint_url(host: &str, port: u16) -> String { @@ -769,4 +778,73 @@ mod tests { "rocm (AMD Instinct MI300X)" ); } + + /// The `device` the Healthcheck RPC *itself* answers with, not the private + /// formatter it is supposed to call. + /// + /// Every other device test in the three engines calls `reported_device` + /// directly, so reverting this handler's `Some(state) => reported_device` + /// wiring to the old hardcoded literal kept the whole suite green. This + /// drives the handler. + /// + /// The planted state carries no endpoint, so no HTTP probe is made and + /// `listed`/`ready` are false — which is the point: this engine reports + /// the device as soon as it has state, independently of readiness (unlike + /// `engines/lemonade`, see `apps/rocm/src/engine_ipc.rs`'s + /// `device_worth_reporting`). + #[test] + fn healthcheck_rpc_answers_with_the_device_the_state_file_describes() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("svc-rpc.json"); + write_state( + &path, + &json!({ + "status": "running", + "gpu_indices": [3], + "gpu_name": "AMD Instinct MI300X", + }), + ) + .unwrap(); + + let response = healthcheck_service_at(&path).unwrap(); + assert_eq!(response.device, "rocm gpu 3 (AMD Instinct MI300X)"); + assert_eq!(response.status, "running"); + assert!(!response.model_loaded); + + // And a service with no state on disk is the `"unknown"` the CLI + // filters out rather than prints. + let response = healthcheck_service_at(&dir.path().join("absent.json")).unwrap(); + assert_eq!(response.device, "unknown"); + } + + /// The post-spawn whole-file rewrite must not drop the Anthropic endpoint + /// a sibling thread of this same process published moments earlier. + /// + /// `rocm`'s `run_builtin_engine_serve_http` runs `rocmd`'s Anthropic proxy + /// as a thread of the process running this adapter, and ROCMAI-550 put a + /// host GPU probe (seconds, on Windows) inside the rewrite — wide enough + /// for the publish to land inside it. Routing [`write_state`] through + /// `rocm_core::write_json_state_file` is what carries it through; putting + /// the raw `fs::write` back drops the key and fails this. + #[test] + fn a_running_state_write_keeps_the_anthropic_endpoint_this_process_published() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("svc-anthropic.json"); + rocm_core::publish_anthropic_endpoint( + &path, + "http://127.0.0.1:9932", + rocm_core::AnthropicEndpointMode::Proxy, + ) + .unwrap(); + + write_state(&path, &json!({ "status": "running", "gpu_indices": [0] })).unwrap(); + + let state = read_service_state(&path).unwrap(); + assert_eq!( + value_string(&state, rocm_core::ANTHROPIC_ENDPOINT_URL_STATE_KEY).as_deref(), + Some("http://127.0.0.1:9932"), + "the whole-file rewrite dropped the endpoint the proxy thread \ + published: {state}" + ); + } } From 4e435f757ea130aa7f7ca552d282e218232f14f3 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 17 Nov 2026 17:04:18 +0200 Subject: [PATCH 35/37] test(core): reach the two branches the GPU classifier's own suite could not MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `gpu_support_level`'s `(_, Some(_)) => RocmHip` arm ignores the detected target on purpose — `resolve_family` asks it about a `--family` the user named, where there may be no detected target — but every call in the suite passed Some/Some, None/None or Some/None, so narrowing the arm to `(Some(_), Some(_))` was invisible. `gpu_name_for_single_gpu_host`'s ordinal guard was in the same position: the only test with a non-zero ordinal also supplied a host count of 2, which the check two lines below already declines on, so the guard could be deleted outright with the suite still green. Both mutations now turn it red. Signed-off-by: Roman Sirokov --- crates/rocm-core/src/gfx_target.rs | 18 ++++++++++++++++++ crates/rocm-core/src/host_gpu.rs | 23 ++++++++++++++++++++++- 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/crates/rocm-core/src/gfx_target.rs b/crates/rocm-core/src/gfx_target.rs index ecbdfb4b0a..a73e36e97d 100644 --- a/crates/rocm-core/src/gfx_target.rs +++ b/crates/rocm-core/src/gfx_target.rs @@ -1066,12 +1066,30 @@ mod tests { ); } + /// A covering TheRock family is the whole answer, which is why that arm + /// ignores `detected_gfx_target` — and `resolve_family` depends on it: + /// it asks this about a target named with `--family`, where there may be + /// no detected target at all. Until the `None`/`Some(_)` case below, every + /// call in this suite passed `Some`/`Some`, `None`/`None` or `Some`/`None`, + /// so the arm could be narrowed to `(Some(_), Some(_))` with nothing + /// noticing. #[test] fn gpu_support_level_is_rocm_hip_when_a_family_covers_it() { assert_eq!( gpu_support_level(Some("gfx90a"), Some("gfx90a")), GpuSupportLevel::RocmHip ); + assert_eq!( + gpu_support_level(None, Some("gfx90a")), + GpuSupportLevel::RocmHip + ); + // Even for a target this file's own table says has no ROCm build: the + // family the caller resolved is a statement about ROCm support, and a + // detected target cannot outvote it. + assert_eq!( + gpu_support_level(Some("gfx90c"), Some("gfx90a")), + GpuSupportLevel::RocmHip + ); } #[test] diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index 38ac668e56..b287351c06 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -513,7 +513,7 @@ pub fn pinned_gpu_name(gpu_indices: &[u32]) -> Option { // uses it: off Windows nothing is probed here at all, and on Windows the // count already needed it before any of the decisions below could run. // - // The cost, stated rather than implied (raised in review on #678): this + // The cost, stated rather than implied: this // deduplicates *within* a call and is **not** cached across calls. Each // engine's `write_running_state` calls it, and that runs more than once per // launch (`starting`, then `running`), so a Windows launch pays that many @@ -986,6 +986,27 @@ mod tests { ); } + /// The ordinal guard on its own, with the host count agreeing there is one + /// GPU — the disagreement its doc comment names (a `--gpu 1` that got past + /// validation on a host now reporting a single card). + /// + /// Every other case here supplies a count that already declines, so + /// deleting the ordinal branch left the whole suite green. This is the + /// only call that reaches it. + #[test] + fn a_pinned_ordinal_the_host_count_contradicts_withholds_the_name() { + let mut probed = false; + let name = gpu_name_for_single_gpu_host(&[1], Some(1), false, || { + probed = true; + Some("AMD Instinct MI300X".to_owned()) + }); + assert_eq!(name, None); + assert!( + !probed, + "an ordinal the host's own count contradicts must not be probed" + ); + } + #[test] fn a_single_unmasked_gpu_is_named() { for indices in [&[][..], &[0][..]] { From b8a064babc9ecca5d1592fcf622a9bdc86221a17 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 17 Nov 2026 17:05:02 +0200 Subject: [PATCH 36/37] test(e2e): say that the cross-surface device check cannot fire, and let it read both renderers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `assert_services_list_reports_device`'s comparison against `rocm serve`'s summary was dead twice over. `world.serve_summary` comes from `run_rocm_bounded`, which redirects stdout to a file, so `serve_cmd.rs`'s `summary_mode` is false and the CLI prints `print_managed_launch_plain`, which has no device row in any state; and `device_row_value` looked for `"device:"` while `render_summary` writes a padded label with no colon, so it could not have matched a summary even if one carried the row. The parser now reads both renderers' shapes (and still refuses the `device_policy:` row the serve plan prints), and the step says in full that on today's lanes the comparison does not run and the backend assertion above it is what discriminates — rather than letting the step's name imply coverage it has not got. Making it live means driving the vLLM and Lemonade per-PR serve canaries through a `TuiSession`, which is a change to those canaries' own harness. Signed-off-by: Roman Sirokov --- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 48 +++++++++++++++++-- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 8a2baa6950..b9c8e469ea 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -3682,11 +3682,29 @@ fn parse_device_row(value: &str) -> Option<(&str, Option)> { /// this step flake on timing rather than on behavior. const DEVICE_ROW_DEADLINE: Duration = Duration::from_secs(90); -/// The value of the first `device:` row in `text`, trimmed. +/// The value of the first row labelled `device` in `text`, trimmed. +/// +/// Two renderers, two shapes, and this has to read both: `services_text.rs` +/// writes `device: `, while `serve_summary.rs`'s `render_summary` +/// writes a column of padded labels and no colon at all +/// (` device ` — pinned by its own +/// `device_row_shown_only_when_known` test, which asserts +/// `starts_with("device")`). An earlier revision looked only for `"device:"`, +/// so it could never have matched a deployment summary even when one carried +/// the row. +/// +/// The label has to end where it ends: `rocm serve`'s non-interactive plan +/// prints a `device_policy:` row, and matching that here would compare a +/// policy against a device. fn device_row_value(text: &str) -> Option<&str> { - text.lines() - .find_map(|line| line.trim_start().strip_prefix("device:")) - .map(str::trim) + text.lines().find_map(|line| { + let rest = line.trim_start().strip_prefix("device")?; + let rest = rest.strip_prefix(':').unwrap_or(rest); + if !rest.starts_with(char::is_whitespace) { + return None; + } + Some(rest.trim()).filter(|value| !value.is_empty()) + }) } /// ROCMAI-550: `rocm services list` must show the backend/device the engine @@ -3706,7 +3724,11 @@ fn device_row_value(text: &str) -> Option<&str> { /// 3. it equals what the `rocm serve` deployment summary printed for the same /// service — *when* that summary printed a device row at all, which is a /// documented maybe rather than a promise (see the comment at that -/// comparison, and `serve_summary::DeploymentSummary::device`). +/// comparison, and `serve_summary::DeploymentSummary::device`). Facts 1 and +/// 2 are what discriminate on today's lanes; fact 3 does not run, because +/// the summary these scenarios capture is pipe-redirected and therefore the +/// row-less plain form. The comment at that comparison says so in full +/// rather than letting the name of this step imply coverage it has not got. /// The row is polled rather than read once: see [`DEVICE_ROW_DEADLINE`] for /// the window in which a correct implementation legitimately has no row yet. /// Running out of that window is an assertion naming what was last seen, not a @@ -3795,6 +3817,22 @@ async fn assert_services_list_reports_device(world: &mut E2eWorld) { // that scenario in model_serving.feature). What remains asserted is the // half that is a real invariant: the two surfaces read one engine over one // RPC, so where both speak they must say the same thing. + // + // **State plainly what this cannot do today.** On every current lane the + // comparison below does not run at all, and not for the reason above: + // `world.serve_summary` is `rocm serve`'s stdout captured by + // `run_rocm_bounded`, which redirects it to a file, so + // `serve_cmd.rs`'s `summary_mode` (`background && stdout().is_terminal()`) + // is false and the CLI takes `print_managed_launch_plain`, which renders no + // device row in any state. The device row is an interactive-summary + // feature. So this is a cross-check that is *armed* — `device_row_value` + // reads both renderers' shapes, and it fires for any Given that captures a + // summary through a PTY — but today nothing drives these two scenarios + // that way, and the backend assertion above is what actually discriminates + // for serve-05 and serve-07. Making it live means driving the vLLM and + // Lemonade per-PR serve canaries through a `TuiSession` (as serve-28/41/60 + // do for their mock launches), which is a change to the canaries' own + // harness and is tracked separately rather than smuggled in here. // And only when a summary was recorded at all. The backend assertion above // is this step's real invariant and has already run; the cross-surface // comparison is an extra that needs a Given which captured `rocm serve`'s From 5ba7eb2d7065270f776f6c69c2371c566d6743b0 Mon Sep 17 00:00:00 2001 From: Roman Sirokov Date: Tue, 17 Nov 2026 17:05:24 +0200 Subject: [PATCH 37/37] docs: correct the device-row gating, the gfx_target entry and the Lemonade serve flow `device_worth_reporting` claimed the `"unknown"`-until-ready gate was uniform and cited vLLM and Lemonade as sharing it. They do not: vLLM and AIM answer as soon as a state file exists, while Lemonade keeps saying `"unknown"` until its own inference probe returns. That difference is the whole reason `SERVE_DEVICE_PROBE_BUDGET` exists, so the comment now names it. The probe bound alongside it also cites AIM's own `HEALTHCHECK_TIMEOUT_MS`, which it relied on without naming. `docs/architecture.md`'s `gfx_target.rs` entry still described pure lookups after that file gained the whole `GpuSupportLevel` classifier; the gate checks presence, not accuracy, so nothing caught it. `docs/manual-testing.md` dropped `--foreground` from the Lemonade serve example but still called the result "that foreground server" 34 lines later, with no way to stop it. The prose now matches a backgrounded service and gives the stop command, as the parallel flow below it already does. Also drops review-thread references from source comments and a note to reviewers from docs/testing.md; each says the reason instead. Signed-off-by: Roman Sirokov --- apps/rocm/src/cli.rs | 6 +++--- apps/rocm/src/engine_ipc.rs | 31 ++++++++++++++++++++++--------- apps/rocm/src/therock/family.rs | 2 +- docs/architecture.md | 2 +- docs/manual-testing.md | 13 ++++++++++--- docs/testing.md | 9 ++++----- 6 files changed, 41 insertions(+), 22 deletions(-) diff --git a/apps/rocm/src/cli.rs b/apps/rocm/src/cli.rs index c67be7ff3a..45ace9b237 100644 --- a/apps/rocm/src/cli.rs +++ b/apps/rocm/src/cli.rs @@ -2732,9 +2732,9 @@ mod tests { /// Sibling of `every_help_text_in_the_command_tree_is_ascii`, for the same /// reason one level down: clap renders a doc comment verbatim, with no /// markdown processing anywhere in its pipeline, so `**every**` written in - /// a doc comment reaches the user's terminal as literal asterisks. Review - /// on #678 caught exactly that on `Examine`, which the ASCII test could - /// not see because `*` and `_` are ASCII. + /// a doc comment reaches the user's terminal as literal asterisks. That had + /// already happened on `Examine`, where the ASCII test could not see it + /// because `*` and `_` are ASCII. /// /// Only `**` and `__` are rejected, and only those: they have no honest /// use in plain help text. A single `*` does (`gfx110*`, a glob in an diff --git a/apps/rocm/src/engine_ipc.rs b/apps/rocm/src/engine_ipc.rs index 4954c4e04d..8a7858b480 100644 --- a/apps/rocm/src/engine_ipc.rs +++ b/apps/rocm/src/engine_ipc.rs @@ -329,8 +329,8 @@ pub(crate) fn run_builtin_engine_stdio(engine: &str) -> Result<()> { /// How long one call can take, since `rocm services list` makes one per ready /// service: [`engine_request`] itself imposes no deadline, so the bound comes /// from the engine. The three built-in engines answer in-process and every -/// endpoint probe they make is timeout-bounded (`engines/vllm`'s -/// `HEALTHCHECK_TIMEOUT_MS`, `engines/lemonade`'s +/// endpoint probe they make is timeout-bounded (`engines/vllm`'s and +/// `engines/aim`'s own `HEALTHCHECK_TIMEOUT_MS`, `engines/lemonade`'s /// `rocm_core::INFERENCE_PROBE_TIMEOUT`), so a stalled server costs seconds, /// not forever. An *external* plugin is a spawned child that /// `engine_request` waits on unbounded, exactly as every other engine call @@ -365,11 +365,24 @@ pub(crate) fn fetch_service_device( /// The engine's reported device, or `None` when it said nothing usable. /// -/// `"unknown"` is the engines' own device value when no state is on disk for -/// this service id yet (see `engines/vllm/src/state.rs` and -/// `engines/lemonade/src/lib.rs`), and it is not more informative than showing -/// nothing — a `device: unknown` row claims the CLI asked and learned -/// something, which is the opposite of what happened. +/// `"unknown"` is the engines' own "I cannot say yet", and it is not more +/// informative than showing nothing — a `device: unknown` row claims the CLI +/// asked and learned something, which is the opposite of what happened. +/// +/// *When* an engine stops saying it is not uniform, and that difference is why +/// [`SERVE_DEVICE_PROBE_BUDGET`] exists at all: +/// +/// * `engines/vllm` and `engines/aim` answer as soon as a state file exists +/// for the service id (`Some(state) => reported_device(state)`, +/// `None => "unknown"`), so the row is available from the launch write +/// onwards, readiness or not; +/// * `engines/lemonade` keeps answering `"unknown"` while a state file plainly +/// exists — its `healthcheck_service_at` reports a device only once the +/// service is `ready`, which there means its own inference probe has come +/// back. +/// +/// So this filter is what keeps the slowest of the three from printing a row +/// that says nothing; it is not a uniform no-state-on-disk guard. /// /// A named function rather than an inline `then_some` because the only caller /// is an RPC to a live engine: in default CI that call never succeeds, so the @@ -481,10 +494,10 @@ where /// `device:` row, so a bridge client calling the tool makes no probe and waits /// for nothing — unlike the sibling `engines` MCP tool, which ROCMAI-550 did /// make host-aware. The shared name is the trap, so this says which surface it -/// means (review on #678); giving the tool the row is a change in its own +/// means; giving the tool the row is a change in its own /// right, not something this constant may be read as already covering. /// -/// # The decision, recorded (review on #678) +/// # The decision, recorded /// /// This is accepted rather than merely observed. `rocm services list` and /// `rocm chat`'s `services` command can both block ~18s on a host whose diff --git a/apps/rocm/src/therock/family.rs b/apps/rocm/src/therock/family.rs index 99f09f44dc..aa7c3e250a 100644 --- a/apps/rocm/src/therock/family.rs +++ b/apps/rocm/src/therock/family.rs @@ -178,7 +178,7 @@ pub(super) fn resolve_family( /// /// Asked of [`gpu_support_level`] rather than re-tested here. The install path /// used to call `gfx_target_has_no_rocm_build` directly, which was *exactly* -/// equivalent (review on #678 checked that it was) and still wrong to keep: it +/// equivalent, and still wrong to keep: it /// made this a second, independent definition of the tier whose claim these /// messages then go on to make, so a change to the classification would have /// left `rocm examine` and `rocm install sdk` disagreeing about the same GPU diff --git a/docs/architecture.md b/docs/architecture.md index 1d2ac90756..36cbd50215 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -310,7 +310,7 @@ The library every first-party app and engine crate depends on. It holds host exa | [`fix/render.rs`](../crates/rocm-core/src/fix/render.rs) | Rendering the catalog for people: the listing, a recipe's plan and verify lines, the flag summary `rocm diagnose` shares, and the refusal of a diagnosis position as a fix-id. | flat | | [`fix/runners.rs`](../crates/rocm-core/src/fix/runners.rs) | The host-mutating runners, one per auto-applicable fix, each printing the exact change, honouring `--dry-run` and asking consent first. | flat | | [`foreign_temp_probe.rs`](../crates/rocm-core/src/foreign_temp_probe.rs) | Test probe that a writer publishing through `atomic_write` never touches another writer's temp file; built only for tests and the `test-support` feature. | flat | -| [`gfx_target.rs`](../crates/rocm-core/src/gfx_target.rs) | Pure gfx-target lookups: extracting a gfx token, mapping a target to its TheRock family, the AMD PCI-id and marketing-name tables, and kernel gfx-version decoding. | domain | +| [`gfx_target.rs`](../crates/rocm-core/src/gfx_target.rs) | Pure gfx-target lookups: extracting a gfx token, mapping a target to its TheRock family, the AMD PCI-id and marketing-name tables, and kernel gfx-version decoding. Also the `GpuSupportLevel` classifier (`gpu_support_level`, `gpu_support_level_for_host_gpu`, `VULKAN_ONLY_SUPPORT_CLAIM`) that turns those two facts into the one ROCm/Vulkan support claim `examine`, `install` and the engine notes all render, so no surface invents its own. | domain | | [`gguf.rs`](../crates/rocm-core/src/gguf.rs) | Reads a GGUF model's shape (layers, heads, head width, parameter count, data offset) from its header, never the weights. | domain | | [`gguf_fit.rs`](../crates/rocm-core/src/gguf_fit.rs) | Pure arithmetic on whether a GGUF model fits in GPU memory at a context size: weights, KV cache, compute buffer and reserve against VRAM. | domain | | [`gguf_source.rs`](../crates/rocm-core/src/gguf_source.rs) | Where a GGUF header comes from: a local file, or a Hugging Face file read by HTTP range requests. | domain | diff --git a/docs/manual-testing.md b/docs/manual-testing.md index fba91c457a..bc0d1b50d8 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -439,9 +439,16 @@ in `MSVCP140.dll`), the Visual C++ runtime on the machine is probably older than fix-21-msvc-redist-outdated` prints the update plan and changes nothing. After the update, restart Lemonade and repeat the serve command above. -With that foreground server stopped, check that a catalogued recipe reaches -Lemonade with a build it can load, from a short name alone — no `--engine`, no -quantization variant typed by hand: +That server is in the background — `rocm serve` returns to the shell once it is +ready — so stop it before continuing, or it keeps the GPU and port 11435: + +```powershell +rocm services stop --yes +``` + +With it stopped, check that a catalogued recipe reaches Lemonade with a build +it can load, from a short name alone — no `--engine`, no quantization variant +typed by hand: ```powershell rocm serve qwen-small --managed diff --git a/docs/testing.md b/docs/testing.md index 6101476fac..0a83fefc46 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -4458,11 +4458,10 @@ same report prints, so every lane checks the level its own hardware implies: cargo xtask e2e -- -n '(examine-55)' ``` -Vulkan-only here is inferred from the gfx table, not probed: this change has -three levels and this section describes three. Probing the host for a Vulkan -driver, and the further levels that evidence makes knowable, is separate work. -Written this way deliberately — a forward reference to an unmerged change reads -as false the moment this merges on its own. +Vulkan-only here is inferred from the gfx table, not probed: there are three +support levels and this section describes three. Probing the host for a Vulkan +driver, and the further levels that evidence would make knowable, is separate +work. ## Windows Tool Notes