From 1821ff134186cf344b8e49e109b96eb7cf264327 Mon Sep 17 00:00:00 2001 From: Roman Inflianskas Date: Tue, 17 Nov 2026 21:28:49 +0200 Subject: [PATCH] fix(examine): match rocminfo GPU readings by PCI address (ROCMAI-40) `rocm examine` on Linux paired the GPU agents `rocminfo` reports with the listed AMD GPUs by list position. The ROCm runtime can expose only some of a machine's GPUs (for example under ROCR_VISIBLE_DEVICES), and nothing guarantees the two lists share an order, so one card's model name, gfx target and is_apu verdict could land on another card. The rocminfo parser is fixed as well. Each agent ends with an `ISA Info:` block whose entries carry `Name:` lines of their own, and the parser read the last of them as the agent's name. That name is not a gfx target, so every GPU agent of a real rocminfo listing was dropped (shown with a captured ROCm 7.14 listing, now a test fixture). Before this change the rocminfo pass therefore had no effect on a real host. From this change on, its model names, gfx targets and integrated/discrete verdicts reach `rocm examine` output, and everything that reads it, for the first time. That is the widest effect of this change. Behaviour changes on Linux: - Each agent is matched to a listed GPU by its `BDFID` against the entry's PCI bus, device and function. One helper decodes both BDFID and the KFD topology's `location_id`, and refuses zero and values wider than 16 bits. A match that two entries in different PCI domains share is refused and noted. - An agent that prints no BDFID is never paired with an entry, by position or otherwise. Where any AMD GPU is listed, its reading is reported in a note and no entry takes it; where none is, every agent is listed as a GPU of its own. Whether any supported rocminfo omits BDFID is unconfirmed; the captured ROCm 7.14 listing prints it for every agent. - An agent that matches no entry is listed as a GPU of its own only when every AMD entry has an address; otherwise it is noted. Readings in excess of the entries at one address are noted rather than applied. - A rocminfo reading fills a gfx target only where the entry has none, because the runtime reports the ISA that HSA_OVERRIDE_GFX_VERSION names, and `rocm diagnose`'s override check needs the card's own target. As a result, on a host with lspci but no readable KFD topology, a wrong target guessed from the marketing-name table is never corrected. The previous code would have let rocminfo overwrite it, but because of the parser bug it never did so on a real host. - A KFD node at the same address replaces a target the PCI scan guessed from the name table, and the verdict that came with the guess; before, it only filled an empty target. - is_apu: a rocminfo agent's `Memory Properties:` line decides it when printed (`APU` means integrated, a line without it discrete), except that a target in an APU family stays integrated, because the "discrete" reading rests on one documentation sample. Without the line, an APU-family target makes a GPU integrated and any other target leaves the verdict as it was, so a kernel-only or sysfs-fallback entry with an APU-family target is now integrated and one with another target stays null rather than being called discrete. Covered by unit tests over the match's cases, the address decode, the verdict rules and the captured listing, and by scenarios examine-55 (a GPU hidden from the runtime keeps its own identity), examine-56 (an overridden runtime target does not replace the native one), examine-57 (the kernel's target replaces a name-table guess), examine-58 (the kernel's target fills an unnamed GPU and makes it integrated) and examine-59 (readings with no address are reported, not applied). Signed-off-by: Roman Inflianskas --- crates/rocm-core/Cargo.toml | 7 +- crates/rocm-core/src/examine.rs | 15 +- crates/rocm-core/src/examine/linux_gpus.rs | 1867 ++++++++++++++++- crates/rocm-core/src/host_gpu.rs | 1 + crates/rocm-core/src/host_gpu/linux.rs | 58 +- .../tests/fixtures/rocminfo_gfx1151_apu.txt | 257 +++ docs/host-probing.md | 8 +- docs/manual-testing.md | 26 + docs/testing.md | 17 + tests/e2e-cucumber/Cargo.toml | 5 +- tests/e2e-cucumber/features/examine.feature | 96 +- tests/e2e-cucumber/tests/e2e/examine_steps.rs | 473 +++++ 12 files changed, 2714 insertions(+), 116 deletions(-) create mode 100644 crates/rocm-core/tests/fixtures/rocminfo_gfx1151_apu.txt diff --git a/crates/rocm-core/Cargo.toml b/crates/rocm-core/Cargo.toml index f6b7d64c41..2510451bd7 100644 --- a/crates/rocm-core/Cargo.toml +++ b/crates/rocm-core/Cargo.toml @@ -16,9 +16,10 @@ workspace = true e2e-test-hooks = [] # Compiles test helpers so a dependent crate's tests can use them: the # `foreign_temp_probe` probe, `atomic_write::stage_owner_only_file_unpublished` -# (the leftover a killed owner-only store leaves) and, on Windows, -# `owner_only_file::owner_only_dacl_mismatch` (reads a file's DACL back). -# Enabled only from a `[dev-dependencies]` entry; never in a shipped build. +# (the leftover a killed owner-only store leaves), the examine scenarios' +# planted `lspci` lines (`rocm_core::examine::TEST_FIXTURE_LSPCI_LINE_*`) and, +# on Windows, `owner_only_file::owner_only_dacl_mismatch` (reads a file's DACL +# back). Enabled only from a `[dev-dependencies]` entry; never in a shipped build. test-support = [] [build-dependencies] diff --git a/crates/rocm-core/src/examine.rs b/crates/rocm-core/src/examine.rs index 805bcb9191..9b8dc423d0 100644 --- a/crates/rocm-core/src/examine.rs +++ b/crates/rocm-core/src/examine.rs @@ -42,10 +42,15 @@ pub(crate) use host_env::{probe_cpu_windows_with, record_cpu}; pub(crate) use library_copies::comgr_matches_runtime; use library_copies::probe_comgr; use linux_driver::{probe_devices, probe_modules, probe_secure_boot, probe_user}; -pub use linux_gpus::gfx_is_apu_family; use linux_gpus::{ GpuProbeSources, probe_gpus_after_lspci, probe_gpus_lspci, summarise_gpu_categories, }; +pub use linux_gpus::{KERNEL_TOPOLOGY_GPU_NAME, gfx_is_apu_family}; +#[cfg(any(test, feature = "test-support"))] +pub use linux_gpus::{ + TEST_FIXTURE_LSPCI_LINE_NAME_TABLE_READS_AS_GFX1103, + TEST_FIXTURE_LSPCI_LINE_NAME_TABLE_READS_AS_NOTHING, +}; use rocm_install_probe::probe_rocm_install; pub(crate) use windows::parse_dotted_version; use windows::{ @@ -54,8 +59,12 @@ use windows::{ use wsl::probe_wsl; pub use wsl::{parse_wsl_distro_list, probe_wsl_distro_from_host}; -/// A single GPU as enumerated by `lspci`/`rocminfo` (Linux) or the display -/// inventory (Windows). +/// A single GPU, as the examination enumerated it. +/// +/// On Linux from `lspci`, the kernel's KFD topology (or its sysfs fallback) +/// and `rocminfo`; on Windows from the display inventory. `is_apu` is `None` +/// when nothing said which package it is; see `examine::linux_gpus`'s +/// `apu_verdict_from_target` for the Linux rules. #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] pub struct Gpu { pub name: String, diff --git a/crates/rocm-core/src/examine/linux_gpus.rs b/crates/rocm-core/src/examine/linux_gpus.rs index 5cfa8220da..d57b3201a8 100644 --- a/crates/rocm-core/src/examine/linux_gpus.rs +++ b/crates/rocm-core/src/examine/linux_gpus.rs @@ -217,8 +217,73 @@ fn extract_lspci_name(line: &str) -> String { /// A placeholder, not a name: it says "AMD GPU, source known, model unknown". /// Both probes that can produce a GPU without a marketing name use it, and /// [`gpu_name_is_unknown`] reads it back, so it has to be one string rather -/// than three copies that could drift apart. -const KERNEL_TOPOLOGY_GPU_NAME: &str = "AMD GPU (from kernel topology)"; +/// than three copies that could drift apart. Public so the e2e scenario that +/// asserts a GPU has no model name (`examine-55`) reads this string rather +/// than a copy of it. +pub const KERNEL_TOPOLOGY_GPU_NAME: &str = "AMD GPU (from kernel topology)"; + +/// Test fixture, not production data: the `lspci -nn -D` line scenario +/// examine-57 plants. +/// +/// The examination's name table reads it as gfx1103, an +/// integrated GPU, which is what lets that scenario show a kernel target +/// replacing the name table's; the unit test +/// `the_pci_lines_examine_57_and_58_plant_read_as_their_scenarios_assume` +/// pins that reading on this very constant. Compiled only for tests and the +/// `test-support` feature, which the e2e suite enables from its +/// `[dev-dependencies]`. +#[cfg(any(test, feature = "test-support"))] +pub const TEST_FIXTURE_LSPCI_LINE_NAME_TABLE_READS_AS_GFX1103: &str = "0000:05:00.0 Display \ + controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Phoenix1 [1002:15bf] (rev c1)"; + +/// Test fixture, not production data: the `lspci -nn -D` line scenario +/// examine-58 plants. +/// +/// Its model name (`Device 7fff`) is one no table knows, so the +/// name table reads it as no target and not integrated, which is what lets +/// that scenario show a kernel target filling the gap. Pinned and gated like +/// [`TEST_FIXTURE_LSPCI_LINE_NAME_TABLE_READS_AS_GFX1103`]. +#[cfg(any(test, feature = "test-support"))] +pub const TEST_FIXTURE_LSPCI_LINE_NAME_TABLE_READS_AS_NOTHING: &str = "0000:c3:00.0 Display \ + controller [0380]: Advanced Micro Devices, Inc. [AMD/ATI] Device 7fff [1002:7fff] (rev c1)"; + +/// The APU verdict an entry takes from a gfx target that reached it from the +/// kernel or the ROCm runtime, given the verdict it already has. +/// +/// A target in an APU family makes the entry integrated. Anything else leaves +/// the verdict as it was, and an entry with none keeps none: "discrete" is +/// never concluded from a target, because [`gfx_is_apu_family`] does not list +/// every APU (gfx902 / gfx909, Raven and Picasso, are not on it), so a target +/// outside it is not evidence of a discrete card. It must not downgrade an APU +/// codename's `true` either. The kernel offers nothing better: the KFD node +/// properties carry no documented APU flag, so a kernel-only entry gets this +/// target rule alone. The ROCm runtime does state the package, in +/// `rocminfo`'s `Memory Properties:` line; [`parse_rocminfo_memory_properties`] +/// gives the evidence for how that line is read. +/// +/// The one rule for `is_apu`, by path (each site cites this table): +/// +/// | Path | Situation | `is_apu` | +/// |---|---|---| +/// | `lspci` scan | AMD row | from the name: `true` for an APU name or codename, else `false` | +/// | `lspci` scan | NVIDIA row | `false` | +/// | KFD membership | node target replaces a non-empty scan guess | `gfx_is_apu_family(node)`: the scan derived its verdict together with the guess, so the correction replaces both | +/// | KFD membership | node target fills an empty row target | this function | +/// | KFD membership | node with no parsed target, or the same target | unchanged | +/// | KFD membership | kernel-only entry (no row) | this function from `None` | +/// | sysfs fallback | no AMD GPU listed at all | this function from `None` | +/// | `rocminfo` | agent matched, flag `APU` | `true` (the runtime's own reading of the device, which an override does not change) | +/// | `rocminfo` | agent matched, flag without `APU`, final target in an APU family | `true`: the family list outranks the flag's "discrete" reading, whose evidence [`parse_rocminfo_memory_properties`] gives | +/// | `rocminfo` | agent matched, flag without `APU`, any other final target | `false` | +/// | `rocminfo` | agent matched, no `Memory Properties:` line | this function, on the target the entry ends with | +/// | `rocminfo` | agent listed as a GPU of its own | the same three flag rows on the agent's own target, or this function from `None` when it has no flag | +fn apu_verdict_from_target(current: Option, target: &str) -> Option { + if gfx_is_apu_family(target) { + Some(true) + } else { + current + } +} /// Whether this GPU still has no real model name. /// @@ -257,42 +322,245 @@ fn probe_gpus_rocminfo(e: &mut Examination) { apply_rocminfo_gpu_agents(e, &out); } -/// Fold a `rocminfo` reading into the GPU list, against caller-supplied output. +/// One GPU agent `rocminfo` listed. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +struct RocminfoGpuAgent { + /// The agent's `Name:`, which for a GPU is its gfx target. + gfx_target: String, + /// The agent's `Marketing Name:`, empty when it printed none. + marketing_name: String, + /// The `bus:device.function` its `BDFID:` decodes to, `None` when the line + /// is absent or zero. `rocminfo` prints no PCI domain. + bus_address: Option, + /// Whether the runtime reports the agent as integrated, from its + /// `Memory Properties:` line: `Some(true)` when the line carries `APU`, + /// `Some(false)` when the line is there without it, `None` when the agent + /// prints no such line. See [`parse_rocminfo_memory_properties`]. + is_apu: Option, +} + +/// The GPU agents in a `rocminfo` listing, in the order it printed them. /// -/// Split from the process launch the same way [`apply_kernel_gpu_membership`] is -/// split from the `/sys` read, and for the same reason: the interesting -/// behaviour here is how an agent is matched onto an existing entry, and a test -/// that had to run the real `rocminfo` could only assert it on a host with a -/// GPU. See `a_rocminfo_marketing_name_outranks_the_kernel_topology_placeholder`. -fn apply_rocminfo_gpu_agents(e: &mut Examination, out: &str) { - let mut gfx_targets: Vec<(String, String)> = Vec::new(); - let mut cur_name = String::new(); - let mut cur_marketing = String::new(); - let mut cur_is_gpu = false; +/// Each agent ends with an `ISA Info:` block whose `ISA N` entries carry a +/// `Name:` of their own (`amdgcn-amd-amdhsa--gfx1151`, then +/// `amdgcn-amd-amdhsa--gfx11-generic`). Those are not the agent's name. When +/// they were read as if they were, the last one replaced `gfx1151` with a +/// string that is no gfx target, and every GPU agent of a real listing was +/// dropped. So nothing after `ISA Info:` is read until the next agent starts. See +/// `a_real_rocminfo_listing_yields_its_gpu_agent`, which parses a captured one. +fn parse_rocminfo_gpu_agents(out: &str) -> Vec { + let mut agents = Vec::new(); + let mut current = RocminfoGpuAgent::default(); + let mut is_gpu = false; + let mut in_isa_info = false; + let mut finish = |agent: RocminfoGpuAgent, is_gpu: bool| { + if is_gpu && agent.gfx_target.starts_with("gfx") { + agents.push(agent); + } + }; for line in out.lines() { let s = line.trim(); if s.starts_with("Agent ") { - if cur_is_gpu && cur_name.starts_with("gfx") { - gfx_targets.push((cur_name.clone(), cur_marketing.clone())); - } - cur_name.clear(); - cur_marketing.clear(); - cur_is_gpu = false; + finish(std::mem::take(&mut current), std::mem::take(&mut is_gpu)); + in_isa_info = false; + } else if in_isa_info { + // The agent's ISA entries: nothing here describes the agent. + } else if s.starts_with("ISA Info:") { + in_isa_info = true; } else if let Some(rest) = s.strip_prefix("Name:") { - cur_name = rest.trim().to_owned(); + current.gfx_target = rest.trim().to_owned(); } else if let Some(rest) = s.strip_prefix("Marketing Name:") { - cur_marketing = rest.trim().to_owned(); + current.marketing_name = rest.trim().to_owned(); } else if let Some(rest) = s.strip_prefix("Device Type:") { - cur_is_gpu = rest.contains("GPU"); + is_gpu = rest.contains("GPU"); + } else if let Some(rest) = s.strip_prefix("BDFID:") { + current.bus_address = parse_rocminfo_bdfid(rest); + } else if let Some(rest) = s.strip_prefix("Memory Properties:") { + current.is_apu = Some(parse_rocminfo_memory_properties(rest)); } } - if cur_is_gpu && cur_name.starts_with("gfx") { - gfx_targets.push((cur_name, cur_marketing)); + finish(current, is_gpu); + agents +} + +/// Whether an agent's `Memory Properties:` value says it is integrated. +/// +/// `rocminfo` prints `APU` there for an agent whose +/// `HSA_AMD_AGENT_INFO_MEMORY_PROPERTIES` has the APU flag, and leaves the +/// value empty otherwise: the Strix Halo capture in this module's tests prints +/// `Memory Properties: APU` for its gfx1151 and an empty value for its +/// CPU and NPU agents, and the MI300X sample output in AMD's "Using rocminfo" +/// page () +/// prints an empty value for that discrete GPU. So a line present without +/// `APU` is read as the runtime's statement that the GPU is discrete. That +/// reading rests on that one documentation sample: no discrete GPU's output +/// was captured for this repository. It is why +/// [`apu_verdict_with_runtime_flag`] lets an APU-family target outrank it. +/// The runtime reads this from the device, not from the ISA, so +/// `HSA_OVERRIDE_GFX_VERSION` does not change it. A `rocminfo` that prints no such line says nothing, which the caller +/// keeps as `None`. +fn parse_rocminfo_memory_properties(value: &str) -> bool { + value.split_whitespace().any(|token| token == "APU") +} + +/// The `bus:device.function` a `rocminfo` `BDFID:` value names. +/// +/// `BDFID` is decimal and uses the kernel's `pci_dev_id()` encoding, the same +/// as the KFD topology's `location_id`, so both decode through +/// [`crate::host_gpu::pci_bus_device_function`] and cannot disagree about a +/// card. Zero, which `rocminfo` prints for a CPU agent, is no address. +fn parse_rocminfo_bdfid(value: &str) -> Option { + crate::host_gpu::pci_bus_device_function(value.trim().parse::().ok()?) +} + +/// Split a listed GPU's `domain:bus:device.function` address into its domain +/// and the `bus:device.function` that `rocminfo` can be compared with. +/// +/// Every address a Linux GPU entry carries has the domain: `lspci` runs with +/// `-D`, and the KFD topology's address is built with one. `None` for an +/// entry with no address, or one not in that shape, which is "address +/// unknown" rather than a guess at the domain. +fn split_pci_domain(pci_id: &str) -> Option<(&str, &str)> { + let (domain, bdf) = pci_id.split_once(':')?; + (!domain.is_empty() && bdf.contains(':')).then_some((domain, bdf)) +} + +/// Why a `rocminfo` agent could not be attributed to a listed GPU. +fn unattributed_agent_label(agent: &RocminfoGpuAgent, why: &str) -> String { + match &agent.bus_address { + Some(address) => format!("{} at bus address {address} ({why})", agent.gfx_target), + None => format!("{} ({why})", agent.gfx_target), } - if gfx_targets.is_empty() { - return; +} + +/// Write one `rocminfo` agent's reading onto the listed GPU it describes. +/// +/// The agent's target only fills an entry that has none. `rocminfo` inherits +/// `HSA_OVERRIDE_GFX_VERSION`, and ROCr then reports the ISA the override +/// names rather than the card's own: on a gfx1151 host, `rocminfo` run with +/// an 11.0.0 override lists the GPU as gfx1100. A target the entry already holds is the KFD node's +/// (see [`apply_kernel_gpu_membership`], which also replaces a name-table +/// guess with it) or, where the kernel offered none, the PCI scan's guess from +/// the marketing name. Neither is replaced by a runtime reading an override +/// can spoof. That is a deliberate tradeoff with a residual: on a host with +/// `lspci` but no readable KFD topology (some containers), a wrong name-table +/// guess is never corrected, because from here a right runtime reading and an +/// overridden one look alike. The test +/// `rocminfo_never_replaces_a_name_table_guess_even_without_a_kernel_node` +/// pins that. +/// What it buys: `gpus[].gfx_target` is never an override's ISA laid over +/// a held target, so `rocm diagnose`'s override check holds the entry's own +/// target against the framework's arch list. The APU verdict is decided by +/// [`apu_verdict_with_runtime_flag`] on the target the entry ends with, never +/// on a spoofed agent target laid over a held one. +fn apply_agent_to_gpu(gpu: &mut Gpu, agent: RocminfoGpuAgent) { + if gpu.gfx_target.is_empty() { + gpu.gfx_target = agent.gfx_target; + } + gpu.is_apu = apu_verdict_with_runtime_flag(agent.is_apu, gpu.is_apu, &gpu.gfx_target); + if !agent.marketing_name.is_empty() && gpu_name_is_unknown(&gpu.name) { + gpu.name = agent.marketing_name; + } +} + +/// The APU verdict for an entry a `rocminfo` agent reached, from the agent's +/// `Memory Properties:` flag, the verdict the entry already has, and the +/// target that decides (the entry's final target for a matched agent, the +/// agent's own for an agent listed as a GPU of its own). +/// +/// - The flag says `APU`: integrated. The runtime reads it from the device, so +/// an override does not change it. +/// - The flag is there without `APU`, but the target is in an APU family: +/// still integrated. The "discrete" reading of an empty flag rests on the +/// evidence [`parse_rocminfo_memory_properties`] gives, so the family list +/// outranks it. Where the entry already held its target, that target is the +/// native one, so this costs no override safety; an entry whose empty target the agent filled can carry +/// an overridden one, which is the residual docs/host-probing.md names. +/// - The flag is there without `APU`, and the target is outside the APU +/// families: discrete. +/// - No flag (a `rocminfo` that does not print it): [`apu_verdict_from_target`]. +fn apu_verdict_with_runtime_flag( + flag: Option, + current: Option, + target: &str, +) -> Option { + match flag { + Some(false) if gfx_is_apu_family(target) => Some(true), + Some(flag) => Some(flag), + None => apu_verdict_from_target(current, target), } +} + +/// List a `rocminfo` agent as a GPU of its own, for an agent no listed entry +/// can be. +fn gpu_from_agent(agent: RocminfoGpuAgent) -> Gpu { + Gpu { + name: if agent.marketing_name.is_empty() { + "AMD GPU".to_owned() + } else { + agent.marketing_name + }, + is_apu: apu_verdict_with_runtime_flag(agent.is_apu, None, &agent.gfx_target), + gfx_target: agent.gfx_target, + is_amd: true, + ..Gpu::default() + } +} +/// Fold a `rocminfo` reading into the GPU list, against caller-supplied output. +/// +/// Split from the process launch the same way [`apply_kernel_gpu_membership`] is +/// split from the `/sys` read, and for the same reason: the interesting +/// behaviour here is how an agent is matched onto an existing entry, and a test +/// that had to run the real `rocminfo` could only assert it on a host with a +/// GPU. See `a_rocminfo_marketing_name_outranks_the_kernel_topology_placeholder`. +/// +/// Matched by PCI address, not by position: `rocminfo` lists the agents the +/// ROCm runtime exposes, which a visibility mask such as `ROCR_VISIBLE_DEVICES` +/// can make a subset of the cards listed here, and nothing guarantees the two +/// lists share an order. Pairing by +/// position wrote one card's model name, target and APU verdict onto another. +/// +/// The cases, which the tests below cite by letter: +/// +/// - (a) An agent's `BDFID` is matched against each AMD entry's bus, device +/// and function, and a matched agent supplies the model name where the entry +/// has none and the gfx target only where the entry has none (see +/// [`apply_agent_to_gpu`] for why a held target is never replaced). +/// `rocminfo` prints no domain, so a match that two entries in different +/// domains share is refused and noted. Should several entries share one +/// address, they are paired with that address's agents in order. That is a +/// fallback, not how partitions are expected to arrive: per upstream kernel +/// and ROCm sources (not verified on hardware here), a partitioned GC 9.4.3 +/// card puts its partition id into `location_id`, so its partitions have +/// addresses of their own (`11:00.1`, `11:00.2`, ...) on both sides. +/// - (b) An agent no entry matches is listed as a GPU of its own when every +/// AMD entry has an address, since it then cannot be any of them. +/// - (c) When some AMD entry has no address, such an agent could be that one, +/// so nothing is written and a note says so. +/// - (d) An agent that states no address is never paired with an entry: not +/// by position, whatever the counts or targets, and whether or not other +/// agents state one. Its reading goes into the note. Per AGENTS.md §6, two +/// independently ordered lists are paired by id, and where no id exists the +/// answer degrades to unknown; device ids are not compared here. Whether any +/// supported `rocminfo` omits `BDFID` is unconfirmed: the ROCm 7.14 listing +/// captured for the tests (`REAL_ROCMINFO_GFX1151`) prints it for every +/// agent, and the release that first printed it was not found. So this case +/// may never arise on a supported host; it is kept so that one that does +/// arise is reported rather than guessed at. +/// - (e) An agent at an address whose entries other agents already took is +/// noted rather than written anywhere. +/// - (f) An entry no agent matches is left as it was: its own target and no +/// model name, rather than another card's. +/// - (g) When no AMD entry is listed at all, every agent is listed as a GPU of +/// its own, whether or not it states an address: there is nothing to +/// confuse it with. Its verdict comes from [`apu_verdict_with_runtime_flag`] +/// on its own target. +fn apply_rocminfo_gpu_agents(e: &mut Examination, out: &str) { + let agents = parse_rocminfo_gpu_agents(out); + if agents.is_empty() { + return; + } let amd_indices: Vec = e .gpus .iter() @@ -300,28 +568,79 @@ fn apply_rocminfo_gpu_agents(e: &mut Examination, out: &str) { .filter(|(_, g)| g.is_amd) .map(|(idx, _)| idx) .collect(); - for (idx, (gfx, marketing)) in gfx_targets.into_iter().enumerate() { - if let Some(&gpu_idx) = amd_indices.get(idx) { - let gpu = &mut e.gpus[gpu_idx]; - gpu.gfx_target = gfx.clone(); - if !marketing.is_empty() && gpu_name_is_unknown(&gpu.name) { - gpu.name = marketing; - } - gpu.is_apu = Some(gfx_is_apu_family(&gfx)); - } else { - let is_apu = gfx_is_apu_family(&gfx); - e.gpus.push(Gpu { - name: if marketing.is_empty() { - "AMD GPU".to_owned() + if amd_indices.is_empty() { + // Nothing listed for an agent to be confused with. + e.gpus.extend(agents.into_iter().map(gpu_from_agent)); + return; + } + + let mut unattributed: Vec = Vec::new(); + { + let some_entry_has_no_address = amd_indices + .iter() + .any(|&idx| split_pci_domain(&e.gpus[idx].pci_id).is_none()); + let mut claimed = vec![false; amd_indices.len()]; + let mut appended = Vec::new(); + for agent in agents { + let Some(address) = agent.bus_address.clone() else { + unattributed.push(unattributed_agent_label(&agent, "no bus address")); + continue; + }; + let candidates: Vec<(usize, &str)> = amd_indices + .iter() + .enumerate() + .filter_map(|(slot, &idx)| { + let (domain, bdf) = split_pci_domain(&e.gpus[idx].pci_id)?; + bdf.eq_ignore_ascii_case(&address).then_some((slot, domain)) + }) + .collect(); + if candidates.is_empty() { + if some_entry_has_no_address { + unattributed.push(unattributed_agent_label( + &agent, + "no listed GPU has that address, and one has no address at all", + )); } else { - marketing - }, - gfx_target: gfx, - is_amd: true, - is_apu: Some(is_apu), - ..Gpu::default() - }); + appended.push(gpu_from_agent(agent)); + } + continue; + } + let first_domain = candidates[0].1; + if candidates + .iter() + .any(|(_, domain)| !domain.eq_ignore_ascii_case(first_domain)) + { + unattributed.push(unattributed_agent_label( + &agent, + "GPUs in more than one PCI domain have that address", + )); + continue; + } + let Some(slot) = candidates + .iter() + .map(|(slot, _)| *slot) + .find(|&slot| !claimed[slot]) + else { + unattributed.push(unattributed_agent_label( + &agent, + "more rocminfo readings at this address than listed GPUs", + )); + continue; + }; + claimed[slot] = true; + apply_agent_to_gpu(&mut e.gpus[amd_indices[slot]], agent); } + e.gpus.extend(appended); + } + + if !unattributed.is_empty() { + e.notes.push(format!( + "{} GPU reading(s) from rocminfo could not be matched to a listed GPU, so rocminfo's \ + model names, gfx targets and integrated/discrete verdicts for them are not applied \ + to any GPU: {}.", + unattributed.len(), + unattributed.join("; ") + )); } } @@ -360,15 +679,14 @@ fn probe_gpus_sysfs_fallback(e: &mut Examination, gfx_target: fn() -> Option GpuProbeSources<'a> { /// Every GPU pass after the PCI scan, in the order they have to run in. /// /// The order is the whole point of gathering them here: each pass is correct -/// only in one position, and three of the four constraints below are invisible -/// to any test that calls the passes directly in an order it chose itself. +/// only in one position, and the three orderings below (membership before +/// `rocminfo`; the unnamed-GPU note after `rocminfo` and before the sysfs +/// fallback) are invisible to any test that calls the passes directly in an +/// order it chose itself. /// /// - The membership pass decides *which* AMD GPUs exist, so it runs before /// `rocminfo`: the PCI scan over-reports in a container, and `rocminfo` maps -/// its agents onto the surviving entries by position, so it has to see the -/// reconciled list rather than the whole bus. +/// its agents onto the surviving entries by PCI address (never by position; +/// see [`apply_rocminfo_gpu_agents`]), +/// so it has to see the reconciled list rather than the whole bus. The +/// membership pass is also what gives a kernel-only entry the address that +/// match needs. /// - [`note_unnamed_kernel_topology_gpus`] runs after `rocminfo`, which is the /// probe most likely to name a card the PCI scan missed, and before the sysfs /// fallback, which plants the same placeholder with a note of its own. @@ -562,21 +885,42 @@ fn apply_kernel_gpu_membership(e: &mut Examination, nodes: &[crate::KfdGpuNode]) gfx_target: node.gfx_target.clone(), pci_id: node.pci_id.clone(), is_amd: true, - // Left unset rather than guessed, for the same reason - // `probe_gpus_sysfs_fallback` leaves it unset: KFD gives the - // target, not the packaging. - is_apu: None, + is_apu: apu_verdict_from_target(None, &node.gfx_target), }); continue; }; claimed[index] = true; let mut gpu = from_pci[index].clone(); // The node's own target, so an APU+dGPU host labels each card with the - // target that belongs to it. Never an overwrite: `rocminfo` has not run - // yet here, but a target `lspci` resolved from the marketing name is a - // statement about this device and the node's is only a better one when - // there is nothing to compare it against. - if gpu.gfx_target.is_empty() { + // target that belongs to it. It fills an empty target and replaces one + // the PCI scan guessed from the marketing-name table + // (`classify_amd_marketing_name`): the guess is a lookup on a name, + // while the node's `gfx_target_version` is the kernel's statement + // about this device, and `HSA_OVERRIDE_GFX_VERSION` does not reach it. + // Krackan is where the tables disagree: the name table says gfx1150, + // the PCI device-id map says gfx1152, and which is right is + // unconfirmed (docs/host-probing.md). A node whose version did not parse + // to a target has nothing to offer and leaves the entry alone. + // + // The APU verdict, by which of the two it was: + // + // - Replacing a guess: the scan derived its verdict together with that + // guess, so the correction replaces both, and the verdict is the + // one the node's target gives -- `false` included. This is the only + // path that concludes "discrete" from a target, and only because it + // is correcting a verdict that came from the same wrong guess. + // - Filling an empty target: the scan's verdict was not derived from a + // target at all -- a default `false` for a name the table does not + // know, or `true` from an APU codename such as Renoir -- so + // [`apu_verdict_from_target`] decides: an APU-family target + // makes it `Some(true)` (a nameless Strix Halo "Device" row would + // otherwise read as discrete), and any other keeps the scan's verdict. + if !node.gfx_target.is_empty() && !gpu.gfx_target.eq_ignore_ascii_case(&node.gfx_target) { + gpu.is_apu = if gpu.gfx_target.is_empty() { + apu_verdict_from_target(gpu.is_apu, &node.gfx_target) + } else { + Some(gfx_is_apu_family(&node.gfx_target)) + }; gpu.gfx_target = node.gfx_target.clone(); } gpus.push(gpu); @@ -859,28 +1203,170 @@ mod tests { } #[test] - fn a_target_the_pci_scan_already_resolved_is_never_overwritten() { - // lspci resolves a target from the marketing name, which is a statement - // about that specific device. The node's answer is only a better one - // when there is nothing to compare it against. + fn the_kernels_target_fills_or_replaces_the_scans_and_the_verdict_follows_the_rules() { + // lspci's target is a lookup on the marketing name; the node's is the + // kernel's statement about the device, and an override cannot reach + // it. Krackan is where the tables disagree (the name table says + // gfx1150, the device-id map gfx1152; which is right is unconfirmed), + // so the node is planted with the device-id map's answer. + let entry = |name: &str, gfx: &str, is_apu: bool, pci_id: &str| Gpu { + name: name.to_owned(), + gfx_target: gfx.to_owned(), + pci_id: pci_id.to_owned(), + is_amd: true, + is_apu: Some(is_apu), + }; + let node = |pci_id: &str, gfx: &str| crate::KfdGpuNode { + pci_id: pci_id.to_owned(), + gfx_target: gfx.to_owned(), + }; + let mut e = Examination { + gpus: vec![ + entry("AMD Krackan", "gfx1150", true, "0000:64:00.0"), + // A guess whose packaging is wrong too: the verdict the scan + // derived alongside it is re-derived from the node's target. + entry("AMD Navi 31", "gfx1100", false, "0000:65:00.0"), + // A guessed APU the kernel says is something else: the + // correction replaces the verdict that came with the guess, + // so this one becomes discrete. + entry("AMD Phoenix", "gfx1103", true, "0000:6b:00.0"), + // A node with no parseable target has nothing to offer. + entry("AMD Phoenix", "gfx1103", true, "0000:66:00.0"), + // The same target in other case is the same target: kept as + // spelled, not replaced. + entry("AMD Navi 31", "GFX1100", false, "0000:68:00.0"), + // Filling an empty target with one in an APU family: the + // nameless Strix Halo row, whose scan verdict is the default + // `false`, becomes an APU. + entry( + "Advanced Micro Devices, Inc. [AMD/ATI] Device", + "", + false, + "0000:67:00.0", + ), + // Filling with a target outside every APU family keeps the + // scan's `true` from a codename: gfx909 (Picasso) is an APU + // `gfx_is_apu_family` does not list. + entry("AMD Picasso", "", true, "0000:69:00.0"), + // ...and an entry with no verdict keeps none: a target outside + // the APU families is no claim of discrete. + Gpu { + is_apu: None, + ..entry("AMD Device 7448", "", false, "0000:6a:00.0") + }, + ], + ..Examination::default() + }; + apply_kernel_gpu_membership( + &mut e, + &[ + node("0000:64:00.0", "gfx1152"), + node("0000:65:00.0", "gfx1103"), + node("0000:66:00.0", ""), + node("0000:67:00.0", "gfx1151"), + node("0000:68:00.0", "gfx1100"), + node("0000:69:00.0", "gfx909"), + node("0000:6b:00.0", "gfx1100"), + node("0000:6a:00.0", "gfx1100"), + ], + ); + let at = |pci_id: &str| { + e.gpus + .iter() + .find(|gpu| gpu.pci_id == pci_id) + .unwrap_or_else(|| panic!("{pci_id}: {:#?}", e.gpus)) + }; + assert_eq!(at("0000:64:00.0").gfx_target, "gfx1152"); + assert_eq!(at("0000:64:00.0").is_apu, Some(true)); + assert_eq!(at("0000:65:00.0").gfx_target, "gfx1103"); + assert_eq!( + at("0000:65:00.0").is_apu, + Some(true), + "the verdict follows the target that replaced the guess" + ); + assert_eq!(at("0000:66:00.0").gfx_target, "gfx1103"); + assert_eq!(at("0000:67:00.0").gfx_target, "gfx1151"); + assert_eq!( + at("0000:67:00.0").is_apu, + Some(true), + "a filled APU-family target makes the entry an APU" + ); + assert_eq!(at("0000:69:00.0").gfx_target, "gfx909"); + assert_eq!( + at("0000:69:00.0").is_apu, + Some(true), + "a codename's APU verdict is not downgraded by a target the family list lacks" + ); + assert_eq!(at("0000:6a:00.0").gfx_target, "gfx1100"); + assert_eq!(at("0000:6a:00.0").is_apu, None); + assert_eq!(at("0000:6b:00.0").gfx_target, "gfx1100"); + assert_eq!( + at("0000:6b:00.0").is_apu, + Some(false), + "a replaced guess's verdict is replaced with the node's" + ); + assert_eq!(at("0000:68:00.0").gfx_target, "GFX1100"); + } + + #[test] + fn a_kernel_node_matches_a_pci_entry_whatever_the_case_of_its_hex() { + // `lspci` prints lower-case hex; the comparison must not depend on + // it, or an upper-case spelling would add a second entry for one card. let mut e = Examination { gpus: vec![Gpu { - name: "Advanced Micro Devices, Inc. [AMD/ATI] Phoenix1".to_owned(), - gfx_target: "gfx1103".to_owned(), - pci_id: "0000:64:00.0".to_owned(), + name: "AMD Navi 31".to_owned(), + gfx_target: String::new(), + pci_id: "0000:C3:00.0".to_owned(), + is_apu: Some(false), + is_amd: true, + }], + ..Examination::default() + }; + apply_kernel_gpu_membership( + &mut e, + &[crate::KfdGpuNode { + pci_id: "0000:c3:00.0".to_owned(), + gfx_target: "gfx1100".to_owned(), + }], + ); + assert_eq!(e.gpus.len(), 1, "{:#?}", e.gpus); + assert_eq!(e.gpus[0].name, "AMD Navi 31"); + assert_eq!(e.gpus[0].gfx_target, "gfx1100"); + assert!(e.notes.is_empty(), "{:#?}", e.notes); + } + + #[test] + fn a_kernel_node_with_no_address_does_not_pair_with_an_entry_with_none() { + // An empty address means "unknown", not "wildcard": two unknowns are + // not one card. The entry is reported as not exposed, and the node is + // listed from the topology on its own. + let mut e = Examination { + gpus: vec![Gpu { + name: "AMD Navi 31".to_owned(), + gfx_target: String::new(), + pci_id: String::new(), + is_apu: Some(false), is_amd: true, - is_apu: Some(true), }], ..Examination::default() }; apply_kernel_gpu_membership( &mut e, &[crate::KfdGpuNode { - pci_id: "0000:64:00.0".to_owned(), - gfx_target: "gfx1150".to_owned(), + pci_id: String::new(), + gfx_target: "gfx1100".to_owned(), }], ); - assert_eq!(e.gpus[0].gfx_target, "gfx1103"); + assert_eq!(e.gpus.len(), 1, "{:#?}", e.gpus); + assert_eq!(e.gpus[0].name, KERNEL_TOPOLOGY_GPU_NAME); + assert_eq!(e.gpus[0].gfx_target, "gfx1100"); + assert!( + e.notes + .iter() + .any(|note| note.starts_with("1 AMD PCI device(s) are on the bus but not exposed")), + "{:#?}", + e.notes + ); } #[test] @@ -1103,7 +1589,7 @@ mod tests { &mut e, "Agent 1\n Name: AMD Ryzen\n Device Type: CPU\n\ Agent 2\n Name: gfx942\n Marketing Name: AMD Instinct MI300X\n \ - Device Type: GPU\n", + Device Type: GPU\n BDFID: 23808\n", ); note_unnamed_kernel_topology_gpus(&mut e); @@ -1171,7 +1657,7 @@ mod tests { rocminfo: Some( "Agent 1\n Name: AMD Ryzen\n Device Type: CPU\n\ Agent 2\n Name: gfx942\n Marketing Name: AMD Instinct MI300X\n \ - Device Type: GPU\n", + Device Type: GPU\n BDFID: 23808\n", ), sysfs_gfx_target: || Some("gfx942".to_owned()), }, @@ -1254,27 +1740,35 @@ mod tests { } #[test] - fn a_topology_sourced_gpu_counts_as_present_without_claiming_a_package() { - // What the fallback produces: enough to stop reporting "no AMD GPU", - // and no more. `is_apu: None` is deliberate -- the topology says which - // target, not whether it is integrated -- so the APU and discrete - // tallies must both stay false rather than guess. - let mut e = Examination { - os_family: "linux".to_owned(), - gpus: vec![Gpu { - name: "AMD GPU (from kernel topology)".to_owned(), - gfx_target: "gfx942".to_owned(), - is_amd: true, - is_apu: None, - ..Gpu::default() - }], - ..Examination::default() + fn the_sysfs_fallback_claims_an_apu_only_from_an_apu_family_target() { + // The sysfs fallback: the kernel gives a target and nothing else. A + // target in an APU family makes the entry integrated. + let fallback = |gfx: fn() -> Option| { + let mut e = Examination { + os_family: "linux".to_owned(), + ..Examination::default() + }; + probe_gpus_sysfs_fallback(&mut e, gfx); + summarise_gpu_categories(&mut e); + e }; - summarise_gpu_categories(&mut e); - assert!(e.has_amd_gpu, "the machine has a GPU and must say so"); - assert!(!e.has_apu); - assert!(!e.has_discrete_amd); - assert_eq!(e.compute_status(), "ok"); + let apu = fallback(|| Some("gfx1151".to_owned())); + assert_eq!(apu.gpus.len(), 1, "{:#?}", apu.gpus); + assert_eq!(apu.gpus[0].is_apu, Some(true)); + assert!(apu.has_amd_gpu); + assert!(apu.has_apu); + assert!(!apu.has_discrete_amd); + + // Any other target is no claim: the family list does not name every + // APU, so an Instinct gfx942 counts as an AMD GPU and as neither + // package rather than as a guessed discrete one. + let other = fallback(|| Some("gfx942".to_owned())); + assert_eq!(other.gpus.len(), 1, "{:#?}", other.gpus); + assert_eq!(other.gpus[0].is_apu, None); + assert!(other.has_amd_gpu, "the machine has a GPU and must say so"); + assert!(!other.has_apu); + assert!(!other.has_discrete_amd); + assert_eq!(other.compute_status(), "ok"); } #[test] @@ -1494,4 +1988,1183 @@ mod tests { ("gfx1151".to_owned(), true) ); } + + // -- `rocminfo` agents matched onto the listed GPUs (ROCMAI-40) --------- + // + // Every row of the state matrix in `apply_rocminfo_gpu_agents`'s doc + // comment has a test that cites it by letter, plus the shapes that cross rows: a subset, a reordering, an + // APU beside a dGPU, a domain clash and a partitioned card. + + /// A `rocminfo` listing in the shape the real tool prints (compare the + /// captured [`REAL_ROCMINFO_GFX1151`]): a CPU agent with `BDFID: 0`, then + /// one GPU agent per `(gfx, marketing name, BDFID)`, each value padded to + /// the tool's column and each agent closed by its `ISA Info:` block, whose + /// ISA entries carry `Name:` lines of their own. `None` omits the `BDFID` + /// line, as a `rocminfo` too old to print it would. Each agent's + /// `Memory Properties:` line says `APU` for the integrated parts in + /// [`FIXTURE_INTEGRATED_TARGETS`] and is empty otherwise, as the real tool + /// prints; [`rocminfo_listing_with_memory`] sets it explicitly. + fn rocminfo_listing(gpus: &[(&str, &str, Option)]) -> String { + let with_memory: Vec<(&str, &str, Option, Option<&str>)> = gpus + .iter() + .map(|&(gfx, marketing, bdfid)| { + let integrated = FIXTURE_INTEGRATED_TARGETS.contains(&gfx); + ( + gfx, + marketing, + bdfid, + Some(if integrated { "APU" } else { "" }), + ) + }) + .collect(); + rocminfo_listing_with_memory(&with_memory) + } + + /// The fixture targets that are integrated parts, written out rather than + /// taken from `gfx_is_apu_family`, which is code under test: a fixture + /// that asked the code which parts are integrated could not catch it + /// getting that wrong. + const FIXTURE_INTEGRATED_TARGETS: &[&str] = &[ + "gfx1103", "gfx1150", "gfx1151", "gfx1152", "gfx909", "gfx90c", + ]; + + /// [`rocminfo_listing`] with each agent's `Memory Properties:` value given: + /// `Some("APU")`, `Some("")` (a discrete GPU), or `None` to omit the line, + /// as a `rocminfo` that does not print it would. + fn rocminfo_listing_with_memory(gpus: &[(&str, &str, Option, Option<&str>)]) -> String { + use std::fmt::Write as _; + let mut out = String::from( + "******* \nAgent 1 \n******* \n \ + Name: AMD Ryzen 9 7950X\n \ + Marketing Name: AMD Ryzen 9 7950X\n \ + Device Type: CPU \n \ + BDFID: 0 \n \ + Memory Properties: \n \ + ISA Info: \n", + ); + for (index, (gfx, marketing, bdfid, memory)) in gpus.iter().enumerate() { + write!( + out, + "******* \nAgent {} \n******* \n \ + Name: {gfx} \n \ + Uuid: GPU-XX \n \ + Marketing Name: {marketing} \n \ + Vendor Name: AMD \n \ + Device Type: GPU \n", + index + 2 + ) + .expect("writing to a String cannot fail"); + if let Some(bdfid) = bdfid { + writeln!( + out, + " BDFID: {bdfid} " + ) + .expect("writing to a String cannot fail"); + } + if let Some(memory) = memory { + writeln!(out, " Memory Properties: {memory}") + .expect("writing to a String cannot fail"); + } + write!( + out, + " ISA Info: \n ISA 1 \n \ + Name: amdgcn-amd-amdhsa--{gfx} \n \ + Machine Models: HSA_MACHINE_MODEL_LARGE \n \ + ISA 2 \n \ + Name: amdgcn-amd-amdhsa--gfx11-generic \n" + ) + .expect("writing to a String cannot fail"); + } + out + } + + /// `rocminfo` as a ROCm 7.14 SDK prints it on a Strix Halo host (gfx1151 at + /// c3:00.0), captured verbatim bar trailing padding, colour codes included: + /// a CPU agent, the GPU agent with its `ISA Info:` block, and an NPU agent. + /// The same host's KFD topology, read at capture time, has that GPU as node + /// 1 with `location_id 49920`, `domain 0` and `gfx_target_version 110501` + /// ([`REAL_KFD_NODE_GFX1151`]): the `BDFID` and the `location_id` are the + /// same number, which is what matching an agent to a kernel node rests on. + const REAL_ROCMINFO_GFX1151: &str = + include_str!("../../tests/fixtures/rocminfo_gfx1151_apu.txt"); + + /// Run the GPU passes after the PCI scan over a planted kernel topology + /// (one GPU node per `(location_id, gfx_target_version)`) and a `rocminfo` + /// listing, with an empty PCI scan in front: the ordinary ROCm container. + fn examine_planted_host(tag: &str, nodes: &[(u32, u32)], rocminfo: &str) -> Examination { + let (root, nodes) = plant_kfd_topology(tag, nodes); + let mut e = Examination::default(); + probe_gpus_after_lspci( + &mut e, + GpuProbeSources { + kfd_nodes: &nodes, + rocminfo: Some(rocminfo), + sysfs_gfx_target: || None, + }, + ); + std::fs::remove_dir_all(&root).ok(); + summarise_gpu_categories(&mut e); + e + } + + fn gpu_at<'a>(e: &'a Examination, pci_id: &str) -> &'a Gpu { + e.gpus + .iter() + .find(|gpu| gpu.pci_id == pci_id) + .unwrap_or_else(|| panic!("no GPU listed at {pci_id}: {:#?}", e.gpus)) + } + + /// The note `apply_rocminfo_gpu_agents` takes for readings it could not + /// attribute, and only that one. + fn unattributed_notes(e: &Examination) -> Vec<&String> { + e.notes + .iter() + .filter(|note| note.contains("from rocminfo could not be matched")) + .collect() + } + + // `location_id` / `BDFID` 768 is 03:00.0, 1024 is 04:00.0, 50176 is c4:00.0. + const AT_03: u32 = 768; + const AT_04: u32 = 1024; + const AT_C4: u32 = 50176; + + #[test] + fn a_rocminfo_bdfid_line_in_the_real_shape_decodes_to_its_bus_address() { + // Real `rocminfo` pads the value to a column and prints `BDFID: 0` for + // a CPU agent. 49_920 is the GPU at c3:00.0 on a ROCm 7.14 host. + let agents = parse_rocminfo_gpu_agents(&rocminfo_listing(&[( + "gfx1100", + "AMD Radeon RX 7900 XTX", + Some(49_920), + )])); + assert_eq!( + agents, + vec![RocminfoGpuAgent { + gfx_target: "gfx1100".to_owned(), + marketing_name: "AMD Radeon RX 7900 XTX".to_owned(), + bus_address: Some("c3:00.0".to_owned()), + is_apu: Some(false), + }], + "the CPU agent is not a GPU, and the GPU's BDFID is its address" + ); + assert_eq!(parse_rocminfo_bdfid(" 0"), None); + assert_eq!(parse_rocminfo_bdfid(" not-a-number"), None); + // A missing line is no address, not a zero one. + let without = parse_rocminfo_gpu_agents(&rocminfo_listing(&[("gfx1100", "X", None)])); + assert_eq!(without[0].bus_address, None); + } + + #[test] + fn a_gpu_agent_whose_name_is_no_gfx_target_is_not_a_gpu_reading() { + // An agent can say `Device Type: GPU` and still carry a name that is + // no gfx target; it gives nothing to match or to fill a target with. + let listing = rocminfo_listing(&[ + ("gfx1100", "AMD Radeon RX 7900 XTX", Some(AT_03)), + ("AMD Instinct Virtual", "AMD Radeon RX 7800 XT", Some(AT_04)), + ]); + let agents = parse_rocminfo_gpu_agents(&listing); + assert_eq!( + agents + .iter() + .map(|a| a.gfx_target.as_str()) + .collect::>(), + vec!["gfx1100"], + "{agents:#?}" + ); + } + + #[test] + fn a_kfd_location_id_and_a_rocminfo_bdfid_name_the_same_card_alike() { + // The producer and the consumer of the match: the kernel topology + // gives each listed GPU its address from `location_id`, `rocminfo` + // gives each agent one from `BDFID`, and the match is only right if + // one number decodes to one address on both sides. Includes non-zero + // devices and functions (0x1c7 and 0x305 carry functions 7 and 5). + // Both sides share one decode, so this cannot see that decode being + // wrong on both: `a_dev_id_decodes_every_function_bit_and_refuses_zero_and_wider_values` + // in `host_gpu/linux.rs` pins it to literal addresses. + for dev_id in [AT_03, AT_04, AT_C4, 0x8ffa, 23808, 0x1c7, 0x305] { + let (root, nodes) = plant_kfd_topology("bdf-tie", &[(dev_id, 110_000)]); + let read = crate::kfd_gpu_nodes_in(&nodes).expect("planted topology must be readable"); + std::fs::remove_dir_all(&root).ok(); + let bdf = parse_rocminfo_bdfid(&format!(" {dev_id}")) + .unwrap_or_else(|| panic!("BDFID {dev_id} must decode")); + let (domain, kfd_bdf) = + split_pci_domain(&read[0].pci_id).expect("the node must carry an address"); + assert_eq!(domain, "0000"); + assert_eq!( + kfd_bdf, bdf, + "location_id and BDFID {dev_id} must decode alike" + ); + } + } + + #[test] + fn a_gpu_the_rocm_runtime_cannot_see_does_not_borrow_another_gpus_reading() { + // Rows (a) and (f), and the reported defect: two GPUs in the kernel + // topology, and a `rocminfo` that sees only the second (a visibility + // mask). Pairing by position wrote the second card's model name, + // target and APU verdict onto the first. + let e = examine_planted_host( + "rocminfo-subset", + &[(AT_03, 110_000), (AT_04, 110_001)], + &rocminfo_listing(&[("gfx1101", "AMD Radeon RX 7800 XT", Some(AT_04))]), + ); + assert_eq!(e.gpus.len(), 2, "{:#?}", e.gpus); + let seen = gpu_at(&e, "0000:04:00.0"); + assert_eq!(seen.name, "AMD Radeon RX 7800 XT"); + assert_eq!(seen.gfx_target, "gfx1101"); + // The runtime's own `Memory Properties:` reading, with no `APU` in it. + assert_eq!(seen.is_apu, Some(false)); + let unseen = gpu_at(&e, "0000:03:00.0"); + assert_eq!( + unseen.name, KERNEL_TOPOLOGY_GPU_NAME, + "the card rocminfo did not describe keeps no model name" + ); + assert_eq!(unseen.gfx_target, "gfx1100", "and keeps its own target"); + assert_eq!( + unseen.is_apu, None, + "and no packaging claim, rather than the other card's verdict" + ); + assert_eq!(unattributed_notes(&e), Vec::<&String>::new()); + // The unknown-name note now counts the right card: one is unnamed. + assert_eq!(unknown_name_notes(&e).len(), 1, "{:#?}", e.notes); + } + + #[test] + fn rocminfo_agents_listed_in_another_order_each_reach_their_own_card() { + // Row (a) with the two lists in opposite orders: every agent matched, + // each by its own address. + let e = examine_planted_host( + "rocminfo-reordered", + &[(AT_03, 110_000), (AT_04, 110_001)], + &rocminfo_listing(&[ + ("gfx1101", "AMD Radeon RX 7800 XT", Some(AT_04)), + ("gfx1100", "AMD Radeon RX 7900 XTX", Some(AT_03)), + ]), + ); + assert_eq!(e.gpus.len(), 2, "{:#?}", e.gpus); + assert_eq!(gpu_at(&e, "0000:03:00.0").name, "AMD Radeon RX 7900 XTX"); + assert_eq!(gpu_at(&e, "0000:03:00.0").gfx_target, "gfx1100"); + assert_eq!(gpu_at(&e, "0000:04:00.0").name, "AMD Radeon RX 7800 XT"); + assert_eq!(gpu_at(&e, "0000:04:00.0").gfx_target, "gfx1101"); + assert!(e.notes.is_empty(), "{:#?}", e.notes); + } + + #[test] + fn an_apu_the_runtime_hides_is_still_reported_as_an_apu() { + // The APU+dGPU host with the iGPU masked out of the runtime: the + // dGPU's "discrete" verdict landed on the APU, so `has_apu` read + // false, which `rocm diagnose`'s iGPU/dGPU check reads. + let mut e = examine_planted_host( + "rocminfo-apu-hidden", + &[(AT_C4, 110_003), (AT_03, 110_000)], + &rocminfo_listing(&[("gfx1100", "AMD Radeon RX 7900 XTX", Some(AT_03))]), + ); + let apu = gpu_at(&e, "0000:c4:00.0"); + assert_eq!(apu.gfx_target, "gfx1103"); + assert_eq!(apu.name, KERNEL_TOPOLOGY_GPU_NAME); + let dgpu = gpu_at(&e, "0000:03:00.0"); + assert_eq!(dgpu.gfx_target, "gfx1100"); + // The hidden APU is still an APU: its verdict comes from the target + // the kernel gives it. The dGPU the runtime can see is discrete by the + // runtime's own reading (`Memory Properties:` with no `APU`), so even + // with no PCI scan the host reads as integrated plus discrete, the pair + // `rocm diagnose`'s iGPU/dGPU check looks for. + assert_eq!(apu.is_apu, Some(true)); + assert_eq!(dgpu.is_apu, Some(false)); + assert!(e.has_apu, "{e:#?}"); + assert!(e.has_discrete_amd, "{e:#?}"); + + // With `lspci` naming both (which carries the packaging), the hidden + // APU keeps the verdict the scan gave it. + e = Examination { + gpus: vec![ + Gpu { + name: "AMD Phoenix".to_owned(), + gfx_target: "gfx1103".to_owned(), + pci_id: "0000:c4:00.0".to_owned(), + is_apu: Some(true), + is_amd: true, + }, + Gpu { + name: "Navi 31 [Radeon RX 7900 XT/7900 XTX]".to_owned(), + gfx_target: String::new(), + pci_id: "0000:03:00.0".to_owned(), + is_apu: Some(false), + is_amd: true, + }, + ], + ..Examination::default() + }; + apply_rocminfo_gpu_agents( + &mut e, + &rocminfo_listing(&[("gfx1100", "AMD Radeon RX 7900 XTX", Some(AT_03))]), + ); + summarise_gpu_categories(&mut e); + assert_eq!(gpu_at(&e, "0000:c4:00.0").gfx_target, "gfx1103"); + assert_eq!(gpu_at(&e, "0000:c4:00.0").is_apu, Some(true)); + assert_eq!(gpu_at(&e, "0000:03:00.0").gfx_target, "gfx1100"); + assert!(e.has_apu, "the hidden APU must still be reported as one"); + assert!(e.has_discrete_amd); + } + + #[test] + fn an_agent_at_an_address_no_listed_gpu_has_is_listed_as_its_own_gpu() { + // Row (b): every listed AMD GPU has an address and none is the + // agent's, so the agent cannot be any of them and is added, as before. + let e = examine_planted_host( + "rocminfo-unmatched-addressed", + &[(AT_03, 110_000)], + // The last agent has no marketing name and no `Memory Properties:` + // line (a `rocminfo` that does not print it), so its verdict comes + // from the target rule on its own gfx1151. + &rocminfo_listing_with_memory(&[ + ("gfx1100", "AMD Radeon RX 7900 XTX", Some(AT_03), Some("")), + ("gfx1101", "AMD Radeon RX 7800 XT", Some(AT_04), Some("")), + ("gfx1151", "", Some(AT_C4), None), + ]), + ); + assert_eq!(e.gpus.len(), 3, "{:#?}", e.gpus); + assert_eq!(gpu_at(&e, "0000:03:00.0").name, "AMD Radeon RX 7900 XTX"); + let added = e + .gpus + .iter() + .find(|gpu| gpu.name == "AMD Radeon RX 7800 XT") + .expect("the unmatched agent must be listed"); + assert_eq!(added.gfx_target, "gfx1101"); + assert_eq!( + added.is_apu, + Some(false), + "the runtime's own reading: `Memory Properties:` with no `APU`" + ); + assert!( + added.pci_id.is_empty(), + "rocminfo gives no domain to build one from" + ); + // An agent that printed no marketing name is still listed, under the + // generic name, and with no flag the target rule makes its gfx1151 + // integrated. + let unnamed = e + .gpus + .iter() + .find(|gpu| gpu.gfx_target == "gfx1151") + .expect("the agent with no marketing name must be listed"); + assert_eq!(unnamed.name, "AMD GPU"); + assert_eq!(unnamed.is_apu, Some(true)); + assert!(unattributed_notes(&e).is_empty(), "{:#?}", e.notes); + } + + #[test] + fn an_agent_that_could_be_the_gpu_with_no_address_is_noted_not_added() { + // Row (c): a listed GPU has no address (a KFD node with no + // `location_id`), so an agent that matches no address could be that + // very card. Adding it would list one card twice; writing onto it + // would be a guess. Neither: the entry stays as it was, with a note. + let mut e = Examination { + gpus: vec![Gpu { + name: KERNEL_TOPOLOGY_GPU_NAME.to_owned(), + gfx_target: "gfx1100".to_owned(), + pci_id: String::new(), + is_apu: None, + is_amd: true, + }], + ..Examination::default() + }; + let before = e.gpus.clone(); + apply_rocminfo_gpu_agents( + &mut e, + &rocminfo_listing(&[("gfx1101", "AMD Radeon RX 7800 XT", Some(AT_04))]), + ); + assert_eq!(e.gpus, before, "nothing added and nothing written"); + let notes = unattributed_notes(&e); + assert_eq!(notes.len(), 1, "{:#?}", e.notes); + assert!( + notes[0].starts_with("1 GPU reading(s)") + && notes[0].contains("gfx1101 at bus address 04:00.0"), + "the note must name the reading it set aside: {notes:?}" + ); + assert!( + notes[0].is_ascii(), + "examine notes must be ASCII-only: {notes:?}" + ); + // What the note claims, held against the entry asserted unchanged + // above: rocminfo's reading was not applied, and the entry keeps its own. + assert!( + notes[0].contains( + "rocminfo's model names, gfx targets and integrated/discrete verdicts for them \ + are not applied to any GPU" + ), + "{notes:?}" + ); + } + + #[test] + fn an_address_shared_across_pci_domains_is_refused() { + // `rocminfo` prints no domain, so 03:00.0 in domain 0 and in domain 1 + // are indistinguishable to it. Picking either would be a guess. + let mut e = Examination { + gpus: ["0000:03:00.0", "0001:03:00.0"] + .into_iter() + .map(|pci_id| Gpu { + name: "Advanced Micro Devices, Inc. [AMD/ATI] Device".to_owned(), + gfx_target: String::new(), + pci_id: pci_id.to_owned(), + is_apu: Some(false), + is_amd: true, + }) + .collect(), + ..Examination::default() + }; + let before = e.gpus.clone(); + apply_rocminfo_gpu_agents( + &mut e, + &rocminfo_listing(&[("gfx942", "AMD Instinct MI300X", Some(AT_03))]), + ); + assert_eq!(e.gpus, before, "neither card may take the reading"); + let notes = unattributed_notes(&e); + assert_eq!(notes.len(), 1, "{:#?}", e.notes); + assert!(notes[0].contains("more than one PCI domain"), "{notes:?}"); + } + + #[test] + fn an_agent_with_no_address_beside_agents_with_one_is_set_aside() { + // Row (d), with other agents that do state an address: those match + // by address, and the one that states none is not paired with + // whatever entry is left over. + let e = examine_planted_host( + "rocminfo-mixed-bdfid", + &[(AT_03, 110_000), (AT_04, 110_001)], + &rocminfo_listing(&[ + ("gfx1101", "AMD Radeon RX 7800 XT", Some(AT_04)), + ("gfx1100", "AMD Radeon RX 7900 XTX", None), + ]), + ); + assert_eq!(e.gpus.len(), 2, "{:#?}", e.gpus); + assert_eq!(gpu_at(&e, "0000:04:00.0").name, "AMD Radeon RX 7800 XT"); + assert_eq!(gpu_at(&e, "0000:03:00.0").name, KERNEL_TOPOLOGY_GPU_NAME); + let notes = unattributed_notes(&e); + assert_eq!(notes.len(), 1, "{:#?}", e.notes); + assert!(notes[0].contains("gfx1100 (no bus address)"), "{notes:?}"); + } + + #[test] + fn a_real_rocminfo_listing_yields_its_gpu_agent() { + // A real listing closes each agent with an `ISA Info:` block whose ISA + // entries have `Name:` lines of their own. Read as the agent's name, + // the last of them (`amdgcn-amd-amdhsa--gfx11-generic`) replaced + // `gfx1151`, and no GPU agent of any real listing survived. + let agents = parse_rocminfo_gpu_agents(REAL_ROCMINFO_GFX1151); + assert_eq!( + agents, + vec![RocminfoGpuAgent { + gfx_target: "gfx1151".to_owned(), + marketing_name: "AMD Radeon 8060S Graphics".to_owned(), + bus_address: Some("c3:00.0".to_owned()), + // The capture's `Memory Properties: APU`. + is_apu: Some(true), + }], + "one GPU agent; the CPU and the NPU agents are not GPUs" + ); + + // Folded into what `lspci -nn -D` lists for that host: its `pci.ids` + // has no name for the device, so the scan guessed no target and + // called it discrete. The reading fills the target, and the verdict + // follows it. + let mut e = Examination { + gpus: vec![Gpu { + name: "Advanced Micro Devices, Inc. [AMD/ATI] Device".to_owned(), + gfx_target: String::new(), + pci_id: "0000:c3:00.0".to_owned(), + is_apu: Some(false), + is_amd: true, + }], + ..Examination::default() + }; + apply_rocminfo_gpu_agents(&mut e, REAL_ROCMINFO_GFX1151); + summarise_gpu_categories(&mut e); + assert_eq!(e.gpus.len(), 1, "{:#?}", e.gpus); + assert_eq!(e.gpus[0].gfx_target, "gfx1151"); + assert_eq!(e.gpus[0].is_apu, Some(true)); + assert!(e.has_apu, "{e:#?}"); + assert!(!e.has_discrete_amd, "{e:#?}"); + assert!(e.notes.is_empty(), "{:#?}", e.notes); + } + + #[test] + fn a_name_the_pci_scan_gave_is_not_replaced_by_the_marketing_name() { + // The name guard: a matched agent's marketing name fills only an entry + // with no name or the kernel-topology placeholder. Whatever the PCI + // scan called the card stays, whether or not it is a good name. + let mut e = Examination { + gpus: vec![Gpu { + name: "Navi 31 [Radeon RX 7900 XT/7900 XTX]".to_owned(), + gfx_target: "gfx1100".to_owned(), + pci_id: "0000:03:00.0".to_owned(), + is_apu: Some(false), + is_amd: true, + }], + ..Examination::default() + }; + let before = e.gpus[0].name.clone(); + apply_rocminfo_gpu_agents( + &mut e, + &rocminfo_listing(&[("gfx1100", "AMD Radeon RX 7900 XTX", Some(AT_03))]), + ); + assert_eq!( + e.gpus[0].name, before, + "the name is unchanged from the lspci entry" + ); + } + + #[test] + fn a_listed_address_splits_only_in_its_domain_qualified_shape() { + assert_eq!(split_pci_domain("0000:03:00.0"), Some(("0000", "03:00.0"))); + assert_eq!(split_pci_domain("0001:C4:00.0"), Some(("0001", "C4:00.0"))); + // No address, an empty domain, no domain at all, and no separator are + // all "address unknown", never an address in a guessed domain. + for unknown in ["", ":03:00.0", "03:00.0", "0000"] { + assert_eq!(split_pci_domain(unknown), None, "{unknown:?}"); + } + } + + #[test] + fn a_listed_gpu_with_a_malformed_address_counts_as_having_none() { + // An address not in the `domain:bus:dev.fn` shape cannot be compared + // with an agent's, so it is as unknown as an empty one: the agent at + // 04:00.0 could be that card, and is set aside rather than added as a + // second GPU or written onto it. + let mut e = Examination { + gpus: [("0000:03:00.0", "gfx1100"), ("04:00.0", "gfx1101")] + .into_iter() + .map(|(pci_id, gfx)| Gpu { + name: KERNEL_TOPOLOGY_GPU_NAME.to_owned(), + gfx_target: gfx.to_owned(), + pci_id: pci_id.to_owned(), + is_apu: None, + is_amd: true, + }) + .collect(), + ..Examination::default() + }; + let before = e.gpus.clone(); + apply_rocminfo_gpu_agents( + &mut e, + &rocminfo_listing(&[("gfx1101", "AMD Radeon RX 7800 XT", Some(AT_04))]), + ); + assert_eq!(e.gpus, before, "nothing added and nothing written"); + let notes = unattributed_notes(&e); + assert_eq!(notes.len(), 1, "{:#?}", e.notes); + assert!(notes[0].contains("one has no address at all"), "{notes:?}"); + } + + #[test] + fn an_overridden_runtime_target_does_not_replace_the_cards_own() { + // `HSA_OVERRIDE_GFX_VERSION=11.0.0` on a gfx1103 APU: `rocminfo` + // inherits it and lists the agent as gfx1100, at the APU's own + // address. The kernel's target is the card's; replacing it would make + // the APU a discrete gfx1100 and hide the override from every check + // that compares the two. + // The real runtime still prints `Memory Properties: APU` under the + // override (on the gfx1151 host, `rocminfo` with an 11.0.0 override + // reports gfx1100 and APU), so the listing does too. + let e = examine_planted_host( + "rocminfo-override", + &[(AT_C4, 110_003)], + &rocminfo_listing_with_memory(&[( + "gfx1100", + "AMD Radeon 780M Graphics", + Some(AT_C4), + Some("APU"), + )]), + ); + let apu = gpu_at(&e, "0000:c4:00.0"); + assert_eq!(apu.gfx_target, "gfx1103", "the native target is kept"); + assert_eq!( + apu.is_apu, + Some(true), + "the runtime's APU reading, which the override does not change" + ); + assert_eq!( + apu.name, "AMD Radeon 780M Graphics", + "the name is still taken" + ); + assert!(e.has_apu); + assert!(!e.has_discrete_amd); + + // An entry with no target yet takes the agent's: it is the only + // reading there is. + let mut empty = Examination { + gpus: vec![Gpu { + name: "Navi 31 [Radeon RX 7900 XT/7900 XTX]".to_owned(), + gfx_target: String::new(), + pci_id: "0000:03:00.0".to_owned(), + is_apu: Some(false), + is_amd: true, + }], + ..Examination::default() + }; + apply_rocminfo_gpu_agents( + &mut empty, + &rocminfo_listing(&[("gfx1100", "AMD Radeon RX 7900 XTX", Some(AT_03))]), + ); + assert_eq!( + empty.gpus[0].gfx_target, "gfx1100", + "an empty target is filled" + ); + assert_eq!(empty.gpus[0].is_apu, Some(false)); + } + + /// The Strix Halo host's KFD node for its GPU, as read when + /// [`REAL_ROCMINFO_GFX1151`] was captured: `(location_id, + /// gfx_target_version)`, in domain 0. + const REAL_KFD_NODE_GFX1151: (u32, u32) = (49_920, 110_501); + + #[test] + fn a_real_rocminfo_listing_matches_its_real_kernel_node() { + // Both halves from the same real host: the kernel node as read from + // its sysfs, and the `rocminfo` listing captured beside it. The match + // rests on `BDFID` and `location_id` being one number; here they are. + let e = examine_planted_host( + "rocminfo-real-host", + &[REAL_KFD_NODE_GFX1151], + REAL_ROCMINFO_GFX1151, + ); + assert_eq!(e.gpus.len(), 1, "{:#?}", e.gpus); + let gpu = gpu_at(&e, "0000:c3:00.0"); + assert_eq!(gpu.name, "AMD Radeon 8060S Graphics"); + assert_eq!(gpu.gfx_target, "gfx1151"); + assert_eq!(gpu.is_apu, Some(true)); + assert!(e.notes.is_empty(), "{:#?}", e.notes); + } + + #[test] + fn an_agent_that_is_no_gpu_is_not_a_gpu_reading_whatever_its_name() { + // `Device Type` decides: an agent typed CPU or DSP is skipped even when + // its `Name:` reads like a gfx target. + let listing = "*******\nAgent 1\n*******\n Name: gfx1100\n \ + Marketing Name: Not a GPU\n \ + Device Type: DSP\n BDFID: 768\n \ + ISA Info:\n"; + assert_eq!(parse_rocminfo_gpu_agents(listing), Vec::new()); + let mut e = Examination::default(); + apply_rocminfo_gpu_agents(&mut e, listing); + assert!(e.gpus.is_empty(), "{:#?}", e.gpus); + } + + #[test] + fn rocminfo_never_replaces_a_name_table_guess_even_without_a_kernel_node() { + // The tradeoff `apply_agent_to_gpu` takes on purpose. With no KFD node + // to correct it, the PCI scan's guess from the name table is the only + // other reading, and it is kept over the runtime's: the runtime's can + // be the ISA an `HSA_OVERRIDE_GFX_VERSION` names, and from here the two + // cannot be told apart. So a wrong guess stands on such a host. + let mut e = Examination { + gpus: vec![Gpu { + name: "Advanced Micro Devices, Inc. [AMD/ATI] Phoenix1".to_owned(), + gfx_target: "gfx1103".to_owned(), + pci_id: "0000:03:00.0".to_owned(), + is_apu: Some(true), + is_amd: true, + }], + ..Examination::default() + }; + // The agent an 11.0.0 override would print for this APU: gfx1100, + // still flagged APU. + apply_rocminfo_gpu_agents( + &mut e, + &rocminfo_listing_with_memory(&[( + "gfx1100", + "AMD Radeon 780M Graphics", + Some(AT_03), + Some("APU"), + )]), + ); + assert_eq!(e.gpus[0].gfx_target, "gfx1103", "the guess is kept"); + assert_eq!(e.gpus[0].is_apu, Some(true)); + } + + #[test] + fn addresses_and_domains_match_whatever_their_case() { + // A listed address in upper-case hex still matches the agent's + // lower-case one, and two spellings of one domain are one domain, not + // the cross-domain clash that is refused. + let entry = |pci_id: &str| Gpu { + name: KERNEL_TOPOLOGY_GPU_NAME.to_owned(), + gfx_target: "gfx1103".to_owned(), + pci_id: pci_id.to_owned(), + is_apu: None, + is_amd: true, + }; + let mut e = Examination { + gpus: vec![entry("0000:C4:00.0")], + ..Examination::default() + }; + apply_rocminfo_gpu_agents( + &mut e, + &rocminfo_listing(&[("gfx1103", "AMD Radeon 780M Graphics", Some(AT_C4))]), + ); + assert_eq!(e.gpus[0].name, "AMD Radeon 780M Graphics", "{:#?}", e.notes); + + let mut partitions = Examination { + gpus: vec![entry("000A:03:00.0"), entry("000a:03:00.0")], + ..Examination::default() + }; + apply_rocminfo_gpu_agents( + &mut partitions, + &rocminfo_listing(&[("gfx1103", "AMD Radeon 780M Graphics", Some(AT_03))]), + ); + assert_eq!( + partitions.gpus[0].name, "AMD Radeon 780M Graphics", + "{:#?}", + partitions.notes + ); + assert!(partitions.notes.is_empty(), "{:#?}", partitions.notes); + } + + #[test] + fn an_agent_with_no_marketing_name_does_not_blank_the_entrys_name() { + let mut e = Examination { + gpus: vec![Gpu { + name: KERNEL_TOPOLOGY_GPU_NAME.to_owned(), + gfx_target: "gfx1100".to_owned(), + pci_id: "0000:03:00.0".to_owned(), + is_apu: Some(false), + is_amd: true, + }], + ..Examination::default() + }; + apply_rocminfo_gpu_agents(&mut e, &rocminfo_listing(&[("gfx1100", "", Some(AT_03))])); + assert_eq!(e.gpus[0].name, KERNEL_TOPOLOGY_GPU_NAME); + } + + #[test] + fn the_pci_lines_examine_57_and_58_plant_read_as_their_scenarios_assume() { + // Scenarios examine-57 and examine-58 plant these constants beside a + // kernel node. examine-57 shows the kernel's target replacing the name + // table's only if the table reads its line as something else, and + // examine-58 shows the kernel's target filling a gap only if the table + // reads its line as nothing, so both premises are pinned here. + let phoenix = TEST_FIXTURE_LSPCI_LINE_NAME_TABLE_READS_AS_GFX1103; + assert!(is_lspci_gpu_line(phoenix)); + assert_eq!( + classify_amd_marketing_name(&extract_lspci_name(phoenix)), + ("gfx1103".to_owned(), true) + ); + let unnamed = TEST_FIXTURE_LSPCI_LINE_NAME_TABLE_READS_AS_NOTHING; + assert!(is_lspci_gpu_line(unnamed)); + assert_eq!( + classify_amd_marketing_name(&extract_lspci_name(unnamed)), + (String::new(), false) + ); + } + + #[test] + fn a_matched_agents_verdict_is_its_memory_properties_reading_or_else_the_target_rule() { + // With a `Memory Properties:` line, the runtime's reading of the device + // decides -- except that a flag without `APU` does not outrank an + // APU-family target (the entry at 0000:0a:00.0, the second flag row of + // the table at `apu_verdict_from_target`). Without one (a `rocminfo` that does not print + // it), the target rule applies to the target the entry ends with: an + // entry already integrated (a codename's verdict, with the kernel's + // gfx909, which the APU family list lacks) stays integrated, and one + // with no verdict keeps none. + let entry = |pci_id: &str, gfx: &str, is_apu: Option| Gpu { + name: "AMD GPU".to_owned(), + gfx_target: gfx.to_owned(), + pci_id: pci_id.to_owned(), + is_apu, + is_amd: true, + }; + let mut e = Examination { + gpus: vec![ + entry("0000:03:00.0", "gfx909", Some(true)), + entry("0000:04:00.0", "gfx909", None), + entry("0000:05:00.0", "gfx909", None), + entry("0000:06:00.0", "gfx1100", Some(true)), + // A held gfx1100 the scan called discrete, and an agent at the + // same address reporting gfx1103: the target stays gfx1100, + // and with no `Memory Properties:` line the verdict follows + // that kept target, not the agent's. + entry("0000:07:00.0", "gfx1100", Some(false)), + // The same, with the agent's flag: the flag wins. + entry("0000:08:00.0", "gfx1100", Some(false)), + // An empty target filled from an agent with no flag: the + // target rule reads the filled APU-family target as integrated + // over the scan's default `false`. + entry("0000:09:00.0", "", Some(false)), + // A held gfx1151 whose agent prints `Memory Properties:` with + // no `APU`: the family list outranks the flag. + entry("0000:0a:00.0", "gfx1151", None), + ], + ..Examination::default() + }; + apply_rocminfo_gpu_agents( + &mut e, + &rocminfo_listing_with_memory(&[ + ("gfx909", "AMD Radeon Vega 8", Some(0x300), None), + ("gfx909", "AMD Radeon Vega 8", Some(0x400), None), + ("gfx909", "AMD Radeon Vega 8", Some(0x500), Some("APU")), + ("gfx1100", "AMD Radeon RX 7900 XTX", Some(0x600), Some("")), + ("gfx1103", "AMD Radeon 780M Graphics", Some(0x700), None), + ( + "gfx1103", + "AMD Radeon 780M Graphics", + Some(0x800), + Some("APU"), + ), + ("gfx1151", "AMD Radeon 8060S Graphics", Some(0x900), None), + ( + "gfx1151", + "AMD Radeon 8060S Graphics", + Some(0xa00), + Some(""), + ), + ]), + ); + assert_eq!(gpu_at(&e, "0000:03:00.0").is_apu, Some(true)); + assert_eq!(gpu_at(&e, "0000:04:00.0").is_apu, None); + assert_eq!(gpu_at(&e, "0000:05:00.0").is_apu, Some(true)); + assert_eq!( + gpu_at(&e, "0000:06:00.0").is_apu, + Some(false), + "a runtime reading of discrete replaces the scan's guess" + ); + assert_eq!(gpu_at(&e, "0000:07:00.0").gfx_target, "gfx1100"); + assert_eq!(gpu_at(&e, "0000:07:00.0").is_apu, Some(false)); + assert_eq!(gpu_at(&e, "0000:08:00.0").gfx_target, "gfx1100"); + assert_eq!(gpu_at(&e, "0000:08:00.0").is_apu, Some(true)); + assert_eq!(gpu_at(&e, "0000:09:00.0").gfx_target, "gfx1151"); + assert_eq!(gpu_at(&e, "0000:09:00.0").is_apu, Some(true)); + assert_eq!( + gpu_at(&e, "0000:0a:00.0").is_apu, + Some(true), + "a flag without `APU` does not outrank an APU-family target" + ); + } + + #[test] + fn gpus_on_one_bus_match_by_device_and_function_too() { + // Two cards on bus 03, told apart only by device (03:00.0, 03:01.0) + // and two functions of one device (04:00.0, 04:00.1). Each agent must + // land on its own entry, so the match compares the whole + // bus:device.function and not the bus alone. + let entry = |pci_id: &str| Gpu { + name: KERNEL_TOPOLOGY_GPU_NAME.to_owned(), + gfx_target: String::new(), + pci_id: pci_id.to_owned(), + is_apu: None, + is_amd: true, + }; + let mut e = Examination { + gpus: vec![ + entry("0000:03:00.0"), + entry("0000:03:01.0"), + entry("0000:04:00.0"), + entry("0000:04:00.1"), + ], + ..Examination::default() + }; + apply_rocminfo_gpu_agents( + &mut e, + &rocminfo_listing(&[ + ("gfx1101", "Device one", Some(0x308)), + ("gfx1100", "Device zero", Some(0x300)), + ("gfx1102", "Function one", Some(0x401)), + ("gfx1103", "Function zero", Some(0x400)), + ]), + ); + let got: Vec<(&str, &str, &str)> = e + .gpus + .iter() + .map(|g| (g.pci_id.as_str(), g.name.as_str(), g.gfx_target.as_str())) + .collect(); + assert_eq!( + got, + vec![ + ("0000:03:00.0", "Device zero", "gfx1100"), + ("0000:03:01.0", "Device one", "gfx1101"), + ("0000:04:00.0", "Function zero", "gfx1103"), + ("0000:04:00.1", "Function one", "gfx1102"), + ] + ); + assert!(e.notes.is_empty(), "{:#?}", e.notes); + } + + #[test] + fn an_nvidia_card_beside_amd_ones_is_never_given_a_rocminfo_reading() { + // `rocminfo` lists AMD agents only, so an NVIDIA entry is never a + // candidate. Here it sits at the AMD card's bus address in another + // PCI domain (a multi-segment host): counted, it would make the + // address ambiguous across domains and the AMD card would lose its + // reading. + let gpu = |pci_id: &str, name: &str, is_amd: bool| Gpu { + name: name.to_owned(), + gfx_target: String::new(), + pci_id: pci_id.to_owned(), + is_apu: Some(false), + is_amd, + }; + let mut e = Examination { + gpus: vec![ + gpu("0001:03:00.0", "NVIDIA Corporation Device", false), + gpu("0000:03:00.0", KERNEL_TOPOLOGY_GPU_NAME, true), + ], + ..Examination::default() + }; + apply_rocminfo_gpu_agents( + &mut e, + &rocminfo_listing(&[("gfx1100", "AMD Radeon RX 7900 XTX", Some(AT_03))]), + ); + assert_eq!(gpu_at(&e, "0001:03:00.0").name, "NVIDIA Corporation Device"); + assert_eq!(gpu_at(&e, "0001:03:00.0").gfx_target, ""); + assert_eq!(gpu_at(&e, "0000:03:00.0").name, "AMD Radeon RX 7900 XTX"); + assert_eq!(gpu_at(&e, "0000:03:00.0").gfx_target, "gfx1100"); + assert!(e.notes.is_empty(), "{:#?}", e.notes); + } + + #[test] + fn an_agent_without_a_device_type_after_a_gpu_agent_is_not_a_gpu() { + // The GPU verdict is per agent: an agent that prints no + // `Device Type:` line must not inherit the previous agent's. + let listing = "Agent 1\n Name: gfx1100\n Device Type: GPU\n BDFID: 768\n\ + Agent 2\n Name: gfx1101\n BDFID: 1024\n"; + let agents = parse_rocminfo_gpu_agents(listing); + assert_eq!( + agents + .iter() + .map(|a| a.gfx_target.as_str()) + .collect::>(), + vec!["gfx1100"] + ); + } + + #[test] + fn memory_properties_lines_read_as_the_real_tool_prints_them() { + // From the capture: `Memory Properties: APU` on the gfx1151, and + // the same label with nothing after it on its CPU and NPU agents; + // AMD's rocminfo documentation shows the empty form for an MI300X. + assert!(parse_rocminfo_memory_properties(" APU")); + assert!(!parse_rocminfo_memory_properties("")); + assert!(!parse_rocminfo_memory_properties(" ")); + let agents = parse_rocminfo_gpu_agents(&rocminfo_listing_with_memory(&[ + ("gfx942", "AMD Instinct MI300X", Some(AT_03), Some("")), + ( + "gfx1151", + "AMD Radeon 8060S Graphics", + Some(AT_04), + Some("APU"), + ), + ("gfx1100", "AMD Radeon RX 7900 XTX", Some(AT_C4), None), + ])); + assert_eq!( + agents.iter().map(|a| a.is_apu).collect::>(), + vec![Some(false), Some(true), None] + ); + } + + #[test] + fn with_no_amd_gpu_listed_each_agent_is_listed_with_its_own_verdict() { + // Row (g): with no AMD GPU listed there is nothing to confuse an agent + // with, so every agent is listed as an AMD GPU of its own, with or + // without an address, under its own name and target. Its verdict is + // `apu_verdict_with_runtime_flag` on its own target, from no prior + // verdict. + let mut e = Examination::default(); + apply_rocminfo_gpu_agents( + &mut e, + &rocminfo_listing_with_memory(&[ + // APU family, no flag, and no address: the target rule says + // integrated. + ("gfx1151", "AMD Radeon 8060S Graphics", None, None), + // Flagged `APU` (gfx909 is not on the family list). + ("gfx909", "AMD Radeon Vega 8", Some(AT_04), Some("APU")), + // Outside the families, no flag: no claim. + ("gfx1100", "AMD Radeon RX 7900 XTX", Some(AT_C4), None), + ]), + ); + let verdict = |gfx: &str| { + e.gpus + .iter() + .find(|g| g.gfx_target == gfx) + .unwrap_or_else(|| panic!("{gfx}: {:#?}", e.gpus)) + .is_apu + }; + assert_eq!(e.gpus.len(), 3, "{:#?}", e.gpus); + assert_eq!(verdict("gfx1151"), Some(true)); + assert_eq!(verdict("gfx909"), Some(true)); + assert_eq!(verdict("gfx1100"), None); + let names: Vec<(&str, &str)> = e + .gpus + .iter() + .map(|g| (g.gfx_target.as_str(), g.name.as_str())) + .collect(); + assert_eq!( + names, + vec![ + ("gfx1151", "AMD Radeon 8060S Graphics"), + ("gfx909", "AMD Radeon Vega 8"), + ("gfx1100", "AMD Radeon RX 7900 XTX"), + ] + ); + assert!(e.gpus.iter().all(|g| g.is_amd), "{:#?}", e.gpus); + assert!(e.notes.is_empty(), "{:#?}", e.notes); + summarise_gpu_categories(&mut e); + assert!(e.has_amd_gpu, "a GPU only rocminfo found still counts"); + } + + #[test] + fn agents_without_an_address_are_never_paired_by_position() { + // Row (d). Without a `BDFID` nothing identifies which card an agent + // is, so no shape of listing pairs it: not equal counts, not targets + // that agree in order (in any case), not one agent for two cards, not + // more agents than cards, not targets that disagree. Each time every + // entry keeps its own kernel target and no model name, and the note + // names every reading it set aside. + let cases: &[(&str, &[(&str, &str)])] = &[ + ( + "equal-counts-targets-agree", + &[ + ("gfx1100", "AMD Radeon RX 7900 XTX"), + ("gfx1101", "AMD Radeon RX 7800 XT"), + ], + ), + ( + "equal-counts-targets-swapped", + &[ + ("gfx1101", "AMD Radeon RX 7800 XT"), + ("gfx1100", "AMD Radeon RX 7900 XTX"), + ], + ), + ( + "one-agent-two-cards", + &[("gfx1101", "AMD Radeon RX 7800 XT")], + ), + ( + "more-agents-than-cards", + &[ + ("gfx1100", "AMD Radeon RX 7900 XTX"), + ("gfx1101", "AMD Radeon RX 7800 XT"), + ("gfx1102", "AMD Radeon RX 7600"), + ], + ), + ]; + for (tag, agents) in cases { + let listing = rocminfo_listing( + &agents + .iter() + .map(|&(gfx, name)| (gfx, name, None)) + .collect::>(), + ); + let e = examine_planted_host( + &format!("rocminfo-no-bdfid-{tag}"), + &[(AT_03, 110_000), (AT_04, 110_001)], + &listing, + ); + assert_eq!(e.gpus.len(), 2, "{tag}: {:#?}", e.gpus); + for (pci_id, own_target) in [("0000:03:00.0", "gfx1100"), ("0000:04:00.0", "gfx1101")] { + let gpu = gpu_at(&e, pci_id); + assert_eq!(gpu.name, KERNEL_TOPOLOGY_GPU_NAME, "{tag} {pci_id}"); + assert_eq!(gpu.gfx_target, own_target, "{tag} {pci_id}"); + assert_eq!(gpu.is_apu, None, "{tag} {pci_id}: no packaging claim"); + } + let notes = unattributed_notes(&e); + assert_eq!(notes.len(), 1, "{tag}: {:#?}", e.notes); + assert!( + notes[0].starts_with(&format!("{} GPU reading(s) from rocminfo", agents.len())), + "{tag}: the note counts every reading it set aside: {notes:?}" + ); + for (gfx, _) in *agents { + assert!( + notes[0].contains(&format!("{gfx} (no bus address)")), + "{tag}: {notes:?}" + ); + } + // The claim, held against the entries above keeping their own + // targets and no verdict: rocminfo's reading was not applied. + assert!( + notes[0].contains( + "model names, gfx targets and integrated/discrete verdicts for them are not \ + applied to any GPU" + ), + "{tag}: {notes:?}" + ); + } + } + + #[test] + fn the_partitions_of_one_card_pair_within_that_card() { + // A partitioned Instinct (GC 9.4.3). Per upstream kernel and ROCm + // sources, not verified on hardware here, the kernel puts each + // partition's id into `location_id` and the runtime reports the same + // value as `BDFID`, so partitions arrive at addresses of their own -- + // 4352, 4353, 4354 are 11:00.0, 11:00.1, 11:00.2 -- and match one to + // one like separate cards. The agents are named apart and listed out + // of order, so a pairing by position shows. + let e = examine_planted_host( + "rocminfo-partitions", + &[(4352, 90402), (4353, 90402), (4354, 90402)], + &rocminfo_listing(&[ + ("gfx942", "MI300X partition 2", Some(4354)), + ("gfx942", "MI300X partition 0", Some(4352)), + ("gfx942", "MI300X partition 1", Some(4353)), + ]), + ); + let names: Vec<(&str, &str)> = e + .gpus + .iter() + .map(|gpu| (gpu.pci_id.as_str(), gpu.name.as_str())) + .collect(); + assert_eq!( + names, + vec![ + ("0000:11:00.0", "MI300X partition 0"), + ("0000:11:00.1", "MI300X partition 1"), + ("0000:11:00.2", "MI300X partition 2"), + ], + "{:#?}", + e.gpus + ); + assert!(e.notes.is_empty(), "{:#?}", e.notes); + + // The fallback for entries that do share one address: paired with + // that address's agents in order (case (a)). + let shared = examine_planted_host( + "rocminfo-partitions-shared", + &[(AT_03, 90402), (AT_03, 90402)], + &rocminfo_listing(&[ + ("gfx942", "MI300X shared 0", Some(AT_03)), + ("gfx942", "MI300X shared 1", Some(AT_03)), + ]), + ); + assert_eq!( + shared + .gpus + .iter() + .map(|g| g.name.as_str()) + .collect::>(), + vec!["MI300X shared 0", "MI300X shared 1"] + ); + assert!(shared.notes.is_empty(), "{:#?}", shared.notes); + + // Case (e): one reading too many at an address. Its entry is already + // taken, so the extra is noted, not added, and the entry keeps the + // first reading. + let extra = examine_planted_host( + "rocminfo-partitions-extra", + &[(AT_03, 90402)], + &rocminfo_listing(&[ + ("gfx942", "AMD Instinct MI300X", Some(AT_03)), + ("gfx942", "AMD Instinct MI300X", Some(AT_03)), + ]), + ); + assert_eq!(extra.gpus.len(), 1, "{:#?}", extra.gpus); + assert_eq!(extra.gpus[0].name, "AMD Instinct MI300X"); + let notes = unattributed_notes(&extra); + assert_eq!(notes.len(), 1, "{:#?}", extra.notes); + assert!( + notes[0].contains("more rocminfo readings at this address than listed GPUs"), + "{notes:?}" + ); + } } diff --git a/crates/rocm-core/src/host_gpu.rs b/crates/rocm-core/src/host_gpu.rs index 9a722f8f5e..f4bcd15dae 100644 --- a/crates/rocm-core/src/host_gpu.rs +++ b/crates/rocm-core/src/host_gpu.rs @@ -71,6 +71,7 @@ use linux::detect_linux_amdgpu_driver_version; use linux::detect_linux_primary_gpu_name; pub(crate) use linux::detect_linux_sysfs_gfx_target; pub use linux::is_amdgpu_device; +pub(crate) use linux::pci_bus_device_function; #[cfg(any(target_os = "linux", test))] pub(crate) use linux::{KfdGpuNode, kfd_gpu_nodes_in}; pub(crate) use windows_inventory::detect_windows_amd_driver_info; diff --git a/crates/rocm-core/src/host_gpu/linux.rs b/crates/rocm-core/src/host_gpu/linux.rs index 701071d859..817d996234 100644 --- a/crates/rocm-core/src/host_gpu/linux.rs +++ b/crates/rocm-core/src/host_gpu/linux.rs @@ -133,7 +133,9 @@ pub(crate) fn kfd_gpu_nodes_in(nodes_dir: &Path) -> Option> { /// 12032, 17920, 23808, 35584, 43520, 49664 and 55808, which decode to exactly /// the eight addresses `lspci -D` lists for its accelerators (EAI-8449). /// -/// `None` when the property is absent or zero. Zero is refused rather than +/// `None` when the property is absent, zero, or wider than 16 bits (which +/// `pci_dev_id()` cannot produce; see [`pci_bus_device_function`]). Zero is +/// refused rather than /// decoded: `0000:00:00.0` is the host bridge, so emitting it would be a /// wrong-but-plausible address that could match an unrelated PCI entry. #[cfg(any(target_os = "linux", test))] @@ -141,16 +143,39 @@ fn kfd_node_pci_id(properties: &str) -> Option { let location = kfd_property_value(properties, "location_id")? .parse::() .ok()?; - if location == 0 { - return None; - } let domain = kfd_property_value(properties, "domain") .and_then(|value| value.parse::().ok()) .unwrap_or(0); - let bus = (location >> 8) & 0xff; - let device = (location >> 3) & 0x1f; - let function = location & 0x7; - Some(format!("{domain:04x}:{bus:02x}:{device:02x}.{function}")) + let bdf = pci_bus_device_function(location)?; + Some(format!("{domain:04x}:{bdf}")) +} + +/// The `bus:device.function` part of a PCI address, spelled as `lspci` spells +/// it, from the kernel's `pci_dev_id()` encoding: `(bus << 8) | devfn`, with +/// `devfn` packing the device number in bits 3..8 and the function in bits +/// 0..3. So `4352` (`0x1100`) is `11:00.0`. +/// +/// Two readings carry an address in this encoding, and both decode through +/// here so they cannot disagree about a card: the KFD topology's `location_id` +/// (`kfd_node_pci_id`, which adds the `domain` property in front) and +/// `rocminfo`'s per-agent `BDFID`, which has no domain at all +/// (`examine::linux_gpus`, which matches each `rocminfo` agent onto the GPU +/// with this address). +/// +/// `None` for zero, for the reason `kfd_node_pci_id` gives: `00:00.0` is the +/// host bridge, and `rocminfo` prints `BDFID: 0` for its CPU agents, so a zero +/// is "no address" rather than an address that could match an unrelated entry. +/// `None` above `0xffff` too, which the encoding cannot produce. +pub(crate) fn pci_bus_device_function(dev_id: u32) -> Option { + // Above 16 bits is no `pci_dev_id()`: decoding only the low 16 would turn + // it into an address that could belong to another card. + if dev_id == 0 || dev_id > 0xffff { + return None; + } + let bus = (dev_id >> 8) & 0xff; + let device = (dev_id >> 3) & 0x1f; + let function = dev_id & 0x7; + Some(format!("{bus:02x}:{device:02x}.{function}")) } /// The KFD-topology read, against a caller-supplied nodes directory. @@ -694,6 +719,23 @@ mod tests { assert_eq!(kfd_node_pci_id("gfx_target_version 90402\n"), None); } + #[test] + fn a_dev_id_decodes_every_function_bit_and_refuses_zero_and_wider_values() { + // The decode both the KFD topology and `rocminfo`'s `BDFID` go through, + // pinned to literal addresses: a test that only compares the two sides + // stays green when the shared decode is wrong on both. Functions 5 and + // 7 use the function's high bit, which a two-bit mask would drop. + assert_eq!(pci_bus_device_function(0x1c7).as_deref(), Some("01:18.7")); + assert_eq!(pci_bus_device_function(0x305).as_deref(), Some("03:00.5")); + assert_eq!(pci_bus_device_function(49_920).as_deref(), Some("c3:00.0")); + assert_eq!(pci_bus_device_function(0xfffe).as_deref(), Some("ff:1f.6")); + assert_eq!(pci_bus_device_function(0), None); + // Past 16 bits is not this encoding; its low 16 bits (0x0300, a real + // card's address) must not be read as one. + assert_eq!(pci_bus_device_function(0x1_0300), None); + assert_eq!(pci_bus_device_function(0xffff).as_deref(), Some("ff:1f.7")); + } + #[test] fn kfd_properties_win_over_a_standalone_file() -> Result<()> { // Both present: `properties` is what the kernel maintains, so it decides. diff --git a/crates/rocm-core/tests/fixtures/rocminfo_gfx1151_apu.txt b/crates/rocm-core/tests/fixtures/rocminfo_gfx1151_apu.txt new file mode 100644 index 0000000000..2875db8485 --- /dev/null +++ b/crates/rocm-core/tests/fixtures/rocminfo_gfx1151_apu.txt @@ -0,0 +1,257 @@ +ROCk module is loaded +===================== +HSA System Attributes +===================== +Runtime Version: 1.21 +Runtime Ext Version: 1.32 +System Timestamp Freq.: 1000.000000MHz +Sig. Max Wait Duration: 18446744073709551615 (0xFFFFFFFFFFFFFFFF) (timestamp count) +Machine Model: LARGE +System Endianness: LITTLE +Mwaitx: ENABLED +XNACK enabled: NO +DMAbuf Support: YES +VMM Support: YES +Fabric Support: NO + +========== +HSA Agents +========== +******* +Agent 1 +******* + Name: AMD RYZEN AI MAX+ PRO 395 w/ Radeon 8060S + Uuid: CPU-XX + Marketing Name: AMD RYZEN AI MAX+ PRO 395 w/ Radeon 8060S + Vendor Name: CPU + Feature: None specified + Profile: FULL_PROFILE + Float Round Mode: NEAR + Max Queue Number: 0(0x0) + Queue Min Size: 0(0x0) + Queue Max Size: 0(0x0) + Queue Type: MULTI + Node: 0 + Device Type: CPU + Cache Info: + L1: 49152(0xc000) KB + Chip ID: 0(0x0) + ASIC Revision: 0(0x0) + Cacheline Size: 64(0x40) + Max Clock Freq. (MHz): 5187 + BDFID: 0 + Internal Node ID: 0 + Compute Unit: 32 + SIMDs per CU: 0 + Shader Engines: 0 + Shader Arrs. per Eng.: 0 + WatchPts on Addr. Ranges:1 + Memory Properties: + Features: None + Pool Info: + Pool 1 + Segment: GLOBAL; FLAGS: FINE GRAINED + Size: 98634044 (0x5e1093c) KB + Allocatable: TRUE + Alloc Granule: 4KB + Alloc Recommended Granule:4KB + Alloc Alignment: 4KB + Accessible by all: TRUE + Pool 2 + Segment: GLOBAL; FLAGS: EXTENDED FINE GRAINED + Size: 98634044 (0x5e1093c) KB + Allocatable: TRUE + Alloc Granule: 4KB + Alloc Recommended Granule:4KB + Alloc Alignment: 4KB + Accessible by all: TRUE + Pool 3 + Segment: GLOBAL; FLAGS: KERNARG, FINE GRAINED + Size: 98634044 (0x5e1093c) KB + Allocatable: TRUE + Alloc Granule: 4KB + Alloc Recommended Granule:4KB + Alloc Alignment: 4KB + Accessible by all: TRUE + Pool 4 + Segment: GLOBAL; FLAGS: COARSE GRAINED + Size: 98634044 (0x5e1093c) KB + Allocatable: TRUE + Alloc Granule: 4KB + Alloc Recommended Granule:4KB + Alloc Alignment: 4KB + Accessible by all: TRUE + ISA Info: +******* +Agent 2 +******* + Name: gfx1151 + Uuid: GPU-XX + Marketing Name: AMD Radeon 8060S Graphics + Vendor Name: AMD + Feature: KERNEL_DISPATCH + Profile: BASE_PROFILE + Float Round Mode: NEAR + Max Queue Number: 128(0x80) + Queue Min Size: 64(0x40) + Queue Max Size: 131072(0x20000) + Queue Type: MULTI + Node: 1 + Device Type: GPU + Cache Info: + L1: 32(0x20) KB + L2: 2048(0x800) KB + L3: 32768(0x8000) KB + Chip ID: 5510(0x1586) + ASIC Revision: 0(0x0) + Cacheline Size: 128(0x80) + Max Clock Freq. (MHz): 2900 + BDFID: 49920 + Internal Node ID: 1 + Compute Unit: 40 + SIMDs per CU: 2 + Shader Engines: 2 + Shader Arrs. per Eng.: 2 + WatchPts on Addr. Ranges:4 + Coherent Host Access: FALSE + Memory Properties: APU + Features: KERNEL_DISPATCH + Fast F16 Operation: TRUE + Wavefront Size: 32(0x20) + Workgroup Max Size: 1024(0x400) + Workgroup Max Size per Dimension: + x 1024(0x400) + y 1024(0x400) + z 1024(0x400) + Max Waves Per CU: 32(0x20) + Max Work-item Per CU: 1024(0x400) + Grid Max Size: 2147483647(0x7fffffff) + Grid Max Size per Dimension: + x 2147483647(0x7fffffff) + y 65535(0xffff) + z 65535(0xffff) + Max fbarriers/Workgrp: 32 + Packet Processor uCode:: 34 + SDMA engine uCode:: 18 + IOMMU Support:: None + Pool Info: + Pool 1 + Segment: GLOBAL; FLAGS: COARSE GRAINED + Size: 49317020 (0x2f0849c) KB + Allocatable: TRUE + Alloc Granule: 4KB + Alloc Recommended Granule:2048KB + Alloc Alignment: 4KB + Accessible by all: FALSE + Pool 2 + Segment: GLOBAL; FLAGS: EXTENDED FINE GRAINED + Size: 49317020 (0x2f0849c) KB + Allocatable: TRUE + Alloc Granule: 4KB + Alloc Recommended Granule:2048KB + Alloc Alignment: 4KB + Accessible by all: FALSE + Pool 3 + Segment: GROUP + Size: 64 (0x40) KB + Allocatable: FALSE + Alloc Granule: 0KB + Alloc Recommended Granule:0KB + Alloc Alignment: 0KB + Accessible by all: FALSE + ISA Info: + ISA 1 + Name: amdgcn-amd-amdhsa--gfx1151 + Machine Models: HSA_MACHINE_MODEL_LARGE + Profiles: HSA_PROFILE_BASE + Default Rounding Mode: NEAR + Default Rounding Mode: NEAR + Fast f16: TRUE + Workgroup Max Size: 1024(0x400) + Workgroup Max Size per Dimension: + x 1024(0x400) + y 1024(0x400) + z 1024(0x400) + Grid Max Size: 18446744073709551615 (0xffffffffffffffff) + Grid Max Size per Dimension: + x 2147483647(0x7fffffff) + y 65535(0xffff) + z 65535(0xffff) + FBarrier Max Size: 32 + ISA 2 + Name: amdgcn-amd-amdhsa--gfx11-generic + Machine Models: HSA_MACHINE_MODEL_LARGE + Profiles: HSA_PROFILE_BASE + Default Rounding Mode: NEAR + Default Rounding Mode: NEAR + Fast f16: TRUE + Workgroup Max Size: 1024(0x400) + Workgroup Max Size per Dimension: + x 1024(0x400) + y 1024(0x400) + z 1024(0x400) + Grid Max Size: 18446744073709551615 (0xffffffffffffffff) + Grid Max Size per Dimension: + x 2147483647(0x7fffffff) + y 65535(0xffff) + z 65535(0xffff) + FBarrier Max Size: 32 +******* +Agent 3 +******* + Name: aie2p + Uuid: AIE-XX + Marketing Name: RyzenAI-npu5 + Vendor Name: AMD + Feature: AGENT_DISPATCH + Profile: BASE_PROFILE + Float Round Mode: NEAR + Max Queue Number: 1(0x1) + Queue Min Size: 64(0x40) + Queue Max Size: 64(0x40) + Queue Type: SINGLE + Node: 0 + Device Type: DSP + Cache Info: + L2: 2048(0x800) KB + L3: 32768(0x8000) KB + Chip ID: 0(0x0) + ASIC Revision: 0(0x0) + Cacheline Size: 0(0x0) + Max Clock Freq. (MHz): 0 + BDFID: 0 + Internal Node ID: 0 + Compute Unit: 0 + SIMDs per CU: 0 + Shader Engines: 0 + Shader Arrs. per Eng.: 0 + WatchPts on Addr. Ranges:0 + Memory Properties: + Features: AGENT_DISPATCH + Pool Info: + Pool 1 + Segment: GLOBAL; FLAGS: KERNARG, COARSE GRAINED + Size: 98634044 (0x5e1093c) KB + Allocatable: TRUE + Alloc Granule: 4KB + Alloc Recommended Granule:4KB + Alloc Alignment: 4KB + Accessible by all: TRUE + Pool 2 + Segment: GLOBAL; FLAGS: COARSE GRAINED + Size: 65536 (0x10000) KB + Allocatable: TRUE + Alloc Granule: 4KB + Alloc Recommended Granule:0KB + Alloc Alignment: 4KB + Accessible by all: TRUE + Pool 3 + Segment: GLOBAL; FLAGS: COARSE GRAINED + Size: 98634044 (0x5e1093c) KB + Allocatable: TRUE + Alloc Granule: 4KB + Alloc Recommended Granule:4KB + Alloc Alignment: 4KB + Accessible by all: TRUE + ISA Info: +*** Done *** diff --git a/docs/host-probing.md b/docs/host-probing.md index e6b6b3342f..e12485f424 100644 --- a/docs/host-probing.md +++ b/docs/host-probing.md @@ -198,13 +198,15 @@ Verdict: confirmed duplicate, but not equivalent. The two cover different names. | Answers | a gfx target, for the discrete RDNA2 to RDNA4 parts and for APU names | a gfx target only for Strix Halo, Ryzen AI Max, the Radeon 8045S, 8050S and 8060S, 880M, 890M, Strix Point, Krackan, the Radeon 740M, 760M and 780M, Phoenix and Hawk Point; for the other APU codenames (Rembrandt, Raphael, Barcelo, Lucienne, Renoir, Cezanne) only "is an APU" with an empty target | | Matching | whole tokens, after every non-alphanumeric character becomes a space | substring, after removing `(tm)`, `(r)`, `(c)` | | APU verdict | none (the caller derives it from the target with `gfx_is_apu_family`) | yes, from the name | -| Used on | Windows adapters and the WSL host's display probe | Windows adapters after the PNP id, and the Linux `lspci` scan (`probe_gpus_lspci`), which calls it directly with no PCI-id lookup first | +| Used on | Windows adapters and the WSL host's display probe | Windows adapters after the PNP id, and the Linux `lspci` scan (`probe_gpus_lspci`), which calls it directly with no PCI-id lookup first; on Linux a KFD node at the same address then replaces or fills that target | The two agree where they overlap on `880m`, `780m` and `8060s`. They disagree on Krackan: `classify_amd_marketing_name` returns `gfx1150` for the token `krackan`, while the table maps `860m` and `840m` to `gfx1152`, and `gfx_target_from_amd_pci_device_id` maps device `1114` to `gfx1152`. Which is right for a given part is UNCONFIRMED here (no hardware); the open pull request [#521](https://github.com/ROCm/rocm-cli/pull/521) reaches the same disagreement and resolves it toward `gfx1152`. -What a user can see, on Windows without a HIP SDK: an RX 7900 XTX is not in the PCI-id table and not in `classify_amd_marketing_name`, so `gpus[].gfx_target` is empty while `summary.detected_gfx_target` in the same JSON document is `gfx1100`, from the name table. (Derived from the code; `hipInfo` fills the examination's value in when an SDK is installed.) On native Linux only the examination's `lspci` path can show a name-derived empty target (see the host table). +What a user can see, on Windows without a HIP SDK: an RX 7900 XTX is not in the PCI-id table and not in `classify_amd_marketing_name`, so `gpus[].gfx_target` is empty while `summary.detected_gfx_target` in the same JSON document is `gfx1100`, from the name table. (Derived from the code; `hipInfo` fills the examination's value in when an SDK is installed.) On native Linux the `lspci` scan's target, empty or guessed from the name table, is not final: a KFD node at the same address fills it or replaces it, and a `rocminfo` agent matched to the entry by its `BDFID` only fills a target that is still empty, because the runtime's reading can carry an `HSA_OVERRIDE_GFX_VERSION` and the kernel's cannot. An agent that prints no `BDFID` is never paired with an entry; it is reported in a note instead. `gpus[].is_apu` is `true` or `false` where the ROCm runtime's own reading, the kernel's or the runtime's target, or the PCI scan's name gives a verdict, and `null` where none does. An entry that has no verdict yet does not get `false` from a target outside the APU families, because that family list does not name every APU. The rules, by path, are the table in the doc of `apu_verdict_from_target` in [linux_gpus.rs](../crates/rocm-core/src/examine/linux_gpus.rs), and the doc above that table gives the reasons for them. -Proposal: the table survives. It is pure, data-driven and already the superset for target names. It gains the codenames `classify_amd_marketing_name` knows, and the APU verdict is derived from the target, falling back to a codename list only while some codename still has no target. [#521](https://github.com/ROCm/rocm-cli/pull/521) already proposes exactly this convergence and is open and conflicting with `main`; the cheapest route is to finish and rebase it rather than start a second change. Its Linux device-id work is a separate concern from this pair. Hosts: the table is pure data, so its test needs no host, but the two callers of `classify_amd_marketing_name` differ in what happens around it (on Windows `parse_gpus_windows_output` tries the PNP id first; on native Linux `probe_gpus_lspci` calls it directly, and the later `probe_gpus_after_lspci` steps (the kernel topology, `rocminfo` and the sysfs fallback) may also set a target, which this note did not trace; WSL has no name lookup in the examination at all), so the characterisation test names each caller: a fixture adapter list through `parse_gpus_windows_output`, which a Linux unit test can drive, and a fixture `lspci` output through the Linux scan. +Two residuals follow from these rules. A `rocminfo` agent can fill an empty target under `HSA_OVERRIDE_GFX_VERSION` when no KFD node exists to fill it first, so on such a host the reported target, and an APU verdict decided on it, can be the override's. And on a host with `lspci` but no readable KFD topology (some containers), a wrong name-table target is never corrected: `rocminfo` never replaces a held target, because its reading cannot be told apart from an overridden one. The unit test `rocminfo_never_replaces_a_name_table_guess_even_without_a_kernel_node` pins that. The cases are in [linux_gpus.rs](../crates/rocm-core/src/examine/linux_gpus.rs), at `apply_kernel_gpu_membership`, `apply_rocminfo_gpu_agents` and `apply_agent_to_gpu`. + +Proposal: the table survives. It is pure, data-driven and already the superset for target names. It gains the codenames `classify_amd_marketing_name` knows, and the APU verdict is derived from the target, falling back to a codename list only while some codename still has no target. [#521](https://github.com/ROCm/rocm-cli/pull/521) already proposes exactly this convergence and is open and conflicting with `main`; the cheapest route is to finish and rebase it rather than start a second change. Its Linux device-id work is a separate concern from this pair. Hosts: the table is pure data, so its test needs no host, but the two callers of `classify_amd_marketing_name` differ in what happens around it (on Windows `parse_gpus_windows_output` tries the PNP id first; on native Linux `probe_gpus_lspci` calls it directly, and the later `probe_gpus_after_lspci` steps change its target as described above (a KFD node fills or replaces the scan's guess, a `rocminfo` agent only fills an empty one), with the sysfs fallback adding an entry only when no AMD GPU is listed at all; WSL has no name lookup in the examination at all), so the characterisation test names each caller: a fixture adapter list through `parse_gpus_windows_output`, which a Linux unit test can drive, and a fixture `lspci` output through the Linux scan. ## PATH lookup diff --git a/docs/manual-testing.md b/docs/manual-testing.md index d7a49c7b8d..fc1e4c0d94 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -372,6 +372,32 @@ fallback paired an architecture line with the wrong GPU. On that machine `rocm diagnose` must **not** report `info-20-gpu-no-rocm-support` at all — the discrete card does have ROCm support, so the whole-machine claim is false. +On Linux, check that each GPU keeps its own identity when the ROCm runtime +cannot see all of them. On a host with two or more AMD GPUs and ROCm installed, +hide one card from the runtime and inspect: + +```bash +ROCR_VISIBLE_DEVICES=0 rocm examine --json +``` + +Match the `gpus[]` entries by `pci_id` against `lspci -D`. Every listed card +must keep its own `gfx_target`, name and `is_apu`. The hidden card keeps the +`gfx_target` the kernel gives it, and does **not** carry the visible card's +model name or verdict; it may show the kernel-topology placeholder name. A +hidden card carrying another card's name, target or `is_apu` means `rocminfo`'s +agents were paired with the wrong GPU. + +Then run `rocm examine --json` once with no override set, and once with +`HSA_OVERRIDE_GFX_VERSION` set to a value that names a different target from +every card's own (on a gfx1151 or gfx1103 host, `11.0.0`, which names gfx1100; +an override that names a card's own target cannot show anything). Every card's +`gfx_target` must be the same in both runs, the native one, and an integrated +GPU must still read `is_apu: true`. To read the native target from the kernel +instead, `grep gfx_target_version /sys/class/kfd/kfd/topology/nodes/*/properties` +gives a decimal value per GPU node: major is `v / 10000`, minor is +`(v / 100) % 100` and stepping is `v % 100`, with minor and stepping written as +hex digits, so `110001` is gfx1101 and `90010` is gfx90a. + ## 2b. Bare-Metal Driver Install On a real apt/dnf/zypper Linux host: diff --git a/docs/testing.md b/docs/testing.md index 5435b6db64..912bf982e3 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -5267,6 +5267,23 @@ Deleting the call in `install_wheel_runtime` is caught by the dead-code lint und RUSTC_WRAPPER= cargo xtask e2e -- -n 'examine-(49|50|51|52|53) ' ``` +`examine-55` plants a native Linux machine whose kernel topology has two GPUs, at `0000:03:00.0` (gfx1100) and `0000:04:00.0` (gfx1101), with an `lspci` stand-in that lists an empty bus and a `rocminfo` stand-in that lists only the second, as "AMD Radeon RX 7800 XT" with its `BDFID`: the ROCm runtime with the first GPU hidden from it. The stand-in prints each agent the way the real tool does, ending with an `ISA Info:` block whose entries have `Name:` lines of their own. So the scenario also requires the parser to skip those lines rather than take them for the agent's name. `rocm examine --json` must report the second GPU with that model name, its own gfx1101 and `is_apu: false` (the stand-in prints the empty `Memory Properties:` line a discrete card gets), and the first with gfx1100, no model name (the kernel-topology placeholder, explained by one "marketing name is unknown" note) and `is_apu: null`, since nothing that can see it gives a verdict (the rules are the table at `apu_verdict_from_target` in [linux_gpus.rs](../crates/rocm-core/src/examine/linux_gpus.rs)). So a reading the runtime gives for one GPU must not land on the other. It runs on the Linux lanes and needs no GPU. The other rows of the match (agents with no `BDFID`; an address no listed GPU has; an address shared across PCI domains; more readings at an address than listed GPUs; partitions at addresses of their own; agents listed with no AMD GPU to match) are unit tests in [linux_gpus.rs](../crates/rocm-core/src/examine/linux_gpus.rs), next to `apply_rocminfo_gpu_agents`, and `a_real_rocminfo_listing_yields_its_gpu_agent` parses a captured real listing, [rocminfo_gfx1151_apu.txt](../crates/rocm-core/tests/fixtures/rocminfo_gfx1151_apu.txt). The commands for examine-55 to examine-59 follow examine-59. + +`examine-56` plants a native Linux machine whose kernel topology has one gfx1103 integrated GPU at `0000:c4:00.0`, an `lspci` stand-in that lists an empty bus, and a `rocminfo` stand-in that reports that GPU at the same `BDFID` as "AMD Radeon 780M Graphics", with the architecture gfx1100 only while `HSA_OVERRIDE_GFX_VERSION` is 11.0.0 in its environment (as the real runtime does) and gfx1103 otherwise, and `Memory Properties: APU` either way (the runtime reads that from the device). The scenario sets the variable, and the stand-in leaves a marker file in the scenario directory when it runs with it; the Then step requires that marker, so the inspection's own `rocminfo` run must have seen the override, and a scenario that lost the variable fails instead of passing on the native listing. `rocm examine --json` must keep the card's own gfx1103 with `is_apu: true` (so `has_apu` is true and `has_discrete_amd` false) and take the runtime's model name: a `rocminfo` reading fills a gfx target only where the entry has none, so `gpus[].gfx_target` stays the native target that `rocm diagnose`'s override check holds against the framework's arch list. What it proves is that the overridden target does not replace the card's own; it does not prove where the integrated verdict comes from, because gfx1103 is an APU-family target, which gives `true` without the runtime's reading. The runtime's `Memory Properties` reading deciding the verdict is proved by the unit test `a_matched_agents_verdict_is_its_memory_properties_reading_or_else_the_target_rule`. It runs on the Linux lanes and needs no GPU; the unit test behind the kept target is `an_overridden_runtime_target_does_not_replace_the_cards_own`, and the command after examine-59 runs both. + +`examine-57` plants a native Linux machine whose `lspci` stand-in lists a "Phoenix1" at `0000:05:00.0`, which the examination's name table reads as gfx1103, an integrated GPU, and whose kernel topology has a gfx1100 node at that address. `rocm examine --json` must list the one GPU with the kernel's gfx1100 and `is_apu: false` (so `has_apu` is false and `has_discrete_amd` true): the kernel's target replaces a target the PCI scan guessed from the name, and the verdict the scan derived with that guess goes with it (the verdict rules are the table at `apu_verdict_from_target` in [linux_gpus.rs](../crates/rocm-core/src/examine/linux_gpus.rs)). Krackan Point is where the tables themselves disagree (the name table gfx1150, the device-id map gfx1152, which is right unconfirmed). The disagreement is planted rather than taken from a real part so that it does not depend on one table entry. It runs on the Linux lanes and needs no GPU; the unit test behind it is `the_kernels_target_fills_or_replaces_the_scans_and_the_verdict_follows_the_rules`. + +`examine-58` plants a native Linux machine whose `lspci` stand-in lists a nameless "Device 7fff" at `0000:c3:00.0`, an id no table knows, so the scan has no target for it and calls it discrete by default; its kernel topology has a gfx1151 node at that address, and its `rocminfo` stand-in lists no GPU. `rocm examine --json` must list the one GPU with gfx1151 and `is_apu: true` (so `has_apu` is true and `has_discrete_amd` false), the shape of a Strix Halo whose `pci.ids` has no name for it. It runs on the Linux lanes and needs no GPU; the unit test behind it is the same, and `the_pci_lines_examine_57_and_58_plant_read_as_their_scenarios_assume` pins what the name table reads from both planted lines. + +`examine-59` plants the same two GPUs as `examine-55`, with a `rocminfo` stand-in that lists both, in the same order and with the same architectures, but prints no `BDFID:` line for either. Without a bus address nothing says which card a reading belongs to, so `rocm examine --json` must apply neither: each GPU keeps its own gfx target, the kernel-topology placeholder name and `is_apu: null`, and the notes carry one "2 GPU reading(s) from rocminfo could not be matched to a listed GPU" note that names both readings with "(no bus address)". Both are asserted in the one scenario, so a note that is printed while a reading is applied anyway, or a refusal with no note, fails it. It runs on the Linux lanes and needs no GPU; the unit test behind it is `agents_without_an_address_are_never_paired_by_position`. + +```bash +cargo test -p rocm-core --lib -- examine::linux_gpus host_gpu::linux +RUSTC_WRAPPER= cargo xtask e2e -- -n 'examine-(55|56|57|58|59) ' +``` + +The `host_gpu::linux` filter holds `a_dev_id_decodes_every_function_bit_and_refuses_zero_and_wider_values`, which pins the PCI-address decode the kernel topology's `location_id` and `rocminfo`'s `BDFID` share to literal addresses. + `serve-42` plants the same WSL machine with `LD_LIBRARY_PATH` empty and runs `rocm serve`: it is refused, its advice names the WSL `driver_status` and not `/dev/kfd`, and `rocm examine` on that machine reports `wsl_rocdxg_missing`, so the command the refusal names exists and shows the condition. The launch that follows when `LD_LIBRARY_PATH` does reach the library has no scenario, because it would start preparing a real engine; the serve gate's decision there is covered by unit tests in `gpu_visibility.rs`. ```bash diff --git a/tests/e2e-cucumber/Cargo.toml b/tests/e2e-cucumber/Cargo.toml index 2c61136e44..8092bf2fbf 100644 --- a/tests/e2e-cucumber/Cargo.toml +++ b/tests/e2e-cucumber/Cargo.toml @@ -99,7 +99,10 @@ shlex.workspace = true # each repeats that helper's spawn, poll, kill and reap steps on `std` alone, # deliberately, and a normal dependency would be a new edge to review. A change # to any one of these copies should look at the others. -rocm-core = { path = "../../crates/rocm-core" } +# +# `test-support` for the planted `lspci` lines examine-57 and examine-58 share +# with the rocm-core unit test that pins how the name table reads them. +rocm-core = { path = "../../crates/rocm-core", features = ["test-support"] } # `test-util` pauses tokio's clock in the retry loops' unit tests # (`#[tokio::test(start_paused = true)]`), so their timing assertions read a # virtual clock that sleeps advance exactly, not the host timer, whose diff --git a/tests/e2e-cucumber/features/examine.feature b/tests/e2e-cucumber/features/examine.feature index a5a7d1c52b..3426caa4f1 100644 --- a/tests/e2e-cucumber/features/examine.feature +++ b/tests/e2e-cucumber/features/examine.feature @@ -217,7 +217,8 @@ Feature: GPU detection and system inspection # half of that fix: `lspci` resolves a target from the marketing name, and on # an Instinct host `pci.ids` frequently spells that "Device 74a1", so without # `rocminfo` the per-node target the kernel reports is the only thing that can - # fill the field. + # fill the field. Where the name does resolve a target, the kernel's target + # replaces it (see examine-57). # # The step no-ops where a premise does not hold -- no readable KFD topology, # no `lspci` to supply PCI addresses, or a topology whose nodes disagree on a @@ -793,3 +794,96 @@ Feature: GPU detection and system inspection Given the active runtime is a nightly runtime installed for one exact GPU When the user inspects the system Then the runtime state names the channel, package source, source layout and exact GPU target that runtime was installed from + # ROCMAI-40. The ROCm runtime can be told to show only some of a machine's + # GPUs (a visibility mask, for example to hide an integrated GPU), and the + # GPUs it lists then have to be matched to the ones the machine has. They were + # matched by list position, so with the first GPU hidden its entry took the + # second GPU's model name, architecture and integrated/discrete verdict, and + # the second kept none. Each is now matched by its own PCI address. The machine + # is simulated: a native Linux root whose kernel topology has two GPUs, at + # 0000:03:00.0 (gfx1100) and 0000:04:00.0 (gfx1101), an `lspci` stand-in that + # lists an empty bus, and a `rocminfo` stand-in that lists only the second, as + # "AMD Radeon RX 7800 XT" with that card's `BDFID`. The other rows (no address + # from the runtime, counts that differ, an address in two PCI domains, a + # partitioned card) are the unit tests in `examine/linux_gpus.rs`. + @id:examine-each-gpu-keeps-its-own-reading @requires-os:linux + Scenario: examine-55 - A GPU the ROCm runtime cannot see does not borrow another GPU's identity + Given a machine with two AMD GPUs of which the ROCm runtime can see only the second + When the user inspects that machine in machine-readable form + Then the second GPU is reported with the model name the runtime gave it, its own architecture and as a discrete GPU + And the first GPU is reported with its own architecture, no model name and no claim about whether it is integrated + + # ROCMAI-40. `HSA_OVERRIDE_GFX_VERSION` makes the ROCm runtime report another + # GPU architecture than the card's own -- a common workaround for a card the + # runtime does not support -- and `rocminfo` inherits it. The runtime's reading + # is matched to the card by PCI address and supplies the model name, but it + # does not replace the architecture the card already has from the kernel, or + # the integrated/discrete verdict that follows from it, so `rocm diagnose`'s + # override check holds the card's native architecture against the framework's + # arch list. The machine is simulated: a native Linux root whose kernel + # topology has one gfx1103 integrated GPU at 0000:c4:00.0, an `lspci` stand-in + # that lists an empty bus, and a `rocminfo` stand-in that, like the real + # runtime, reports that GPU as gfx1100 at the same address only while + # HSA_OVERRIDE_GFX_VERSION is 11.0.0, which the scenario sets, and as an APU + # either way. The stand-in leaves a marker when it runs with the override, + # and the scenario requires it, so it cannot pass on the native listing. + # What it proves is that the overridden target does not replace the card's + # own. It does not prove where the integrated verdict comes from: gfx1103 is + # an APU under the target rule as well, so the runtime's `Memory Properties` + # reading deciding the verdict is proved by the unit test + # `a_matched_agents_verdict_is_its_memory_properties_reading_or_else_the_target_rule`. + @id:examine-overridden-runtime-target-keeps-native @requires-os:linux + Scenario: examine-56 - A runtime told to report a different GPU architecture does not change the architecture reported for the card + Given a machine whose integrated GPU the ROCm runtime is told to report as another architecture + When the user inspects that machine in machine-readable form + Then the integrated GPU is reported with its own architecture and as integrated + And the integrated GPU is reported with the model name the runtime gave it + + # ROCMAI-40. The PCI scan can only guess a GPU's architecture from its model + # name, through a name table, and the guess can be wrong (for Krackan Point + # the name table says gfx1150 while the PCI device-id map says gfx1152; which + # is right is unconfirmed). The kernel states the architecture of each GPU it + # exposes, so where the two disagree at the same PCI address the kernel's is + # reported, and the integrated/discrete verdict follows it. The machine is + # simulated so the disagreement does not depend on any one table entry: a + # native Linux root whose `lspci` stand-in + # lists a "Phoenix1" (which the table reads as gfx1103, integrated) at + # 0000:05:00.0, and whose kernel topology has a gfx1100 node at that address. + @id:examine-kernel-architecture-outranks-the-name-guess @requires-os:linux + Scenario: examine-57 - The kernel's architecture is reported over one guessed from the GPU's model name + Given a machine whose PCI listing names a GPU model the kernel reports as another architecture + When the user inspects that machine in machine-readable form + Then the GPU is reported with the architecture the kernel gives it, as a discrete GPU + + # ROCMAI-40. When the PCI listing cannot name a GPU at all (no model name the + # table knows, as on a Strix Halo whose `pci.ids` has no entry for it), the + # scan has no architecture for it and calls it discrete by default. The + # kernel's architecture fills that gap, and when it is an integrated GPU's the + # GPU is reported as integrated. This scenario covers the host where the ROCm + # runtime lists no GPU; a runtime that does list it states the verdict itself + # (unit-tested in `examine/linux_gpus.rs`). The machine is simulated: a native + # Linux root whose `lspci` stand-in lists a nameless "Device 7fff" at 0000:c3:00.0 (an id no table + # knows, so the gap does not depend on one table entry), whose kernel + # topology has a gfx1151 node at that address, and whose `rocminfo` stand-in + # lists no GPU. + @id:examine-kernel-architecture-fills-an-unnamed-gpu @requires-os:linux + Scenario: examine-58 - A GPU the PCI listing cannot name takes the kernel's architecture and is reported as integrated + Given a machine whose PCI listing cannot name its GPU and whose kernel reports an integrated architecture for it + When the user inspects that machine in machine-readable form + Then the unnamed GPU is reported with the kernel's architecture, as an integrated GPU + + # ROCMAI-40. Without a bus address in the ROCm runtime's listing, nothing says + # which card a reading belongs to. Pairing the readings with the listed GPUs + # by position would be a guess, and on a multi-GPU host a wrong guess puts one + # card's model name and architecture on another, so the readings are not + # applied to any GPU and the inspection says so. The machine is simulated: a + # native Linux root whose kernel topology has two GPUs, at 0000:03:00.0 + # (gfx1100) and 0000:04:00.0 (gfx1101), an `lspci` stand-in that lists an + # empty bus, and a `rocminfo` stand-in that lists both GPUs in the same order + # and with the same architectures but prints no `BDFID:` line for either. + @id:examine-unaddressed-runtime-readings-are-reported-not-guessed @requires-os:linux + Scenario: examine-59 - GPU readings the ROCm runtime does not tie to a card are reported, not guessed + Given a machine with two AMD GPUs whose ROCm runtime lists both without saying which card is which + When the user inspects that machine in machine-readable form + Then each GPU is reported with its own architecture and no model name + And the inspection says the runtime's readings could not be matched to a listed GPU diff --git a/tests/e2e-cucumber/tests/e2e/examine_steps.rs b/tests/e2e-cucumber/tests/e2e/examine_steps.rs index a3b34fef3c..a2288e098c 100644 --- a/tests/e2e-cucumber/tests/e2e/examine_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/examine_steps.rs @@ -3169,6 +3169,479 @@ async fn assert_the_gpu_lspci_printed_is_listed(world: &mut E2eWorld) { ); } +// ── Each GPU keeps the runtime's reading of itself, not another GPU's ── + +/// The two GPUs `examine-55` plants: `(PCI address, KFD location_id, KFD +/// gfx_target_version, the gfx target that version decodes to)`. The +/// `location_id` doubles as the `BDFID` the stand-in `rocminfo` prints for the +/// second, since both use the kernel's `(bus << 8) | devfn` encoding. +const HIDDEN_GPU: (&str, u32, u32, &str) = ("0000:03:00.0", 768, 110_000, "gfx1100"); +const VISIBLE_GPU: (&str, u32, u32, &str) = ("0000:04:00.0", 1024, 110_001, "gfx1101"); +const VISIBLE_GPU_MODEL: &str = "AMD Radeon RX 7800 XT"; + +/// Plant a simulated native Linux machine with one KFD GPU node per +/// `(location_id, gfx_target_version)`, an `lspci` that prints `lspci_lines` +/// (none is an empty bus), +/// and a `rocminfo` that lists one GPU agent per `(gfx, marketing name, +/// BDFID)`. +/// +/// Native rather than WSL because the GPU probes are skipped on WSL, so a +/// runner that is a WSL host would otherwise never reach the planted GPUs. +/// The `rocminfo` is in the shape the real tool prints (compare the capture in +/// `crates/rocm-core/tests/fixtures/rocminfo_gfx1151_apu.txt`): a CPU agent +/// with BDFID 0, then the GPU agents, each closed by its `ISA Info:` block, +/// whose ISA entries carry `Name:` lines of their own that are not the agent's +/// name. Each GPU agent prints the given `Memory Properties:` value: `APU` as +/// the capture prints for its integrated GPU, or empty for a discrete one, +/// which rests on the MI300X sample output in AMD's "Using rocminfo" page +/// (), +/// a documentation sample and not a capture taken for this repository. +/// +/// `prelude` is shell run before the listing is printed, and the listing is +/// expanded by the shell, so an agent's fields can depend on the environment +/// the runtime is started in (`examine-56` uses that for +/// `HSA_OVERRIDE_GFX_VERSION`). The listing holds no other `$`, backtick or +/// backslash for the shell to read. +fn plant_native_gpus_and_runtime( + world: &mut E2eWorld, + lspci_lines: &[&str], + kfd_gpus: &[(u32, u32)], + prelude: &str, + agents: &[(&str, &str, Option, &str)], +) { + use std::fmt::Write as _; + plant_os_release(world, "ID=ubuntu\nVERSION_ID=\"24.04\"\n"); + let nodes = world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("host-root/sys/class/kfd/kfd/topology/nodes"); + let cpu = nodes.join("0"); + std::fs::create_dir_all(&cpu).expect("failed to create the planted CPU node"); + std::fs::write( + cpu.join("properties"), + "cpu_cores_count 16\nsimd_count 0\ngfx_target_version 0\nlocation_id 0\ndomain 0\n", + ) + .expect("failed to write the planted CPU node properties"); + for (index, (location_id, version)) in kfd_gpus.iter().enumerate() { + let node = nodes.join((index + 1).to_string()); + std::fs::create_dir_all(&node).expect("failed to create a planted GPU node"); + std::fs::write( + node.join("properties"), + format!( + "cpu_cores_count 0\nsimd_count 96\ngfx_target_version {version}\n\ + location_id {location_id}\ndomain 0\n" + ), + ) + .expect("failed to write a planted GPU node's properties"); + } + // The listing goes through an unquoted heredoc and the `lspci` lines + // through single quotes, so nothing interpolated may carry what the shell + // would read: a `$` (other than the prelude's own variables, which the + // caller passes in agent fields on purpose), a backtick, a backslash or a + // single quote. + let shell_safe = |value: &str| !value.contains(['`', '\\', '\'']); + for line in lspci_lines { + assert!( + shell_safe(line) && !line.contains('$'), + "a planted lspci line the shell would rewrite: {line:?}" + ); + } + for (gfx, marketing, _, memory) in agents { + assert!( + shell_safe(memory) && !memory.contains('$'), + "a planted rocminfo field the shell would rewrite: {memory:?}" + ); + assert!( + shell_safe(gfx) && shell_safe(marketing) && !marketing.contains('$'), + "a planted rocminfo field the shell would rewrite: {gfx:?} {marketing:?}" + ); + } + let mut lspci = String::from("#!/bin/sh\n"); + for line in lspci_lines { + writeln!(lspci, "printf '%s\\n' '{line}'").expect("writing to a String cannot fail"); + } + let mut rocminfo = format!("#!/bin/sh\n{prelude}\ncat < serde_json::Value { + json.pointer("/gpus") + .and_then(serde_json::Value::as_array) + .unwrap_or_else(|| panic!("`examine --json` has no `gpus` array:\n{json:#}")) + .iter() + .find(|gpu| gpu.pointer("/pci_id").and_then(serde_json::Value::as_str) == Some(pci_id)) + .unwrap_or_else(|| panic!("no GPU is listed at {pci_id}:\n{json:#}")) + .clone() +} + +#[then( + "the second GPU is reported with the model name the runtime gave it, its own architecture and as a discrete GPU" +)] +async fn assert_the_visible_gpu_has_the_runtimes_reading(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "`rocm examine --json` failed"); + let json = parsed_json(world); + let (address, _, _, gfx) = VISIBLE_GPU; + let gpu = listed_gpu_at(&json, address); + assert_eq!( + gpu["name"], + serde_json::json!(VISIBLE_GPU_MODEL), + "{json:#}" + ); + assert_eq!(gpu["gfx_target"], serde_json::json!(gfx), "{json:#}"); + // "As a discrete GPU": the runtime's own reading, an empty + // `Memory Properties:` line. + assert_eq!(gpu["is_apu"], serde_json::json!(false), "{json:#}"); +} + +#[then( + "the first GPU is reported with its own architecture, no model name and no claim about whether it is integrated" +)] +async fn assert_the_hidden_gpu_keeps_its_own_reading(world: &mut E2eWorld) { + let json = parsed_json(world); + let gpus = json["gpus"].as_array().map_or(0, Vec::len); + assert_eq!(gpus, 2, "one entry per planted GPU:\n{json:#}"); + let (address, _, _, gfx) = HIDDEN_GPU; + let gpu = listed_gpu_at(&json, address); + assert_eq!(gpu["gfx_target"], serde_json::json!(gfx), "{json:#}"); + // "No model name" as the report spells it: the kernel-topology + // placeholder, which the notes explain as an unknown marketing name. + assert_eq!( + gpu["name"], + serde_json::json!(rocm_core::examine::KERNEL_TOPOLOGY_GPU_NAME), + "{json:#}" + ); + // "No claim about whether it is integrated": gfx1100 is outside the APU + // families, which is not proof of a discrete card, and no PCI scan named + // it -- so `is_apu` is null, not a verdict borrowed from the other card. + assert_eq!(gpu["is_apu"], serde_json::Value::Null, "{json:#}"); + let notes: Vec<&str> = json["notes"] + .as_array() + .into_iter() + .flatten() + .filter_map(serde_json::Value::as_str) + .collect(); + assert!( + notes + .iter() + .any(|note| note.starts_with("1 GPU(s)") && note.contains("marketing name is unknown")), + "the one unnamed GPU must be explained exactly as one:\n{notes:#?}" + ); +} + +// ── A runtime override does not change the architecture reported for a card ── + +/// The integrated GPU `examine-56` plants: `(PCI address, KFD location_id, +/// KFD gfx_target_version, its own gfx target)`, a gfx1103 APU. The stand-in +/// `rocminfo` reports it at the same address as [`OVERRIDDEN_GFX`], the ISA an +/// `HSA_OVERRIDE_GFX_VERSION` of 11.0.0 makes the ROCm runtime report. +const OVERRIDDEN_APU: (&str, u32, u32, &str) = ("0000:c4:00.0", 50_176, 110_003, "gfx1103"); +const OVERRIDDEN_GFX: &str = "gfx1100"; +/// The `HSA_OVERRIDE_GFX_VERSION` value that names [`OVERRIDDEN_GFX`]. +const OVERRIDE_VALUE: &str = "11.0.0"; +const OVERRIDDEN_APU_MODEL: &str = "AMD Radeon 780M Graphics"; + +#[given( + "a machine whose integrated GPU the ROCm runtime is told to report as another architecture" +)] +async fn machine_whose_runtime_reports_an_overridden_architecture(world: &mut E2eWorld) { + let (_, location, version, native_gfx) = OVERRIDDEN_APU; + // The stand-in runtime does what ROCr does: with the override set it + // reports the ISA the override names, and without it the card's own. Its + // `Memory Properties:` says `APU` either way, because the runtime reads + // that from the device. When it runs with the override it also leaves a + // marker, so the Then can show the inspection's own `rocminfo` run saw the + // variable rather than only that the planted files say so. + let marker = override_marker(world); + let marker = marker + .to_str() + .expect("the scenario directory is valid UTF-8"); + assert!( + !marker.contains(['$', '`', '\\', '\'', '"']), + "a marker path the shell would rewrite: {marker:?}" + ); + let prelude = format!( + "if [ \"$HSA_OVERRIDE_GFX_VERSION\" = \"{OVERRIDE_VALUE}\" ]; then GFX={OVERRIDDEN_GFX}; \ + : > \"{marker}\"; else GFX={native_gfx}; fi" + ); + plant_native_gpus_and_runtime( + world, + &[], + &[(location, version)], + &prelude, + &[("${GFX}", OVERRIDDEN_APU_MODEL, Some(location), "APU")], + ); + world + .command_env + .push(("HSA_OVERRIDE_GFX_VERSION", OVERRIDE_VALUE.into())); +} + +/// Where the examine-56 stand-in `rocminfo` leaves its marker when it runs +/// with the override set. +fn override_marker(world: &E2eWorld) -> std::path::PathBuf { + world + .isolated_root + .as_ref() + .expect("no isolated root") + .path() + .join("rocminfo-ran-with-the-override") +} + +#[then("the integrated GPU is reported with its own architecture and as integrated")] +async fn assert_the_overridden_apu_keeps_its_own_architecture(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "`rocm examine --json` failed"); + // Only the inspection ran the planted `rocminfo`, so the marker shows the + // runtime it read was started with the override: without it, the stand-in + // reports the native target and this step would hold for no reason. + assert!( + override_marker(world).exists(), + "the inspection's rocminfo did not run with HSA_OVERRIDE_GFX_VERSION={OVERRIDE_VALUE}, \ + so nothing here was overridden" + ); + let json = parsed_json(world); + let (address, _, _, native_gfx) = OVERRIDDEN_APU; + let gpu = listed_gpu_at(&json, address); + assert_eq!( + gpu["gfx_target"], + serde_json::json!(native_gfx), + "the runtime's overridden {OVERRIDDEN_GFX} replaced the card's own target:\n{json:#}" + ); + assert_eq!(gpu["is_apu"], serde_json::json!(true), "{json:#}"); + assert_eq!(json["has_apu"], serde_json::json!(true), "{json:#}"); + assert_eq!( + json["has_discrete_amd"], + serde_json::json!(false), + "{json:#}" + ); +} + +#[then("the integrated GPU is reported with the model name the runtime gave it")] +async fn assert_the_overridden_apu_takes_the_runtimes_name(world: &mut E2eWorld) { + let json = parsed_json(world); + let (address, _, _, _) = OVERRIDDEN_APU; + let gpu = listed_gpu_at(&json, address); + assert_eq!( + gpu["name"], + serde_json::json!(OVERRIDDEN_APU_MODEL), + "{json:#}" + ); +} + +// ── The kernel's architecture wins over one guessed from the model name ── + +/// The GPU `examine-57` plants: an `lspci -nn -D` line whose model name the +/// examination's name table reads as gfx1103, an integrated GPU, and a KFD +/// node at the same address (`location_id`, `gfx_target_version`) saying +/// gfx1100, a discrete one. The line is rocm-core's own constant, whose +/// name-table reading the unit test +/// `the_pci_lines_examine_57_and_58_plant_read_as_their_scenarios_assume` +/// pins, so the scenario cannot pass with the two already agreeing. +const NAME_GUESSED_GPU_ADDRESS: &str = "0000:05:00.0"; +const NAME_GUESSED_GPU_LSPCI: &str = + rocm_core::examine::TEST_FIXTURE_LSPCI_LINE_NAME_TABLE_READS_AS_GFX1103; +const NAME_GUESSED_GPU_NODE: (u32, u32) = (1280, 110_000); +const KERNEL_GFX: &str = "gfx1100"; + +#[given("a machine whose PCI listing names a GPU model the kernel reports as another architecture")] +async fn machine_whose_pci_name_and_kernel_disagree(world: &mut E2eWorld) { + plant_native_gpus_and_runtime( + world, + &[NAME_GUESSED_GPU_LSPCI], + &[NAME_GUESSED_GPU_NODE], + "", + &[], + ); +} + +#[then("the GPU is reported with the architecture the kernel gives it, as a discrete GPU")] +async fn assert_the_kernels_architecture_is_reported(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "`rocm examine --json` failed"); + let json = parsed_json(world); + let gpus = json["gpus"].as_array().map_or(0, Vec::len); + assert_eq!(gpus, 1, "one entry for the one planted GPU:\n{json:#}"); + let gpu = listed_gpu_at(&json, NAME_GUESSED_GPU_ADDRESS); + assert_eq!( + gpu["gfx_target"], + serde_json::json!(KERNEL_GFX), + "the architecture guessed from the model name outranked the kernel's:\n{json:#}" + ); + assert_eq!(gpu["is_apu"], serde_json::json!(false), "{json:#}"); + assert_eq!(json["has_apu"], serde_json::json!(false), "{json:#}"); + assert_eq!( + json["has_discrete_amd"], + serde_json::json!(true), + "{json:#}" + ); +} + +// ── The kernel's architecture fills a GPU the PCI listing cannot name ── + +/// The GPU `examine-58` plants: an `lspci -nn -D` line with no model name any +/// table knows (`Device 7fff`), and a KFD node at that address +/// (`location_id`, `gfx_target_version`) saying gfx1151, an integrated GPU. +/// The line is rocm-core's own constant, whose empty name-table reading the +/// same unit test pins. +const UNNAMED_GPU_ADDRESS: &str = "0000:c3:00.0"; +const UNNAMED_GPU_LSPCI: &str = + rocm_core::examine::TEST_FIXTURE_LSPCI_LINE_NAME_TABLE_READS_AS_NOTHING; +const UNNAMED_GPU_NODE: (u32, u32) = (49_920, 110_501); +const UNNAMED_GPU_KERNEL_GFX: &str = "gfx1151"; + +#[given( + "a machine whose PCI listing cannot name its GPU and whose kernel reports an integrated architecture for it" +)] +async fn machine_whose_pci_listing_cannot_name_an_integrated_gpu(world: &mut E2eWorld) { + plant_native_gpus_and_runtime(world, &[UNNAMED_GPU_LSPCI], &[UNNAMED_GPU_NODE], "", &[]); +} + +#[then("the unnamed GPU is reported with the kernel's architecture, as an integrated GPU")] +async fn assert_the_unnamed_gpu_takes_the_kernels_architecture(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "`rocm examine --json` failed"); + let json = parsed_json(world); + let gpus = json["gpus"].as_array().map_or(0, Vec::len); + assert_eq!(gpus, 1, "one entry for the one planted GPU:\n{json:#}"); + let gpu = listed_gpu_at(&json, UNNAMED_GPU_ADDRESS); + assert_eq!( + gpu["gfx_target"], + serde_json::json!(UNNAMED_GPU_KERNEL_GFX), + "{json:#}" + ); + assert_eq!( + gpu["is_apu"], + serde_json::json!(true), + "the scan's default `discrete` stood over the kernel's integrated architecture:\n{json:#}" + ); + assert_eq!(json["has_apu"], serde_json::json!(true), "{json:#}"); + assert_eq!( + json["has_discrete_amd"], + serde_json::json!(false), + "{json:#}" + ); +} + +// ── Runtime readings that name no card are reported, not paired by position ── + +#[given( + "a machine with two AMD GPUs whose ROCm runtime lists both without saying which card is which" +)] +async fn machine_whose_runtime_lists_gpus_without_addresses(world: &mut E2eWorld) { + // The same two cards as examine-55, and a runtime that lists both, in the + // same order and with matching architectures, but prints no `BDFID:` for + // either: a position-and-count pairing would take this listing at face + // value, which is exactly what must not happen. + let (_, first_location, first_version, first_gfx) = HIDDEN_GPU; + let (_, second_location, second_version, second_gfx) = VISIBLE_GPU; + plant_native_gpus_and_runtime( + world, + &[], + &[ + (first_location, first_version), + (second_location, second_version), + ], + "", + &[ + (first_gfx, UNPLACED_FIRST_MODEL, None, ""), + (second_gfx, VISIBLE_GPU_MODEL, None, ""), + ], + ); +} + +/// The model name the examine-59 runtime gives its first agent; neither card +/// may end up carrying it or [`VISIBLE_GPU_MODEL`]. +const UNPLACED_FIRST_MODEL: &str = "AMD Radeon RX 7900 XTX"; + +#[then("each GPU is reported with its own architecture and no model name")] +async fn assert_each_gpu_keeps_its_own_reading(world: &mut E2eWorld) { + assert_eq!(world.cli_rc, Some(0), "`rocm examine --json` failed"); + let json = parsed_json(world); + let gpus = json["gpus"].as_array().map_or(0, Vec::len); + assert_eq!( + gpus, 2, + "one entry per planted GPU, and none added:\n{json:#}" + ); + for (address, _, _, gfx) in [HIDDEN_GPU, VISIBLE_GPU] { + let gpu = listed_gpu_at(&json, address); + assert_eq!(gpu["gfx_target"], serde_json::json!(gfx), "{json:#}"); + assert_eq!( + gpu["name"], + serde_json::json!(rocm_core::examine::KERNEL_TOPOLOGY_GPU_NAME), + "{address} took a model name the runtime gave an agent that names no card:\n{json:#}" + ); + assert_eq!(gpu["is_apu"], serde_json::Value::Null, "{json:#}"); + } +} + +#[then("the inspection says the runtime's readings could not be matched to a listed GPU")] +async fn assert_the_unplaced_readings_are_reported(world: &mut E2eWorld) { + let json = parsed_json(world); + let notes: Vec<&str> = json["notes"] + .as_array() + .into_iter() + .flatten() + .filter_map(serde_json::Value::as_str) + .collect(); + let note = notes + .iter() + .find(|note| note.contains("could not be matched to a listed GPU")) + .unwrap_or_else(|| panic!("no note about the unmatched readings:\n{notes:#?}")); + assert!( + note.starts_with("2 GPU reading(s) from rocminfo"), + "the note counts both readings: {note}" + ); + for gfx in [HIDDEN_GPU.3, VISIBLE_GPU.3] { + assert!( + note.contains(&format!("{gfx} (no bus address)")), + "the note names {gfx} and why: {note}" + ); + } +} + // ── `rocm serve` on a WSL host whose ROCDXG library the loader cannot find ── #[when("the user serves a model under the GPU-required default on that machine")]