diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3e6442a63a..fc9b88922c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -91,6 +91,11 @@ Every local (relative-path) Markdown link and `#anchor` fragment in the reposito Since only links are checked, cite a specific repository file as a link relative to the citing file, `[fix.rs](../crates/rocm-core/src/fix.rs)` from `docs/`, rather than a bare backtick path, which nothing checks. The full rule, including the link forms that fail the check and the places it doesn't apply (`.github/`, links leaving a `skills/` folder, and `docs/rocm-docs/` along with the files it includes, which are this file and README.md), is in AGENTS.md section 5 ("Investigate rocm-cli Before Editing"); it applies to citations you add or edit. +### Updating the AIM catalog + +AIM model entries are hand-maintained in `crates/rocm-core/src/model_catalog.json`; +see `docs/aim-catalog.md` for the refresh process and staleness warning. + ### Test commands | Component | Command | diff --git a/README.md b/README.md index 71e8da0eba..0bcb92e10f 100644 --- a/README.md +++ b/README.md @@ -821,7 +821,14 @@ rocm model [--verbose] `rocm model` (alias `rocm models`) lists the recommended local models, grouped by the hardware they target. Pass `--verbose` to list every recipe, including ones hidden from the default list, with details such as aliases, data type, minimum -GPU memory, and preferred engines. +GPU memory, and preferred engines. The AMD Inference Microservices (AIM) Docker +images are among the hidden recipes: `--verbose` shows each one's supported GPU +families, the GPU count it needs per family, and whether it needs a Hugging Face +token (`requires_hf_token=yes`), and +`rocm serve --engine aim` runs it (needs a reachable Docker daemon). +AIM entries are hand-maintained and record when they were last verified; +`rocm model --verbose` warns when one is over 180 days old (see +`docs/aim-catalog.md`). ### Model serving @@ -912,7 +919,10 @@ instead of silently substituting a different model. Some models (such as Llama) are gated and require Hugging Face authentication. Log in with `huggingface-cli login` or set `HF_TOKEN` in your environment -before serving gated models. +before serving gated models. The exception is `--engine aim`, whose container +does not read your environment on its own: export `HF_TOKEN` and pass +`--engine-env HF_TOKEN` (see above), or `rocm serve` refuses a gated AIM image +before pulling anything. `--gpu` selects which AMD GPU the server runs on. `auto` (the default) probes per-GPU VRAM (through `amd-smi`, or the amdgpu DRM sysfs counters when `amd-smi` diff --git a/apps/rocm/src/chat_host_facts.rs b/apps/rocm/src/chat_host_facts.rs index 84dc740a75..74d985a994 100644 --- a/apps/rocm/src/chat_host_facts.rs +++ b/apps/rocm/src/chat_host_facts.rs @@ -79,6 +79,23 @@ impl HostFacts { self.os_name != "windows" } + /// Whether any catalogued AIM image has a profile for this GPU's family. + /// `None` when the family is unknown, where `rocm serve` enforces nothing. + fn aim_catalog_covers_gpu(&self) -> Option { + let family = self + .gfx_target + .as_deref() + .or(self.therock_family.as_deref()) + .and_then(rocm_core::normalize_therock_family)?; + Some(rocm_core::builtin_model_recipes().iter().any(|recipe| { + recipe + .engine_recipes + .iter() + .filter(|record| record.engine.eq_ignore_ascii_case("aim")) + .any(|record| record.supported_gfx_families.contains(&family)) + })) + } + /// How to describe the operating system to a non-technical user. WSL is /// called out because it is the case the old prompt got wrong: it asserted /// "on native Windows, vLLM is skipped" at a user who was on WSL, where @@ -123,12 +140,18 @@ impl HostFacts { "- Serving engines that run here: Lemonade. vLLM and AIM are skipped on native Windows; \ tell the user to use WSL2 or Linux for those ROCm GPU engines.\n" }; + let aim_note = if self.vllm_available() && self.aim_catalog_covers_gpu() == Some(false) { + "- No catalogued AIM model has a profile for this GPU family, and `rocm serve` \ + refuses them here; do not suggest an AIM catalog model.\n" + } else { + "" + }; format!( "Facts about the machine you are running on, detected by ROCm CLI. Trust these \ over anything you remember about ROCm:\n\ - Operating system: {os}\n\ - AMD GPU: {gpu}\n\ - {family}{engines}\ + {family}{engines}{aim_note}\ ROCm and ROCm CLI support this operating system. Never tell the user ROCm is \ unavailable on their platform, and never suggest CUDA, DirectML, or CPU fallback.", os = self.operating_system(), @@ -175,6 +198,11 @@ mod tests { block.contains("Lemonade, vLLM and AIM"), "vLLM and AIM are available under WSL:\n{block}" ); + // Strix Halo has no profile in any catalogued AIM image. + assert!( + block.contains("do not suggest an AIM catalog model"), + "{block}" + ); assert!( !block.contains("ROCm/TheRock family"), "a family identical to the target adds nothing:\n{block}" @@ -204,6 +232,20 @@ mod tests { ); } + #[test] + fn aim_catalog_note_only_appears_where_no_catalogued_image_covers_the_gpu() { + let block = |target: Option<&str>| { + HostFacts::from_gpu_summary("linux", false, gpu(None, target, target)).render() + }; + let note = "do not suggest an AIM catalog model"; + assert!(!block(Some("gfx942")).contains(note)); + assert!(!block(Some("gfx1100")).contains(note)); + assert!(!block(Some("gfx1201")).contains(note)); + assert!(block(Some("gfx1151")).contains(note)); + // No detected GPU: nothing is enforced, so nothing is claimed. + assert!(!block(None).contains(note)); + } + #[test] fn bare_linux_is_not_reported_as_wsl() { let facts = diff --git a/apps/rocm/src/dash.rs b/apps/rocm/src/dash.rs index 891255c1d7..2e88ef3d3a 100644 --- a/apps/rocm/src/dash.rs +++ b/apps/rocm/src/dash.rs @@ -190,10 +190,12 @@ fn model_recipe_summaries() -> Vec { builtin_model_recipes() .into_iter() .map(|r| ModelRecipeSummary { + // Same rule as the chat planner: a host-gated AIM first choice is + // not pre-selected when the recipe lists another engine. + preferred_engine: crate::freeform_planner::planner_recipe_engine(&r).map(str::to_owned), id: r.canonical_model_id, aliases: r.aliases, task: r.task, - preferred_engine: r.preferred_engines.into_iter().next(), }) .collect() } diff --git a/apps/rocm/src/examine_report_text.rs b/apps/rocm/src/examine_report_text.rs index 29aeaba4ac..2a6fed7763 100644 --- a/apps/rocm/src/examine_report_text.rs +++ b/apps/rocm/src/examine_report_text.rs @@ -29,14 +29,15 @@ //! `render_model_registry_verbose_text_with_context_and_host`, //! `model_catalog_header`, `append_model_recipe_registry_source`, //! `append_model_recipe_metadata_lines`, -//! `append_model_engine_recipe_settings_lines`, `format_list_or_none`, +//! `append_model_engine_recipe_settings_lines`, +//! `append_model_recipe_staleness_lines`, `format_list_or_none`, //! `format_string_map_or_none`, `append_model_artifact_lines`, //! `append_artifact_source_policy_lines`, `artifact_source_policy_label`, //! `format_bytes`, `append_model_host_ram_fit_lines`, //! `append_model_fit_lines`, `append_manual_alternative_lines`, //! `manual_alternative_recommendations`, `recipe_is_manual_fit`, //! `append_model_engine_support_lines`, `recipe_display_ref`, -//! `preferred_engine_action_target`, `model_registry_reason`, `format_gib`, +//! `preferred_engine_action_target`, `once_per_call`, `model_registry_reason`, `format_gib`, //! `append_engine_detect_summary`, `friendly_engine_runtime_kind`, //! `friendly_engine_detect_notes`, `friendly_engine_detect_note_fallback`, //! and `engine_runtime_status_label`). @@ -134,7 +135,8 @@ use crate::runtimes_cmd::{ }; use crate::therock; use crate::{ - PlatformRuleOut, engine_inventory, engine_ruled_out_by_platform, missing_packaged_engine_reason, + PlatformRuleOut, engine_inventory, engine_ruled_out_by_platform, + engine_ruled_out_by_platform_with, missing_packaged_engine_reason, }; pub(crate) fn render_launch_summary(paths: &AppPaths, config: &RocmCliConfig) -> String { @@ -561,14 +563,7 @@ pub(crate) fn render_model_registry_text_with_context_and_host( for platform in &platforms { let mut group = visible .iter() - .filter(|recipe| { - recipe.preferred_engines.first().is_some_and(|engine| { - platform - .engines - .iter() - .any(|e| e.eq_ignore_ascii_case(engine)) - }) - }) + .filter(|recipe| rocm_core::model_recipe_targets_platform(recipe, platform)) .peekable(); if group.peek().is_none() { continue; @@ -649,6 +644,11 @@ pub(crate) fn render_model_registry_verbose_text_with_context_and_host( return output; } }; + // Every AIM recipe asks whether Docker is reachable; one `docker info` per + // listing answers all of them. + let docker_reachable = once_per_call(rocm_engine_aim::builtin_docker_daemon_reachable); + // Detected only if a recipe's first engine is AIM, whose usability depends on it. + let host_gpu = std::cell::OnceCell::new(); for recipe in ®istry.recipes { let aliases = if recipe.aliases.is_empty() { "".to_owned() @@ -692,19 +692,38 @@ pub(crate) fn render_model_registry_verbose_text_with_context_and_host( } append_model_recipe_metadata_lines(&mut output, recipe, paths); append_model_host_ram_fit_lines(&mut output, recipe, host_ram_gib); - append_model_fit_lines(&mut output, recipe, aggregate_gpu_vram_gib); - append_model_engine_support_lines(&mut output, recipe, paths); + append_model_fit_lines( + &mut output, + recipe, + aggregate_gpu_vram_gib, + &host_gpu, + paths, + ); + append_model_engine_support_lines(&mut output, recipe, paths, &docker_reachable); if recipe.trust_remote_code { let _ = writeln!(output, " trust_remote_code: true"); } for warning in &recipe.warnings { let _ = writeln!(output, " warning: {warning}"); } + append_model_recipe_staleness_lines(&mut output, recipe, rocm_core::unix_time_millis()); } append_model_recipe_registry_source(&mut output, ®istry); output } +fn append_model_recipe_staleness_lines( + output: &mut String, + recipe: &ModelRecipeRecord, + now_ms: u128, +) { + for engine_recipe in &recipe.engine_recipes { + if let Some(warning) = rocm_core::model_recipe_staleness_warning(engine_recipe, now_ms) { + let _ = writeln!(output, " warning: {warning}"); + } + } +} + /// The `rocm model` header. For the default built-in list it just names the /// action; a configured recipe index instead advertises its provenance (the /// only case where the source differs from the default and is worth surfacing). @@ -800,6 +819,31 @@ fn append_model_engine_recipe_settings_lines(output: &mut String, recipe: &Model .join("; ") }; let notes = format_list_or_none(&engine_recipe.notes); + if !engine_recipe.supported_gfx_families.is_empty() + || engine_recipe.requires_hf_token + || !engine_recipe.profile_ids.is_empty() + || !engine_recipe.min_gpu_counts.is_empty() + { + let _ = writeln!( + output, + " engine_recipe {} supported_gfx_families=[{}] requires_hf_token={} profile_ids=[{}] min_gpu_counts=[{}]", + engine_recipe.engine, + format_list_or_none(&engine_recipe.supported_gfx_families), + if engine_recipe.requires_hf_token { + "yes" + } else { + "no" + }, + format_string_map_or_none(&engine_recipe.profile_ids), + format_string_map_or_none( + &engine_recipe + .min_gpu_counts + .iter() + .map(|(family, count)| (family.clone(), count.to_string())) + .collect() + ) + ); + } let _ = writeln!( output, " engine_recipe {} required_flags=[{}] parser_settings=[{}] preferred_endpoint={} unsupported_combinations=[{}] notes=[{}]", @@ -1026,7 +1070,10 @@ fn append_model_fit_lines( output: &mut String, recipe: &ModelRecipeRecord, aggregate_gpu_vram_gib: Option, + host_gpu: &std::cell::OnceCell, + paths: Option<&AppPaths>, ) { + let engine_target = preferred_engine_action_target(recipe, host_gpu, paths); match recipe.min_gpu_mem_gb { None => { let _ = writeln!(output, " gpu_fit: supported"); @@ -1039,7 +1086,7 @@ fn append_model_fit_lines( output, " action: use /plan serve {} with {}", recipe_display_ref(recipe), - preferred_engine_action_target(recipe) + engine_target ); } Some(required) => match aggregate_gpu_vram_gib { @@ -1055,7 +1102,7 @@ fn append_model_fit_lines( output, " action: use /plan serve {} with {}", recipe_display_ref(recipe), - preferred_engine_action_target(recipe) + engine_target ); } Some(available) => { @@ -1177,11 +1224,19 @@ fn recipe_is_manual_fit(recipe: &ModelRecipeRecord, aggregate_gpu_vram_gib: Opti } } +/// Wraps `probe` so it runs at most once, however many times the result is asked +/// for, for as long as the returned closure lives. +fn once_per_call(probe: impl Fn() -> bool) -> impl Fn() -> bool { + let cached = std::cell::OnceCell::new(); + move || *cached.get_or_init(&probe) +} + #[allow(dead_code)] fn append_model_engine_support_lines( output: &mut String, recipe: &ModelRecipeRecord, paths: Option<&AppPaths>, + docker_reachable: &dyn Fn() -> bool, ) { if recipe.preferred_engines.is_empty() { let _ = writeln!(output, " engine_support: unknown"); @@ -1195,7 +1250,7 @@ fn append_model_engine_support_lines( let _ = writeln!(output, " engine_support:"); for engine in &recipe.preferred_engines { if builtin_engine_available(engine) { - if let Some(note) = model_registry_adapter_availability_note(engine) { + if let Some(note) = model_registry_adapter_availability_note(engine, docker_reachable) { let _ = writeln!( output, " {engine}: adapter_available path= {note}" @@ -1206,7 +1261,9 @@ fn append_model_engine_support_lines( } else if let Some(paths) = paths { match resolve_engine_binary_path_with_paths(engine, paths) { Ok(path) => { - if let Some(note) = model_registry_adapter_availability_note(engine) { + if let Some(note) = + model_registry_adapter_availability_note(engine, docker_reachable) + { let _ = writeln!( output, " {engine}: adapter_available path={} {note}", @@ -1245,8 +1302,11 @@ fn append_model_engine_support_lines( } #[allow(dead_code)] -fn model_registry_adapter_availability_note(engine: &str) -> Option<&'static str> { - match engine_ruled_out_by_platform(engine) { +fn model_registry_adapter_availability_note( + engine: &str, + docker_reachable: &dyn Fn() -> bool, +) -> Option<&'static str> { + match engine_ruled_out_by_platform_with(engine, docker_reachable) { PlatformRuleOut::Allowed => None, PlatformRuleOut::AimDockerUnavailable => Some( "runtime_status=unsupported_docker_unavailable reason=Docker daemon not reachable; start Docker or choose a different engine; run /engine for adapter details", @@ -1269,11 +1329,22 @@ fn recipe_display_ref(recipe: &ModelRecipeRecord) -> &str { } #[allow(dead_code)] -fn preferred_engine_action_target(recipe: &ModelRecipeRecord) -> &str { - recipe +fn preferred_engine_action_target( + recipe: &ModelRecipeRecord, + host_gpu: &std::cell::OnceCell, + paths: Option<&AppPaths>, +) -> String { + let first = recipe .preferred_engines .first() - .map_or("", String::as_str) + .map_or("", String::as_str); + if !first.eq_ignore_ascii_case("aim") { + return first.to_owned(); + } + // AIM is host-gated (GPU family, platform), so name the engine `rocm serve` + // would pick on this host instead of assuming AIM. + let host = host_gpu.get_or_init(|| rocm_core::detect_host_gpu_summary(paths)); + crate::serve_cmd::select_serve_engine(None, None, Some(recipe), Some(host)).engine } #[allow(dead_code)] @@ -1828,12 +1899,12 @@ mod tests { "unsupported_native_windows" ); assert!( - model_registry_adapter_availability_note("vllm") + model_registry_adapter_availability_note("vllm", &|| true) .is_some_and(|note| note.contains("unsupported_native_windows")) ); } else { assert_eq!(engine_runtime_status_label("vllm", &detect), "not found"); - assert!(model_registry_adapter_availability_note("vllm").is_none()); + assert!(model_registry_adapter_availability_note("vllm", &|| true).is_none()); } } @@ -1909,8 +1980,8 @@ mod tests { assert!(rendered.contains("Recommended models - run one with `rocm serve `")); // Featured current models are grouped under their hardware target, each // with the quant that fits a single GPU. - assert!(rendered.contains("AMD Ryzen AI - Strix Halo (Lemonade / llama.cpp / AIM)")); - assert!(rendered.contains("AMD Instinct - MI300X, MI350X, MI355X (vLLM)")); + assert!(rendered.contains("AMD Ryzen AI - Strix Halo (Lemonade / llama.cpp)")); + assert!(rendered.contains("AMD Instinct - MI300X, MI350X, MI355X (vLLM / AIM)")); // Strix Halo GGUF entries carry the servable owner/repo:variant ref. assert!(rendered.contains("unsloth/Qwen3.6-35B-A3B-GGUF:Q4_K_M")); assert!(rendered.contains("Q4_K_M GGUF")); @@ -2002,7 +2073,7 @@ mod tests { assert!(rendered.contains("source: built-in recipe registry")); // Verbose keeps every recipe (including hidden ones) and annotates each // with its curated catalog placement. - assert!(rendered.contains("catalog: AMD Instinct - MI300X, MI350X, MI355X (vLLM)")); + assert!(rendered.contains("catalog: AMD Instinct - MI300X, MI350X, MI355X (vLLM / AIM)")); assert!(rendered.contains("catalog: hidden (resolvable via rocm serve")); assert!(rendered.contains("sshleifer/tiny-gpt2")); // ROCMAI-543: `rocm model --verbose` output must stick to ASCII too, not @@ -2050,6 +2121,12 @@ mod tests { notes: vec!["metadata only; not applied to launches yet".to_owned()], model_id_override: None, image_digest: None, + generated_at_unix_ms: None, + supported_gfx_families: Vec::new(), + profile_ids: BTreeMap::new(), + requires_hf_token: false, + supported_boards: Vec::new(), + min_gpu_counts: BTreeMap::new(), extra_env: BTreeMap::new(), }]; let mut output = String::new(); @@ -2112,6 +2189,107 @@ mod tests { assert!(rendered.contains("tiny-gpt2 (2 GiB min GPU)")); } + #[test] + fn verbose_model_listing_shows_aim_gpu_families_and_token_requirement() { + let engine_lines = |id: &str| { + let recipe = rocm_core::resolve_builtin_model_recipe(id).expect("aim recipe"); + let mut output = String::new(); + append_model_engine_recipe_settings_lines(&mut output, &recipe); + output + }; + let gated = engine_lines("meta-llama/Llama-3.2-3B-Instruct"); + assert!( + gated.contains( + "engine_recipe aim supported_gfx_families=[gfx94X-dcgpu,gfx950-dcgpu] \ + requires_hf_token=yes profile_ids=[] min_gpu_counts=[]" + ), + "{gated}" + ); + // Radeon images list the profile pinned per family and the GPU counts. + let radeon = engine_lines("zai-org/GLM-4.7-Flash"); + assert!( + radeon.contains( + "supported_gfx_families=[gfx110X-all,gfx120X-all] requires_hf_token=no \ + profile_ids=[gfx110X-all=" + ) && radeon.contains("min_gpu_counts=[gfx110X-all=2,gfx120X-all=4]"), + "{radeon}" + ); + // A recipe with no AIM entry prints no such line. + let lemonade = engine_lines("qwen"); + assert!(!lemonade.contains("supported_gfx_families"), "{lemonade}"); + } + + #[test] + fn engine_action_target_follows_what_serve_would_pick_on_this_host() { + let target = |recipe_id: &str, family: &str| { + let recipe = rocm_core::resolve_builtin_model_recipe(recipe_id).expect("recipe"); + let host = std::cell::OnceCell::new(); + let _ = host.set(rocm_core::HostGpuSummary { + therock_family: Some(family.to_owned()), + ..rocm_core::HostGpuSummary::default() + }); + preferred_engine_action_target(&recipe, &host, None) + }; + // A recipe that lists no AIM is untouched by the host. + assert_eq!(target("tiny-gpt2", "gfx1151"), "vllm"); + if !cfg!(windows) { + // AIM is named only where `rocm serve` would pick it. + assert_eq!(target("Qwen/Qwen3.5-9B", "gfx110X-all"), "aim"); + assert_ne!(target("Qwen/Qwen3.5-9B", "gfx1151"), "aim"); + // Instinct hosts prefer vLLM, which `rocm serve` picks over AIM. + assert_eq!(target("Qwen/Qwen3-32B", "gfx94X-dcgpu"), "vllm"); + } + } + + #[test] + fn once_per_call_runs_the_probe_once() { + let calls = std::cell::Cell::new(0); + let cached = once_per_call(|| { + calls.set(calls.get() + 1); + false + }); + assert!(!cached() && !cached() && !cached()); + assert_eq!(calls.get(), 1); + } + + #[test] + fn verbose_model_listing_shows_a_token_requirement_without_gfx_families() { + // A schema-valid record can carry `requires_hf_token` alone; the listing + // must still say the image needs a token. + let mut recipe = + rocm_core::resolve_builtin_model_recipe("meta-llama/Llama-3.2-3B-Instruct") + .expect("aim recipe"); + for record in &mut recipe.engine_recipes { + record.supported_gfx_families.clear(); + } + let mut output = String::new(); + append_model_engine_recipe_settings_lines(&mut output, &recipe); + assert!( + output.contains( + "engine_recipe aim supported_gfx_families=[] requires_hf_token=yes" + ), + "{output}" + ); + } + + #[test] + fn model_recipe_staleness_lines_warn_only_for_stale_engine_recipes() { + let recipe = rocm_core::resolve_builtin_model_recipe("Qwen/Qwen3-32B").expect("aim entry"); + let verified = recipe.engine_recipes[0] + .generated_at_unix_ms + .expect("aim entries record when they were verified"); + let stale_ms = rocm_core::MODEL_RECIPE_STALE_AFTER_MS; + let mut fresh = String::new(); + append_model_recipe_staleness_lines(&mut fresh, &recipe, verified + stale_ms); + assert!(fresh.is_empty(), "{fresh}"); + let mut stale = String::new(); + append_model_recipe_staleness_lines(&mut stale, &recipe, verified + stale_ms + 1); + assert!( + stale.contains("warning: aim entry was last verified 180 days ago"), + "{stale}" + ); + } + #[test] fn render_model_registry_text_marks_tiny_recipe_as_gpu_smoke_support() { let rendered = render_model_registry_verbose_text_with_context_and_host(None, None, None); @@ -2139,7 +2317,12 @@ mod tests { let recipe = resolve_builtin_model_recipe("qwen32b").expect("qwen32b recipe"); let mut output = String::new(); - append_model_engine_support_lines(&mut output, &recipe, Some(&paths)); + append_model_engine_support_lines( + &mut output, + &recipe, + Some(&paths), + &rocm_engine_aim::builtin_docker_daemon_reachable, + ); let _ = fs::remove_dir_all(root); if cfg!(windows) { diff --git a/apps/rocm/src/freeform_planner.rs b/apps/rocm/src/freeform_planner.rs index 8268bbf580..9891d36b1e 100644 --- a/apps/rocm/src/freeform_planner.rs +++ b/apps/rocm/src/freeform_planner.rs @@ -368,11 +368,7 @@ fn build_freeform_plan_on( .or(requested_model) .unwrap_or(""); let engine = infer_engine_from_request(&lower) - .or_else(|| { - resolved_recipe - .as_ref() - .and_then(|recipe| recipe.preferred_engines.first().map(String::as_str)) - }) + .or_else(|| resolved_recipe.as_ref().and_then(planner_recipe_engine)) .unwrap_or(default_engine); let device_policy = infer_device_policy_from_request(&lower).or_else(|| { resolved_recipe @@ -1139,6 +1135,17 @@ pub(crate) fn render_structured_request_plan( output } +/// The recipe's first preferred engine, unless it is AIM with another engine +/// listed behind it. AIM is host-gated (GPU family, platform, Hugging Face +/// token), so the planner leaves the engine to the host default instead of +/// emitting an explicit `--engine aim` that `rocm serve` would never override. +/// An AIM-only recipe has no alternative and keeps AIM. +pub(crate) fn planner_recipe_engine(recipe: &ModelRecipeRecord) -> Option<&str> { + let first = recipe.preferred_engines.first()?; + (!(first.eq_ignore_ascii_case("aim") && recipe.preferred_engines.len() > 1)) + .then_some(first.as_str()) +} + fn planner_is_serve_request(lower: &str) -> bool { lower.contains("serve") || lower.contains("run a small local model") @@ -1346,6 +1353,44 @@ mod tests { })); } + #[test] + fn planner_does_not_emit_host_gated_aim_as_an_explicit_engine() { + // `Qwen/Qwen3-32B` lists `aim` first with `vllm` behind it; `rocm serve` + // never overrides an explicit `--engine aim`, so the plan must not name it. + let recipe = resolve_builtin_model_recipe("Qwen/Qwen3-32B").expect("catalogued id"); + assert_eq!( + recipe.preferred_engines.first().map(String::as_str), + Some("aim") + ); + assert_eq!(planner_recipe_engine(&recipe), None); + let plan = build_freeform_plan_with_recipes( + "serve Qwen/Qwen3-32B", + &RocmCliConfig::default(), + Some(&[recipe]), + "vllm", + ); + assert!( + plan.parsed + .contains(&("engine".to_owned(), "vllm".to_owned())), + "{:?}", + plan.parsed + ); + // An explicit request for AIM is still honoured. + let explicit = build_freeform_plan_with_recipes( + "serve Qwen/Qwen3-32B with aim", + &RocmCliConfig::default(), + None, + "vllm", + ); + assert!( + explicit + .parsed + .contains(&("engine".to_owned(), "aim".to_owned())), + "{:?}", + explicit.parsed + ); + } + #[test] fn hybrid_planner_can_use_active_recipe_registry_aliases() { let mut recipe = resolve_builtin_model_recipe("tiny-gpt2").expect("tiny recipe"); diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index d695dfebcc..c51a8c0156 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -491,6 +491,12 @@ rocm update --json")] command: EnginesCommand, }, /// Show recommended local models and what this machine can run. + /// + /// The default list is the curated recommendations. The AMD Inference Microservices + /// (AIM) catalog of pre-built Docker images is not part of it: use --verbose to see + /// every AIM image with the GPU families it supports, how many GPUs it needs, and + /// whether it requires a Hugging Face token. Serve one with + /// `rocm serve --engine aim` (needs a reachable Docker daemon). #[command( alias = "models", after_help = "EXAMPLES:\n \ @@ -625,7 +631,8 @@ rocm serve qwen --verbose --device gpu_required")] /// `AIM_PROFILE_ID=...`) and other engines ignore it. `KEY=VALUE` is /// recorded with the service, so for a secret such as `HF_TOKEN` pass a /// bare `KEY` instead: it is forwarded from your environment and never - /// stored. + /// stored. A gated AIM image (one that needs a Hugging Face token) is + /// refused up front unless `HF_TOKEN` is passed this way. #[arg(long = "engine-env", value_name = "KEY[=VALUE]")] engine_env: Vec, }, @@ -2717,7 +2724,7 @@ const fn builtin_engine_inventory() -> &'static [(&'static str, &'static str)] { ), ( "aim", - "AMD Enterprise AI pre-built Docker serving containers", + "AMD Inference Microservices (AIM) pre-built Docker serving containers", ), ] } @@ -3139,13 +3146,21 @@ enum PlatformRuleOut { /// Whether, and why, the platform gate rules this engine out on this host. fn engine_ruled_out_by_platform(engine: &str) -> PlatformRuleOut { + engine_ruled_out_by_platform_with(engine, rocm_engine_aim::builtin_docker_daemon_reachable) +} + +/// [`engine_ruled_out_by_platform`] with the Docker probe injected, so a caller +/// that asks about many engines in one pass (the `/model` listing) can answer +/// the same `docker info` once instead of once per recipe. +fn engine_ruled_out_by_platform_with( + engine: &str, + docker_reachable: impl FnOnce() -> bool, +) -> PlatformRuleOut { if rocm_core::runtime_is_windows() && engine.eq_ignore_ascii_case("vllm") { PlatformRuleOut::NativeWindowsVllm } else if rocm_core::runtime_is_windows() && engine.eq_ignore_ascii_case("aim") { PlatformRuleOut::NativeWindowsAim - } else if engine.eq_ignore_ascii_case("aim") - && !rocm_engine_aim::builtin_docker_daemon_reachable() - { + } else if engine.eq_ignore_ascii_case("aim") && !docker_reachable() { PlatformRuleOut::AimDockerUnavailable } else { PlatformRuleOut::Allowed @@ -3193,6 +3208,14 @@ fn ensure_engine_allowed_on_platform(engine: &str, action: GatedEngineAction) -> refuse_if_ruled_out(engine_ruled_out_by_platform(engine), action) } +/// The part of [`ensure_engine_allowed_on_platform`] that needs no Docker probe: +/// treats the daemon as reachable, so only the OS gate can refuse. `serve` runs +/// it first and the full gate after its own AIM pre-flight, so a request that +/// pre-flight refuses never reaches `docker info`. +fn ensure_engine_allowed_on_os(engine: &str, action: GatedEngineAction) -> Result<()> { + refuse_if_ruled_out(engine_ruled_out_by_platform_with(engine, || true), action) +} + /// [`ensure_engine_allowed_on_platform`] with the platform decision passed in, /// so the refusing branch is testable on any host. fn refuse_if_ruled_out(ruled_out: PlatformRuleOut, action: GatedEngineAction) -> Result<()> { @@ -5123,7 +5146,7 @@ pub(crate) const fn engine_inventory() -> &'static [(&'static str, &'static str) ), ( "aim", - "AMD Enterprise AI pre-built Docker serving containers", + "AMD Inference Microservices (AIM) pre-built Docker serving containers", ), ] } diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index a155d5c75b..abc169e26c 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -38,8 +38,9 @@ use std::fmt::Write as _; use anyhow::{Context as _, Result, bail}; use rocm_core::{ - AppPaths, ModelRecipeRecord, RocmCliConfig, default_engine_for_platform, - detect_host_gpu_summary, generate_service_id, preferred_serve_engine_for_host_gpu_summary, + AppPaths, GfxSupport, ModelRecipeRecord, RocmCliConfig, default_engine_for_platform, + detect_host_gfx_target, detect_host_gpu_summary, engine_recipe_gfx_support, + generate_service_id, normalize_therock_family, preferred_serve_engine_for_host_gpu_summary, resolve_model_recipe, }; use rocm_engine_protocol::{ @@ -69,7 +70,7 @@ use crate::service_launch::{ }; use crate::{ GatedEngineAction, cli_progress, collect_serve_notes, device_policy_name, - ensure_engine_allowed_on_platform, + ensure_engine_allowed_on_os, ensure_engine_allowed_on_platform, }; #[derive(Debug, Clone, Eq, PartialEq)] @@ -114,7 +115,7 @@ pub(crate) fn select_serve_engine( } if let Some(engine) = recipe - .and_then(|recipe| recipe.preferred_engines.first()) + .and_then(|recipe| recipe_preferred_engine(recipe, host_gpu_summary, cfg!(windows))) .filter(|value| !value.trim().is_empty()) { return ServeEngineSelection { @@ -129,6 +130,51 @@ pub(crate) fn select_serve_engine( } } +/// The recipe's first preferred engine that can run on this host. AIM is only +/// chosen on its own when the detected GPU family is one the image has a profile +/// for and the host is Linux or WSL, so an undetected GPU or native Windows is not, +/// and on native Windows vLLM is not an option once AIM has been passed over. When no listed engine fits, the +/// platform default applies, as it did before the recipe was catalogued. An +/// explicit `--engine aim` never comes through here, and still gets the refusal +/// that names the supported GPUs. +fn recipe_preferred_engine<'a>( + recipe: &'a ModelRecipeRecord, + host_gpu_summary: Option<&rocm_core::HostGpuSummary>, + windows: bool, +) -> Option<&'a str> { + let is_aim = |engine: &str| engine.eq_ignore_ascii_case("aim"); + let aim_usable = !windows + && recipe + .engine_recipes + .iter() + .find(|record| is_aim(&record.engine)) + .is_none_or(|record| { + matches!( + engine_recipe_gfx_support( + record, + host_gpu_summary.and_then(|summary| summary.therock_family.as_deref()), + ), + GfxSupport::Supported(_) + ) || record.supported_gfx_families.is_empty() + }); + // Native Windows has no vLLM, but a recipe whose first engine is vLLM keeps + // it there, as it does without the AIM catalog: the skip applies only when + // falling past an AIM first choice that was not usable. + let mut passed_unusable_aim = false; + recipe + .preferred_engines + .iter() + .map(String::as_str) + .find(|engine| { + if is_aim(engine) { + passed_unusable_aim |= !aim_usable; + aim_usable + } else { + !(windows && passed_unusable_aim && engine.eq_ignore_ascii_case("vllm")) + } + }) +} + fn model_recipe_supports_engine(recipe: &ModelRecipeRecord, engine: &str) -> bool { recipe .preferred_engines @@ -193,6 +239,184 @@ fn render_serve_engine_recipe_lines(engine_recipe: &EngineRecipeHint) -> String output } +/// For AIM, refuses a GPU the catalog has no profile for and pins the catalog's +/// `AIM_PROFILE_ID` for the detected GPU family. `host_family` is a normalized +/// TheRock family, `None` when it could not be detected (nothing is enforced +/// then). Runs before `--engine-env`, so an explicit override still wins. +fn engine_recipe_with_aim_gfx_profile( + engine: &str, + recipe: Option<&ModelRecipeRecord>, + hint: Option, + host_family: impl FnOnce() -> Option, +) -> Result> { + let Some(aim) = recipe + .filter(|_| engine.eq_ignore_ascii_case("aim")) + .and_then(|recipe| { + recipe + .engine_recipes + .iter() + .find(|record| record.engine.eq_ignore_ascii_case("aim")) + .map(|record| (recipe, record)) + }) + .filter(|(_, record)| !record.supported_gfx_families.is_empty()) + else { + return Ok(hint); + }; + let (recipe, record) = aim; + let family = host_family(); + match engine_recipe_gfx_support(record, family.as_deref()) { + GfxSupport::Unrestricted => Ok(hint), + GfxSupport::Unsupported => bail!( + "the AIM image for `{}` has no profile for this GPU ({}); it supports: {}. Choose a model \ + or engine that supports this GPU (`rocm model --verbose`)", + recipe.canonical_model_id, + family.as_deref().unwrap_or("unknown"), + record.supported_gfx_families.join(", ") + ), + GfxSupport::Supported(profile) => Ok(match (hint, profile) { + (Some(mut hint), Some(profile)) => { + hint.extra_env + .insert("AIM_PROFILE_ID".to_owned(), profile.to_owned()); + Some(hint) + } + (hint, _) => hint, + }), + } +} + +/// The AIM checks that depend on the host GPU, in the order `serve` applies +/// them: refuse a family the image has no profile for, pin the profile the +/// catalog names for the detected family, then refuse an image that needs more +/// than one GPU. The host is only probed for an AIM record that restricts GPU +/// families, and not at all for `--device cpu_only`, since these checks are about +/// the GPU the container would use. An undetected family enforces nothing. +fn aim_gpu_preflight( + engine: &str, + model: &str, + recipe: Option<&ModelRecipeRecord>, + hint: Option, + cpu_only: bool, + detect_family: impl FnOnce() -> Option, +) -> Result> { + let family = aim_engine_record(engine, recipe) + .filter(|record| !record.supported_gfx_families.is_empty() && !cpu_only) + .and_then(|_| detect_family()); + let hint = engine_recipe_with_aim_gfx_profile(engine, recipe, hint, || family.clone())?; + if let Some(needed) = aim_min_gpu_count(engine, recipe, family.as_deref()) { + ensure_aim_multi_gpu_supported(model, needed)?; + } + Ok(hint) +} + +fn aim_engine_record<'a>( + engine: &str, + recipe: Option<&'a ModelRecipeRecord>, +) -> Option<&'a rocm_core::ModelRecipeEngineRecord> { + recipe + .filter(|_| engine.eq_ignore_ascii_case("aim"))? + .engine_recipes + .iter() + .find(|record| record.engine.eq_ignore_ascii_case("aim")) +} + +/// Refuses to launch a gated AIM image before the container is pulled when no +/// Hugging Face token will reach it: `--engine-env HF_TOKEN=`, or the bare +/// `--engine-env HF_TOKEN` with `HF_TOKEN` exported (the form that is never +/// stored). `hint` carries the `--engine-env` entries. +fn ensure_aim_hf_token( + engine: &str, + recipe: Option<&ModelRecipeRecord>, + hint: Option<&EngineRecipeHint>, + env: impl Fn(&str) -> Option, +) -> Result<()> { + if !aim_engine_record(engine, recipe).is_some_and(|record| record.requires_hf_token) { + return Ok(()); + } + let supplied = hint + .and_then(|hint| hint.extra_env.get("HF_TOKEN")) + .is_some_and(|value| { + !value.trim().is_empty() || env("HF_TOKEN").is_some_and(|v| !v.trim().is_empty()) + }); + if supplied { + return Ok(()); + } + let model = recipe.map_or("", |r| r.canonical_model_id.as_str()); + bail!( + "`{model}` is a gated model and needs a Hugging Face token. Export it and pass the \ + bare name so it is forwarded and never stored: `export HF_TOKEN=`, then \ + `rocm serve {model} --engine aim --engine-env HF_TOKEN`" + ) +} + +/// GPUs an AIM image needs on `family`, when more than one (see +/// `min_gpu_counts`). `None` when it needs one, or the family is unknown. +fn aim_min_gpu_count( + engine: &str, + recipe: Option<&ModelRecipeRecord>, + family: Option<&str>, +) -> Option { + let family = family?; + aim_engine_record(engine, recipe)? + .min_gpu_counts + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(family)) + .map(|(_, count)| *count) +} + +/// Refuses an AIM image that needs `needed` GPUs before anything is pulled: +/// `rocm serve` selects a single GPU, so the container would start with too few +/// devices and fail inside the image. +fn ensure_aim_multi_gpu_supported(model: &str, needed: u32) -> Result<()> { + bail!( + "`{model}` needs {needed} GPUs for this GPU family, but `rocm serve` runs an engine on \ + one GPU and does not support multi-GPU serving yet, so this model cannot be started \ + here however many GPUs the host has. Choose a model with a single-GPU profile \ + (`rocm model --verbose`)" + ) +} + +/// Note shown when the AIM profile that was pinned for this serve targets boards +/// other than the detected GPU (a gfx family can cover cards the profile was not +/// built for). `hint` is the final engine hint: the note needs the catalog's +/// profile to be what will actually run, so it stays quiet when no profile was +/// pinned (GPU family undetected, `--device cpu_only`) or `--engine-env +/// AIM_PROFILE_ID` replaced it. +fn aim_board_note( + engine: &str, + recipe: Option<&ModelRecipeRecord>, + gpu_name: Option<&str>, + hint: Option<&EngineRecipeHint>, +) -> Option { + let record = aim_engine_record(engine, recipe)?; + let name = gpu_name?; + if record.supported_boards.is_empty() { + return None; + } + let pinned = hint?.extra_env.get("AIM_PROFILE_ID")?; + if !record.profile_ids.values().any(|id| id == pinned) { + return None; + } + let squash = |value: &str| { + value + .chars() + .filter(char::is_ascii_alphanumeric) + .collect::() + .to_ascii_lowercase() + }; + let name_key = squash(name); + if record + .supported_boards + .iter() + .any(|board| name_key.contains(&squash(board))) + { + return None; + } + Some(format!( + "the pinned AIM profile `{pinned}` targets {}; `{name}` is not one of them, so it may not run well. Override with `--engine-env AIM_PROFILE_ID=`", + record.supported_boards.join(" / ") + )) +} + fn protocol_engine_recipe_hint( recipe: &ModelRecipeRecord, engine: &str, @@ -534,6 +758,14 @@ fn engine_recipe_with_extra_env( }); for entry in engine_env { let (key, value) = entry.split_once('=').unwrap_or((entry.as_str(), "")); + // A bare `KEY` forwards it from the environment; the hint cannot carry an + // explicitly empty value, so `KEY=` would silently mean the same thing. + if entry.contains('=') && value.is_empty() { + bail!( + "--engine-env `{entry}` sets an empty value; pass `{key}` alone to forward it \ + from your environment, or `{key}=VALUE`" + ); + } if key.is_empty() || !key.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { bail!( "--engine-env expects `KEY=VALUE` (or a bare `KEY` to forward it from the \ @@ -724,7 +956,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // Before runtime resolution and any launch: an engine the platform gate rules // out (vLLM on native Windows) must not reach a "no active runtime" error or // a background server that dies with no output. - ensure_engine_allowed_on_platform(&selected_engine, GatedEngineAction::Serve)?; + ensure_engine_allowed_on_os(&selected_engine, GatedEngineAction::Serve)?; // Fail closed: a public bind must be authenticated, but Windows managed // Lemonade cannot receive the key (see `ensure_public_bind_engine_supported`), // so refuse rather than launch an open public server. @@ -735,6 +967,43 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { .as_ref() .filter(|recipe| model_recipe_supports_engine(recipe, &selected_engine)) .and_then(|recipe| protocol_engine_recipe_hint(recipe, &selected_engine)); + let device_policy = parse_device_policy(device.as_ref().map(|policy| policy.as_policy_str()))?; + // CPU-only serving never pins a GPU, so skip GPU resolution entirely and + // surface the explicit `--gpu` as ignored rather than printing a device the + // server will not use. + // + // Always `false` today, and kept anyway: `parse_device_policy` rejects + // `cpu`/`cpu_only` outright (pinned by + // `parse_device_policy_defaults_to_gpu_required_without_cpu_fallback`), so + // the only policy that reaches here is `GpuRequired`. The term is what makes + // every `!cpu_only` guard below read as a guard on the *policy* rather than + // on a constant, so admitting the policy later cannot silently turn CPU-only + // serving into GPU pre-flight failures. It is not evidence that the CPU path + // is exercised; nothing in the suite can reach it. + let cpu_only = matches!(device_policy, DevicePolicy::CpuOnly); + // The token needs nothing from the GPU, so it is checked first: a gated model + // is refused for its missing token on any host, not only on one whose GPU the + // image supports. Only `--engine-env` can supply it, so a hint built from + // that alone is enough; the real hint below still gets the same env applied + // after the catalog's profile pin, which is what lets the flag override it. + ensure_aim_hf_token( + &selected_engine, + shared_recipe.as_ref(), + engine_recipe_with_extra_env(&selected_engine, None, &engine_env)?.as_ref(), + |key| std::env::var(key).ok(), + )?; + let recipe_hint = aim_gpu_preflight( + &selected_engine, + &model, + shared_recipe.as_ref(), + recipe_hint, + cpu_only, + || { + detect_host_gfx_target() + .as_deref() + .and_then(normalize_therock_family) + }, + )?; // vLLM rejects the TUI chat tab's tool-bearing requests with HTTP 400 unless it // is launched with `--enable-auto-tool-choice`/`--tool-call-parser`. The parser // is model-specific and vLLM does not auto-detect it, so it is never guessed: it @@ -775,7 +1044,26 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { generation_defaults, )?; let engine_recipe = engine_recipe_with_extra_env(&selected_engine, engine_recipe, &engine_env)?; - let engine_env_note = engine_env_ignored_note(&selected_engine, &engine_env); + let aim_gpu_name = aim_engine_record(&selected_engine, shared_recipe.as_ref()) + .filter(|record| !record.profile_ids.is_empty()) + .and_then(|_| { + host_gpu_summary + .clone() + .unwrap_or_else(|| detect_host_gpu_summary(Some(&paths))) + .name + }); + let engine_env_notes: Vec = [ + engine_env_ignored_note(&selected_engine, &engine_env), + aim_board_note( + &selected_engine, + shared_recipe.as_ref(), + aim_gpu_name.as_deref(), + engine_recipe.as_ref(), + ), + ] + .into_iter() + .flatten() + .collect(); let tool_call_note = if tool_call_parser.is_some() && !engine_serves_vllm { Some(format!( "note: --tool-call-parser applies only to vLLM; ignored for engine '{selected_engine}'" @@ -787,21 +1075,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { } else { None }; - let device_policy = parse_device_policy(device.as_ref().map(|policy| policy.as_policy_str()))?; let gpu_selection = parse_gpu_selection(gpu.as_deref())?; - // CPU-only serving never pins a GPU, so skip GPU resolution entirely and - // surface the explicit `--gpu` as ignored rather than printing a device the - // server will not use. - // - // Always `false` today, and kept anyway: `parse_device_policy` rejects - // `cpu`/`cpu_only` outright (pinned by - // `parse_device_policy_defaults_to_gpu_required_without_cpu_fallback`), so - // the only policy that reaches here is `GpuRequired`. The term is what makes - // every `!cpu_only` guard below read as a guard on the *policy* rather than - // on a constant, so admitting the policy later cannot silently turn CPU-only - // serving into GPU pre-flight failures. It is not evidence that the CPU path - // is exercised; nothing in the suite can reach it. - let cpu_only = matches!(device_policy, DevicePolicy::CpuOnly); // AMD GPU ordinals still usable after the active visibility mask // (`HIP_VISIBLE_DEVICES`, then `ROCR_VISIBLE_DEVICES`) is applied, in HIP // ordinal space — the space `--gpu` is validated and exported through. A @@ -984,6 +1258,10 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { if !cpu_only && !scripted_backend_failure && !reuse_existing && host_has_no_usable_gpu { bail!("{}", no_usable_gpu_refusal(&device_policy)); } + // The Docker probe comes after every refusal that needs no daemon (the AIM + // token and GPU pre-flight, the no-usable-GPU bail): a request they refuse + // never reaches `docker info`. + ensure_engine_allowed_on_platform(&selected_engine, GatedEngineAction::Serve)?; // Whether the refusal above was waived *only* because a live service was // expected to be reused. `reuse_existing` already implies `!cpu_only` and // `host_has_no_usable_gpu` (the pre-gate requires both), so the scripted @@ -1185,7 +1463,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { if let Some(note) = &gpu_memory_utilization_note { println!(" note: {note}"); } - if let Some(note) = &engine_env_note { + for note in &engine_env_notes { println!(" note: {note}"); } } @@ -1268,9 +1546,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { host_gpu_summary.as_ref(), engine_serves_vllm, ); - if let Some(note) = &engine_env_note { - notes.push(note.clone()); - } + notes.extend(engine_env_notes.iter().cloned()); let summary = serve_summary::DeploymentSummary { engine: selected_engine.clone(), requested_model: model, @@ -1455,6 +1731,87 @@ mod tests { ); } + #[test] + fn aim_catalogued_hf_ids_pick_aim_only_on_a_gpu_it_supports() { + // These ids had no recipe before the AIM catalog; cataloguing them must not + // move an Instinct host off vLLM, nor send a host the image has no profile + // for (or an undetected GPU, or native Windows) to an engine that refuses it. + let host = |family: Option<&str>| rocm_core::HostGpuSummary { + therock_family: family.map(str::to_owned), + ..rocm_core::HostGpuSummary::default() + }; + let instinct = host(Some("gfx94X-dcgpu")); + let radeon = host(Some("gfx110X-all")); + let strix = host(Some("gfx1151")); + let undetected = host(None); + + // An Instinct-only image (`Qwen/Qwen3-32B`) and a Radeon one + // (`Qwen/Qwen3.5-9B`) each get AIM on their own family and their next + // listed engine (vLLM) elsewhere, or the platform default on Windows. + let instinct_image = resolve_builtin_model_recipe("Qwen/Qwen3-32B").unwrap(); + let radeon_image = resolve_builtin_model_recipe("Qwen/Qwen3.5-9B").unwrap(); + for (recipe, family, windows, expected) in [ + (&instinct_image, &instinct, false, Some("aim")), + (&instinct_image, &instinct, true, None), + (&instinct_image, &radeon, false, Some("vllm")), + (&instinct_image, &strix, false, Some("vllm")), + (&instinct_image, &undetected, false, Some("vllm")), + (&radeon_image, &radeon, false, Some("aim")), + (&radeon_image, &radeon, true, None), + (&radeon_image, &instinct, false, Some("vllm")), + (&radeon_image, &strix, false, Some("vllm")), + ] { + assert_eq!( + recipe_preferred_engine(recipe, Some(family), windows), + expected, + "{} on {:?} (windows: {windows})", + recipe.canonical_model_id, + family.therock_family + ); + } + // No GPU summary at all is treated like an undetected GPU. + assert_eq!( + recipe_preferred_engine(&radeon_image, None, false), + Some("vllm") + ); + // Recipes that do not prefer AIM are untouched. + let vllm_only = resolve_builtin_model_recipe("meta-llama/Llama-3.2-3B-Instruct").unwrap(); + assert_eq!( + recipe_preferred_engine(&vllm_only, Some(&strix), false), + Some("vllm") + ); + // A recipe that lists vLLM first keeps it on native Windows (the skip + // applies only past an unusable AIM first choice), as it does on main. + assert_eq!( + recipe_preferred_engine(&vllm_only, Some(&strix), true), + Some("vllm") + ); + + // End to end through the selector. A vLLM-preferred host keeps vLLM; the + // rest follow the table above (the platform default is OS-specific, so + // compare against it rather than a literal). + let on_instinct = select_serve_engine(None, None, Some(&instinct_image), Some(&instinct)); + if !cfg!(windows) { + assert_eq!(on_instinct.engine, "vllm"); + let on_radeon = select_serve_engine(None, None, Some(&radeon_image), Some(&radeon)); + assert_eq!(on_radeon.engine, "aim"); + } + let on_strix = select_serve_engine(None, None, Some(&radeon_image), Some(&strix)); + // AIM has no profile for Strix Halo, so the recipe's next engine is used; + // native Windows has no vLLM either, so it takes the platform default. + assert_eq!( + on_strix.engine, + if cfg!(windows) { + default_engine_for_platform() + } else { + "vllm" + } + ); + // An explicit engine is never overridden. + let explicit = select_serve_engine(Some("aim"), None, Some(&radeon_image), Some(&strix)); + assert_eq!(explicit.engine, "aim"); + } + #[test] fn serve_qwen_uses_vllm_with_hf_repo_on_vllm_preferred_gpu() { // The qwen alias serves the GGUF via Lemonade by default, but on a vLLM-preferred @@ -1515,17 +1872,32 @@ mod tests { } #[test] - fn catalog_extra_env_reaches_the_aim_hint_and_engine_env_overrides_it() { + fn catalog_profile_pin_reaches_the_aim_hint_and_engine_env_overrides_it() { let recipe = resolve_builtin_model_recipe("aim-qwen3.5-9b").expect("aim catalog entry"); - let hint = protocol_engine_recipe_hint(&recipe, "aim").expect("aim hint"); + let hint = protocol_engine_recipe_hint(&recipe, "aim"); + let pinned = |family: &'static str| { + engine_recipe_with_aim_gfx_profile("aim", Some(&recipe), hint.clone(), || { + Some(family.to_owned()) + }) + .unwrap() + .unwrap() + }; + let w7900 = pinned("gfx110X-all"); assert_eq!( - hint.extra_env.get("AIM_PROFILE_ID").map(String::as_str), + w7900.extra_env.get("AIM_PROFILE_ID").map(String::as_str), Some("vllm-w7900-bf16-tp1-latency") ); + assert_eq!( + pinned("gfx120X-all") + .extra_env + .get("AIM_PROFILE_ID") + .map(String::as_str), + Some("vllm-r9700-bf16-tp1-latency") + ); let overridden = engine_recipe_with_extra_env( "aim", - Some(hint), + Some(w7900), &[ "AIM_PROFILE_ID=vllm-other".to_owned(), "HF_TOKEN".to_owned(), @@ -1544,6 +1916,266 @@ mod tests { assert_eq!(overridden.extra_env.get("HF_TOKEN"), Some(&String::new())); } + #[test] + fn aim_serve_refuses_a_gpu_the_image_has_no_profile_for() { + let radeon = resolve_builtin_model_recipe("aim-qwen3.5-9b").expect("radeon aim entry"); + let hint = protocol_engine_recipe_hint(&radeon, "aim"); + let err = engine_recipe_with_aim_gfx_profile("aim", Some(&radeon), hint, || { + Some("gfx94X-dcgpu".to_owned()) + }) + .unwrap_err() + .to_string(); + assert!( + err.contains("no profile for this GPU (gfx94X-dcgpu)"), + "{err}" + ); + assert!(err.contains("gfx110X-all, gfx120X-all"), "{err}"); + + // Strix Halo has no AIM profile either. + let hint = protocol_engine_recipe_hint(&radeon, "aim"); + assert!( + engine_recipe_with_aim_gfx_profile("aim", Some(&radeon), hint, || Some( + "gfx1151".to_owned() + )) + .is_err() + ); + + // Instinct images serve on Instinct without pinning a profile (AIM + // auto-selects), and are refused on a Radeon. + let instinct = resolve_builtin_model_recipe("Qwen/Qwen3-32B").expect("instinct aim entry"); + let hint = protocol_engine_recipe_hint(&instinct, "aim"); + let served = + engine_recipe_with_aim_gfx_profile("aim", Some(&instinct), hint.clone(), || { + Some("gfx94X-dcgpu".to_owned()) + }) + .unwrap() + .unwrap(); + assert!(!served.extra_env.contains_key("AIM_PROFILE_ID")); + assert!( + engine_recipe_with_aim_gfx_profile("aim", Some(&instinct), hint, || Some( + "gfx110X-all".to_owned() + )) + .is_err() + ); + } + + #[test] + fn gated_aim_model_needs_an_hf_token_before_launch() { + let gated = resolve_builtin_model_recipe("meta-llama/Llama-3.2-3B-Instruct") + .expect("gated aim entry"); + let hint = |engine_env: &[&str]| { + engine_recipe_with_extra_env( + "aim", + protocol_engine_recipe_hint(&gated, "aim"), + &engine_env + .iter() + .map(|s| (*s).to_owned()) + .collect::>(), + ) + .unwrap() + }; + let no_env = |_: &str| None; + let exported = |_: &str| Some("hf_x".to_owned()); + + // Without the flag, or with the bare flag but nothing exported, refuse and + // name the command that clears it. + for env in [&no_env as &dyn Fn(&str) -> Option, &exported] { + let err = ensure_aim_hf_token("aim", Some(&gated), hint(&[]).as_ref(), env) + .unwrap_err() + .to_string(); + assert!(err.contains("--engine-env HF_TOKEN"), "{err}"); + } + assert!( + ensure_aim_hf_token("aim", Some(&gated), hint(&["HF_TOKEN"]).as_ref(), no_env).is_err() + ); + // The advised form (bare flag with the variable exported) or an explicit + // value clears it, and other engines are never checked. + assert!( + ensure_aim_hf_token("aim", Some(&gated), hint(&["HF_TOKEN"]).as_ref(), exported) + .is_ok() + ); + assert!( + ensure_aim_hf_token( + "aim", + Some(&gated), + hint(&["HF_TOKEN=hf_y"]).as_ref(), + no_env + ) + .is_ok() + ); + assert!(ensure_aim_hf_token("vllm", Some(&gated), None, no_env).is_ok()); + let open = resolve_builtin_model_recipe("Qwen/Qwen3-32B").expect("ungated aim entry"); + assert!(ensure_aim_hf_token("aim", Some(&open), None, no_env).is_ok()); + } + + #[test] + fn multi_gpu_aim_image_is_refused_before_launch() { + let id = "deepseek-ai/DeepSeek-R1"; + let r1 = resolve_builtin_model_recipe(id).expect("multi-gpu aim entry"); + assert_eq!( + aim_min_gpu_count("aim", Some(&r1), Some("gfx94X-dcgpu")), + Some(8) + ); + assert_eq!(aim_min_gpu_count("aim", Some(&r1), Some("gfx1151")), None); + assert_eq!(aim_min_gpu_count("aim", Some(&r1), None), None); + assert_eq!( + aim_min_gpu_count("vllm", Some(&r1), Some("gfx94X-dcgpu")), + None + ); + // A single-GPU image has no requirement. + let small = resolve_builtin_model_recipe("Qwen/Qwen3-32B").expect("single-gpu entry"); + assert_eq!( + aim_min_gpu_count("aim", Some(&small), Some("gfx94X-dcgpu")), + None + ); + // Radeon needs differ per family. + let flash = resolve_builtin_model_recipe("zai-org/GLM-4.7-Flash").expect("flash entry"); + assert_eq!( + aim_min_gpu_count("aim", Some(&flash), Some("gfx110X-all")), + Some(2) + ); + assert_eq!( + aim_min_gpu_count("aim", Some(&flash), Some("gfx120X-all")), + Some(4) + ); + + let err = ensure_aim_multi_gpu_supported(id, 8) + .unwrap_err() + .to_string(); + assert!(err.contains("needs 8 GPUs") && err.contains("does not support multi-GPU")); + } + + #[test] + fn aim_board_note_only_fires_when_the_catalog_profile_is_what_runs() { + let radeon = resolve_builtin_model_recipe("aim-qwen3.5-9b").expect("radeon aim entry"); + // The hint the serve path ends up with once a GPU family was detected. + let pinned = engine_recipe_with_aim_gfx_profile( + "aim", + Some(&radeon), + protocol_engine_recipe_hint(&radeon, "aim"), + || Some("gfx110X-all".to_owned()), + ) + .unwrap(); + let note = |gpu: Option<&str>, hint: Option<&EngineRecipeHint>| { + aim_board_note("aim", Some(&radeon), gpu, hint) + }; + assert!(note(Some("AMD Radeon PRO W7900"), pinned.as_ref()).is_none()); + assert!(note(Some("AMD Radeon AI PRO R9700"), pinned.as_ref()).is_none()); + let other = note(Some("AMD Radeon RX 7900 XTX"), pinned.as_ref()).expect("warns"); + assert!( + other.contains("vllm-w7900-bf16-tp1-latency") + && other.contains("W7900 / R9700") + && other.contains("--engine-env AIM_PROFILE_ID"), + "{other}" + ); + assert!(note(None, pinned.as_ref()).is_none()); + + // No pin applied (family undetected, or `--device cpu_only`): no claim about it. + let unpinned = engine_recipe_with_aim_gfx_profile( + "aim", + Some(&radeon), + protocol_engine_recipe_hint(&radeon, "aim"), + || None, + ) + .unwrap(); + assert!(note(Some("AMD Radeon RX 7900 XTX"), unpinned.as_ref()).is_none()); + assert!(note(Some("AMD Radeon RX 7900 XTX"), None).is_none()); + // An explicit AIM_PROFILE_ID replaced the catalog's: its board is the user's call. + let overridden = + engine_recipe_with_extra_env("aim", pinned, &["AIM_PROFILE_ID=vllm-other".to_owned()]) + .unwrap(); + assert!(note(Some("AMD Radeon RX 7900 XTX"), overridden.as_ref()).is_none()); + // Instinct images pin no profile, so there is nothing to mismatch. + let instinct = resolve_builtin_model_recipe("Qwen/Qwen3-32B").expect("instinct entry"); + assert!(aim_board_note("aim", Some(&instinct), Some("anything"), None).is_none()); + } + + #[test] + fn aim_gpu_preflight_probes_only_for_a_restricted_aim_record_off_cpu_only() { + let radeon = resolve_builtin_model_recipe("aim-qwen3.5-9b").expect("radeon aim entry"); + let hint = protocol_engine_recipe_hint(&radeon, "aim"); + let preflight = |engine: &str, cpu_only: bool, family: fn() -> Option| { + aim_gpu_preflight(engine, "m", Some(&radeon), hint.clone(), cpu_only, family) + }; + let no_probe = || -> Option { panic!("host GPU must not be probed") }; + + // An undetected family enforces nothing. + assert_eq!(preflight("aim", false, || None).unwrap(), hint); + // The probe runs for AIM off `cpu_only`, and the result is enforced. + assert!(preflight("aim", false, || Some("gfx94X-dcgpu".to_owned())).is_err()); + // Neither a different engine nor `--device cpu_only` reaches the probe, + // so an unsupported family cannot refuse them. + assert_eq!(preflight("vllm", false, no_probe).unwrap(), hint); + assert_eq!(preflight("aim", true, no_probe).unwrap(), hint); + } + + #[test] + fn aim_gpu_preflight_pins_the_profile_and_then_refuses_multi_gpu_images() { + let radeon = resolve_builtin_model_recipe("aim-qwen3.5-9b").expect("radeon aim entry"); + let pinned = aim_gpu_preflight( + "aim", + "m", + Some(&radeon), + protocol_engine_recipe_hint(&radeon, "aim"), + false, + || Some("gfx110X-all".to_owned()), + ) + .unwrap() + .expect("hint"); + assert_eq!( + pinned.extra_env.get("AIM_PROFILE_ID").map(String::as_str), + Some("vllm-w7900-bf16-tp1-latency") + ); + + // The multi-GPU refusal comes from the same pre-flight, and only for a + // family whose count exceeds one. + let multi = resolve_builtin_model_recipe("zai-org/GLM-4.7-Flash").expect("multi-GPU entry"); + let hint = || protocol_engine_recipe_hint(&multi, "aim"); + let err = aim_gpu_preflight("aim", "m", Some(&multi), hint(), false, || { + Some("gfx120X-all".to_owned()) + }) + .unwrap_err(); + assert!(err.to_string().contains("multi-GPU serving yet"), "{err:#}"); + assert!( + aim_gpu_preflight("aim", "m", Some(&multi), hint(), true, || None).is_ok(), + "cpu_only skips the multi-GPU check" + ); + } + + #[test] + fn recipe_preferred_engine_treats_unrestricted_and_missing_aim_records_as_usable() { + let host = rocm_core::HostGpuSummary { + therock_family: Some("gfx1151".to_owned()), + ..rocm_core::HostGpuSummary::default() + }; + let mut recipe = resolve_builtin_model_recipe("Qwen/Qwen3-32B").unwrap(); + assert_eq!( + recipe_preferred_engine(&recipe, Some(&host), false), + Some("vllm") + ); + // Only a signed external index reaches these two arms, so they are built + // here: an AIM record that restricts no family is usable on any GPU, and + // a recipe that lists AIM first but carries no AIM record has nothing to + // gate it. + for record in &mut recipe.engine_recipes { + if record.engine == "aim" { + record.supported_gfx_families.clear(); + } + } + assert_eq!( + recipe_preferred_engine(&recipe, Some(&host), false), + Some("aim") + ); + recipe + .engine_recipes + .retain(|record| record.engine != "aim"); + assert_eq!( + recipe_preferred_engine(&recipe, Some(&host), false), + Some("aim") + ); + assert_eq!(recipe_preferred_engine(&recipe, Some(&host), true), None); + } + #[test] fn explicit_engine_override_keeps_alias_when_shared_recipe_is_for_another_engine() { // `qwen-smoke` is a Lemonade-only GGUF recipe (no vLLM engine recipe). @@ -1624,6 +2256,12 @@ mod tests { notes: vec!["adapter hint".to_owned()], model_id_override: None, image_digest: None, + generated_at_unix_ms: None, + supported_gfx_families: Vec::new(), + profile_ids: BTreeMap::new(), + requires_hf_token: false, + supported_boards: Vec::new(), + min_gpu_counts: BTreeMap::new(), extra_env: BTreeMap::new(), }, rocm_core::ModelRecipeEngineRecord { @@ -1635,6 +2273,12 @@ mod tests { notes: Vec::new(), model_id_override: None, image_digest: None, + generated_at_unix_ms: None, + supported_gfx_families: Vec::new(), + profile_ids: BTreeMap::new(), + requires_hf_token: false, + supported_boards: Vec::new(), + min_gpu_counts: BTreeMap::new(), extra_env: BTreeMap::new(), }, ]; @@ -2153,6 +2797,20 @@ mod tests { ); } + #[test] + fn engine_env_distinguishes_a_bare_key_from_an_empty_value() { + let env = |entry: &str| engine_recipe_with_extra_env("aim", None, &[entry.to_owned()]); + let bare = env("HF_TOKEN").unwrap().expect("hint"); + assert_eq!(bare.extra_env.get("HF_TOKEN").map(String::as_str), Some("")); + let err = env("HF_TOKEN=").unwrap_err().to_string(); + assert!( + err.contains("empty value") && err.contains("`HF_TOKEN` alone"), + "{err}" + ); + let set = env("A=b").unwrap().expect("hint"); + assert_eq!(set.extra_env.get("A").map(String::as_str), Some("b")); + } + #[test] fn engine_env_ignored_note_is_none_for_aim() { assert_eq!( diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index f425eea774..aa41736f85 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -124,15 +124,17 @@ use managed_runtime::{ TheRockFamilyManifest, managed_sdk_tool_path, sort_records_newest_install_first, }; pub use model_catalog::{ - ModelArtifactCacheStatus, ModelCatalogPlatform, ModelRecipeArtifactRecord, - ModelRecipeArtifactSourcePolicyRecord, ModelRecipeEndpointRecord, ModelRecipeEngineRecord, - ModelRecipeIndexDocument, ModelRecipeRecord, ModelRecipeRegistry, ModelRecipeRegistrySource, - ModelRecipeUnsupportedCombinationRecord, builtin_model_recipe_registry, builtin_model_recipes, + GfxSupport, MODEL_RECIPE_STALE_AFTER_MS, ModelArtifactCacheStatus, ModelCatalogPlatform, + ModelRecipeArtifactRecord, ModelRecipeArtifactSourcePolicyRecord, ModelRecipeEndpointRecord, + ModelRecipeEngineRecord, ModelRecipeIndexDocument, ModelRecipeRecord, ModelRecipeRegistry, + ModelRecipeRegistrySource, ModelRecipeUnsupportedCombinationRecord, + builtin_model_recipe_registry, builtin_model_recipes, engine_recipe_gfx_support, load_model_recipe_index, load_model_recipe_registry, load_signed_model_recipe_index, model_artifact_cache_marker_path, model_artifact_cache_status, model_catalog_platforms, - model_recipe_featured, model_recipe_index_signature_path, model_recipe_target_platform_label, - normalize_model_ref, platform_matches_gfx_family, resolve_builtin_model_recipe, - resolve_model_recipe, resolve_model_recipe_artifact, verify_model_recipe_index_signature, + model_recipe_featured, model_recipe_index_signature_path, model_recipe_staleness_warning, + model_recipe_target_platform_label, model_recipe_targets_platform, normalize_model_ref, + platform_matches_gfx_family, resolve_builtin_model_recipe, resolve_model_recipe, + resolve_model_recipe_artifact, verify_model_recipe_index_signature, }; pub use net::{ Backoff, DEFAULT_LOCAL_HOST, DEFAULT_LOCAL_PORT, DOWNLOAD_MAX_ATTEMPTS, DownloadOutcome, diff --git a/crates/rocm-core/src/model_catalog.json b/crates/rocm-core/src/model_catalog.json index c107637f3a..be7961c3aa 100644 --- a/crates/rocm-core/src/model_catalog.json +++ b/crates/rocm-core/src/model_catalog.json @@ -4,8 +4,8 @@ "generated_at_unix_ms": null, "platforms": [ { - "label": "AMD Ryzen AI - Strix Halo (Lemonade / llama.cpp / AIM)", - "engines": ["lemonade", "llama.cpp", "llamacpp", "aim"], + "label": "AMD Ryzen AI - Strix Halo (Lemonade / llama.cpp)", + "engines": ["lemonade", "llama.cpp", "llamacpp"], "gfx_families": ["gfx1151"] }, { @@ -14,8 +14,8 @@ "gfx_families": ["gfx110X-all", "gfx120X-all"] }, { - "label": "AMD Instinct - MI300X, MI350X, MI355X (vLLM)", - "engines": ["vllm"], + "label": "AMD Instinct - MI300X, MI350X, MI355X (vLLM / AIM)", + "engines": ["vllm", "aim"], "gfx_families": ["gfx906", "gfx908", "gfx90a", "gfx94X-dcgpu", "gfx950-dcgpu"] } ], @@ -330,6 +330,24 @@ "unsupported_combinations": [], "notes": [], "model_id_override": null + }, + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-2-3b-instruct:0.11.1", + "image_digest": "sha256:12e146a90d3ad26bafc102ac0217f5d49b3107c9e7abc476d16ad3f1ae7de4ae", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "requires_hf_token": true, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} } ], "manual_alternatives": [ @@ -429,7 +447,24 @@ "quantization": "BF16, ~62 GiB (1x MI300X)", "artifact_hint": "Hugging Face model id; vLLM ROCm serving validates artifacts during serve", "artifacts": [], - "engine_recipes": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-google-gemma-4-31b-it:0.13.0-preview", + "image_digest": "sha256:ec2c6940ebd7088f112d09c4c6f554e4d3efaa88b61c89be9f1ecd347beaef9a", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], "manual_alternatives": [ "qwen3-32b-fp8" ], @@ -510,8 +545,9 @@ ] }, { - "canonical_model_id": "aim-qwen3.5-9b", + "canonical_model_id": "Qwen/Qwen3.5-9B", "aliases": [ + "aim-qwen3.5-9b", "amdenterpriseai/aim-radeon-qwen-qwen3-5-9b" ], "task": "chat", @@ -524,7 +560,7 @@ "min_gpu_mem_gb": 24, "recommended_system_ram_gb": 32, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -534,23 +570,1607 @@ "preferred_endpoint": null, "unsupported_combinations": [], "notes": [ - "pins AIM_PROFILE_ID=vllm-w7900-bf16-tp1-latency via extra_env; Radeon profile auto-selection isn't reliable yet, override with --engine-env if needed", + "pins AIM_PROFILE_ID per GPU family from profile_ids because Radeon profile auto-selection isn't reliable yet; override with --engine-env AIM_PROFILE_ID= if needed", "requires a reachable Docker daemon; the image is pulled on first serve" ], "model_id_override": "amdenterpriseai/aim-radeon-qwen-qwen3-5-9b:0.12.0-preview", "image_digest": "sha256:ee1201261f358be881318de5930ce9505d17309486f747a83147d7d042fc95ce", - "extra_env": { - "AIM_PROFILE_ID": "vllm-w7900-bf16-tp1-latency" - } + "supported_gfx_families": ["gfx110X-all", "gfx120X-all"], + "supported_boards": ["W7900", "R9700"], + "profile_ids": {"gfx110X-all": "vllm-w7900-bf16-tp1-latency", "gfx120X-all": "vllm-r9700-bf16-tp1-latency"}, + "generated_at_unix_ms": 1793203564000 + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "Radeon-targeted AIM image (W7900 / R9700 profiles only); requires Docker" + ] + }, + { + "canonical_model_id": "CohereLabs/command-a-reasoning-08-2025", + "aliases": [ + "amdenterpriseai/aim-coherelabs-command-a-reasoning-08-2025" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 267, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-coherelabs-command-a-reasoning-08-2025:0.11.1", + "image_digest": "sha256:6da329641e3ea98e945f43b1df17b7efb3ba0a284dfb123297ace72d4084b4d3", + "supported_gfx_families": ["gfx90a", "gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx90a": 8, "gfx94X-dcgpu": 2}, + "requires_hf_token": true, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "deepseek-ai/DeepSeek-R1-0528", + "aliases": [ + "amdenterpriseai/aim-deepseek-ai-deepseek-r1-0528" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 806, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-r1-0528:0.11.1", + "image_digest": "sha256:348f6ee92b07bf150b01c7b37adb8365336ffc98a2a6f747ed73754cd41f61bc", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 8, "gfx950-dcgpu": 8}, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "deepseek-ai/DeepSeek-R1", + "aliases": [ + "amdenterpriseai/aim-deepseek-ai-deepseek-r1" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 806, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-r1:0.11.1", + "image_digest": "sha256:4bbea192e4c3fd11865652119c584e9bad27b07634d5d56da2ca1bcf1c74473d", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 8, "gfx950-dcgpu": 8}, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "deepseek-ai/DeepSeek-V3.1-Terminus", + "aliases": [ + "amdenterpriseai/aim-deepseek-ai-deepseek-v3-1-terminus" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 806, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-v3-1-terminus:0.11.1", + "image_digest": "sha256:5f0335322a88bd4a8abe0865227287f419b906b47d9913cf8de7a35df80142f4", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 8, "gfx950-dcgpu": 8}, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "deepseek-ai/DeepSeek-V3.1", + "aliases": [ + "amdenterpriseai/aim-deepseek-ai-deepseek-v3-1" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 806, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-v3-1:0.11.1", + "image_digest": "sha256:9fc6f3c6e5b165352ed0873d60bc1acfc1aeaebc048da4630d3166e0b498fb33", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 8, "gfx950-dcgpu": 8}, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "deepseek-ai/DeepSeek-V4-Flash-0731", + "aliases": [ + "amdenterpriseai/aim-deepseek-ai-deepseek-v4-flash-0731" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-v4-flash-0731:2026.11.0", + "image_digest": "sha256:bb4e2c8b9ecfcfac886d7c5ebf2629c62ded2debac804d8bf2c11483f7ac6d8c", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 2, "gfx950-dcgpu": 2}, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "google/gemma-3-1b-it", + "aliases": [ + "amdenterpriseai/aim-google-gemma-3-1b-it" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 3, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-google-gemma-3-1b-it:0.12.0", + "image_digest": "sha256:fb49e69fe72ca40efaef97173d1532e09982a141900905015d41267ced1654cf", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "requires_hf_token": true, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} } ], "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "google/gemma-3-27b-it", + "aliases": [ + "amdenterpriseai/aim-google-gemma-3-27b-it" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 65, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-google-gemma-3-27b-it:0.13.0", + "image_digest": "sha256:2340abdb1c7cf34a84c5398607b57e1a7cf73b7099ba3fdbde8fbb810e849e33", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "requires_hf_token": true, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "google/medgemma-27b-it", + "aliases": [ + "amdenterpriseai/aim-google-medgemma-27b-it" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 65, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-google-medgemma-27b-it:0.13.0", + "image_digest": "sha256:a89823487230a8cea1f9d49f881e64f20375ee793739c223ab2c09f7bc30a088", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "requires_hf_token": true, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "meta-llama/Llama-3.1-405B-Instruct", + "aliases": [ + "amdenterpriseai/aim-meta-llama-llama-3-1-405b-instruct" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-1-405b-instruct:0.11.1", + "image_digest": "sha256:bfe038184d0f0d8a415ee057f4fe64b984dbb743874d0907448be9ceed432743", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 4}, + "requires_hf_token": true, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "meta-llama/Llama-3.1-8B-Instruct", + "aliases": [ + "amdenterpriseai/aim-radeon-meta-llama-llama-3-1-8b-instruct" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 20, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", + "requires a reachable Docker daemon; the image is pulled on first serve", + "Radeon-targeted image; the AIM profile is pinned per GPU family from profile_ids, override with --engine-env AIM_PROFILE_ID=" + ], + "model_id_override": "amdenterpriseai/aim-radeon-meta-llama-llama-3-1-8b-instruct:2026.11.0", + "image_digest": "sha256:a300b53f5f919a59d0cdc4bc177a6db5ef1660719cb50c128be6f8ec4a7c79d5", + "supported_gfx_families": ["gfx110X-all", "gfx120X-all"], + "supported_boards": ["W7900", "R9700"], + "requires_hf_token": true, + "profile_ids": {"gfx110X-all": "vllm-w7900-bf16-tp1-latency", "gfx120X-all": "vllm-r9700-bf16-tp1-latency"}, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "meta-llama/Llama-3.2-1B-Instruct", + "aliases": [ + "amdenterpriseai/aim-meta-llama-llama-3-2-1b-instruct" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 3, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-2-1b-instruct:0.11.1", + "image_digest": "sha256:92e87333a235d5af0494d3148a5dea537da041de671b8b9a9db05bfbd2bfb164", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "requires_hf_token": true, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "meta-llama/Llama-3.3-70B-Instruct", + "aliases": [ + "amdenterpriseai/aim-meta-llama-llama-3-3-70b-instruct" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 85, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-3-70b-instruct:0.11.1", + "image_digest": "sha256:c8d1bb99119241592d5749e7e707224154a7acae081558d5a30caefc75c32338", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "requires_hf_token": true, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "MiniMaxAI/MiniMax-M2.5", + "aliases": [ + "amdenterpriseai/aim-minimaxai-minimax-m2-5" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 276, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-minimaxai-minimax-m2-5:0.12.0", + "image_digest": "sha256:0a20a6dd756dc3ebc9cc0ada23f95e8edf49fbe33354652702f6d878380323ed", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 2}, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "MiniMaxAI/MiniMax-M3", + "aliases": [ + "amdenterpriseai/aim-minimaxai-minimax-m3" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-minimaxai-minimax-m3:2026.9.0", + "image_digest": "sha256:2b7221c5fa3592afefb0e904d10b063126ebe76f71ca7a5b8cd2c2a8a26c10c9", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 4, "gfx950-dcgpu": 4}, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "mistralai/Ministral-3-14B-Instruct-2512", + "aliases": [ + "amdenterpriseai/aim-mistralai-ministral-3-14b-instruct-2512" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 17, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-mistralai-ministral-3-14b-instruct-2512:0.11.1", + "image_digest": "sha256:8fa94cb5351554ca32c15543b48d74b79ef214a29f8066501c841d38798f6504", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "mistralai/Ministral-3-14B-Reasoning-2512", + "aliases": [ + "amdenterpriseai/aim-mistralai-ministral-3-14b-reasoning-2512" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 17, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-mistralai-ministral-3-14b-reasoning-2512:0.11.1", + "image_digest": "sha256:50cb5fc4f2a2a6973f63adcf0f8621d091dd825f67f3808cb6d743ded9d2a00a", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "mistralai/Mistral-Large-3-675B-Instruct-2512", + "aliases": [ + "amdenterpriseai/aim-mistralai-mistral-large-3-675b-instruct-2512" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 810, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-mistralai-mistral-large-3-675b-instruct-2512:0.10.0", + "image_digest": "sha256:6119b9c717b44ff4bec3183318623738d2d95a99ecd85e0fe5bdca98bf24dbdd", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 8, "gfx950-dcgpu": 8}, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "mistralai/Mistral-Small-24B-Instruct-2501", + "aliases": [ + "amdenterpriseai/aim-mistralai-mistral-small-24b-instruct-2501" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 58, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-mistralai-mistral-small-24b-instruct-2501:0.13.0", + "image_digest": "sha256:976a1e5a9880756058dad68dd3f543d76f01f1955a436f1bf5b03832704e2b20", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", + "aliases": [ + "amdenterpriseai/aim-mistralai-mistral-small-3-2-24b-instruct-2506" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 58, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-mistralai-mistral-small-3-2-24b-instruct-2506:0.11.1", + "image_digest": "sha256:7883d07cc867795a307073855a716b0c7e623e7f1327a6c347a338f95760f719", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "mistralai/Mixtral-8x22B-Instruct-v0.1", + "aliases": [ + "amdenterpriseai/aim-mistralai-mixtral-8x22b-instruct-v0-1" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 339, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-mistralai-mixtral-8x22b-instruct-v0-1:0.11.1", + "image_digest": "sha256:bcbe47ea00037f1d3b203a47c95bcfaa2dc76b5739a94bfdf0d681be1385c8a7", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 4, "gfx950-dcgpu": 4}, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "mistralai/Mixtral-8x7B-Instruct-v0.1", + "aliases": [ + "amdenterpriseai/aim-mistralai-mixtral-8x7b-instruct-v0-1" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 113, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-mistralai-mixtral-8x7b-instruct-v0-1:0.11.1", + "image_digest": "sha256:9f2abb10dc3a21169d7cd0e6f96294df3683dd42ea49f1d5230e62722f4ae738", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "openai/gpt-oss-120b", + "aliases": [ + "amdenterpriseai/aim-openai-gpt-oss-120b" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-openai-gpt-oss-120b:0.11.1", + "image_digest": "sha256:a3052d6230d7a414837903c31be0122f6615819448598e9caea973c084dc3ea7", + "supported_gfx_families": ["gfx90a", "gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx90a": 2}, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "openai/gpt-oss-20b", + "aliases": [ + "amdenterpriseai/aim-openai-gpt-oss-20b" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-openai-gpt-oss-20b:0.11.1", + "image_digest": "sha256:67cee29f3f9036eb3918190c76bc4442dd48254657e680d0256a5018c2164274", + "supported_gfx_families": ["gfx90a", "gfx94X-dcgpu", "gfx950-dcgpu"], + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "Qwen/Qwen3-235B-A22B", + "aliases": [ + "amdenterpriseai/aim-qwen-qwen3-235b-a22b" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 282, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-qwen-qwen3-235b-a22b:0.11.1", + "image_digest": "sha256:427d25b93cce97b499f9c1e9268b33eea25a8da8a0f7ef7904f4cdeef5f52f5a", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 2}, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "Qwen/Qwen3-32B", + "aliases": [ + "amdenterpriseai/aim-qwen-qwen3-32b" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 40, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-qwen-qwen3-32b:0.13.0", + "image_digest": "sha256:3e8d47db7c88ca1b7bb210b7bfd5a66f0439e3b768f200ab4aa51fb058d2871c", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "Qwen/Qwen3-Coder-Next", + "aliases": [ + "amdenterpriseai/aim-qwen-qwen3-coder-next" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 96, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-qwen-qwen3-coder-next:0.11.1", + "image_digest": "sha256:0874c97e547a59df1893813e3f011c7e5bc1d035b68e99361f4175c0e15010c0", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "Qwen/Qwen3-VL-235B-A22B-Instruct", + "aliases": [ + "amdenterpriseai/aim-qwen-qwen3-vl-235b-a22b-instruct" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 282, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-qwen-qwen3-vl-235b-a22b-instruct:0.11.1", + "image_digest": "sha256:4353ab9fb67d30b54d4c724429c0401490b42aa96045a050faea7b46a9e2d3d6", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 2}, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "Qwen/Qwen3-VL-235B-A22B-Thinking", + "aliases": [ + "amdenterpriseai/aim-qwen-qwen3-vl-235b-a22b-thinking" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 282, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-qwen-qwen3-vl-235b-a22b-thinking:0.11.1", + "image_digest": "sha256:c10ee1a4189708e101802b4b0b3d292dfc4802ad08c6da70a0f56ae2094eeccb", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 2}, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "google/gemma-3n-E4B-it", + "aliases": [ + "amdenterpriseai/aim-radeon-google-gemma-3n-e4b-it" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 20, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", + "requires a reachable Docker daemon; the image is pulled on first serve", + "Radeon-targeted image; the AIM profile is pinned per GPU family from profile_ids, override with --engine-env AIM_PROFILE_ID=" + ], + "model_id_override": "amdenterpriseai/aim-radeon-google-gemma-3n-e4b-it:0.12.0-preview", + "image_digest": "sha256:f5dd9116c6714a8e51e6bdbf94f5a2a8b06db62fc95994e6b331f92931416697", + "supported_gfx_families": ["gfx110X-all", "gfx120X-all"], + "supported_boards": ["W7900", "R9700"], + "requires_hf_token": true, + "profile_ids": {"gfx110X-all": "vllm-w7900-bf16-tp1-latency", "gfx120X-all": "vllm-r9700-bf16-tp1-latency"}, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "Qwen/Qwen3.6-35B-A3B", + "aliases": [ + "amdenterpriseai/aim-radeon-qwen-qwen3-6-35b-a3b" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 84, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "the pinned profiles use tensor parallelism and need 4 GPUs on gfx110X-all / 4 GPUs on gfx120X-all", + "requires a reachable Docker daemon; the image is pulled on first serve", + "Radeon-targeted image; the AIM profile is pinned per GPU family from profile_ids, override with --engine-env AIM_PROFILE_ID=" + ], + "model_id_override": "amdenterpriseai/aim-radeon-qwen-qwen3-6-35b-a3b:2026.11.0", + "image_digest": "sha256:4df926af5c9e792675577cf881fb12b4964b6dac18f6d3f7c1e2cf460834eee2", + "supported_gfx_families": ["gfx110X-all", "gfx120X-all"], + "min_gpu_counts": {"gfx110X-all": 4, "gfx120X-all": 4}, + "supported_boards": ["W7900", "R9700"], + "profile_ids": {"gfx110X-all": "vllm-w7900-bf16-tp4-latency", "gfx120X-all": "vllm-r9700-bf16-tp4-latency"}, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "Qwen/Qwen3-VL-8B-Instruct", + "aliases": [ + "amdenterpriseai/aim-radeon-qwen-qwen3-vl-8b-instruct" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 22, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Radeon-targeted image; the AIM profile is pinned per GPU family from profile_ids, override with --engine-env AIM_PROFILE_ID=" + ], + "model_id_override": "amdenterpriseai/aim-radeon-qwen-qwen3-vl-8b-instruct:2026.11.0", + "image_digest": "sha256:89775113626743d77023e15b595d4218c1e46e89e932b3f3ee5c0350914add55", + "supported_gfx_families": ["gfx110X-all", "gfx120X-all"], + "supported_boards": ["W7900", "R9700"], + "profile_ids": {"gfx110X-all": "vllm-w7900-bf16-tp1-latency", "gfx120X-all": "vllm-r9700-bf16-tp1-latency"}, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "zai-org/GLM-4.7-Flash", + "aliases": [ + "amdenterpriseai/aim-radeon-zai-org-glm-4-7-flash" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "min_gpu_mem_gb": 72, + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "the pinned profiles use tensor parallelism and need 2 GPUs on gfx110X-all / 4 GPUs on gfx120X-all", + "requires a reachable Docker daemon; the image is pulled on first serve", + "Radeon-targeted image; the AIM profile is pinned per GPU family from profile_ids, override with --engine-env AIM_PROFILE_ID=" + ], + "model_id_override": "amdenterpriseai/aim-radeon-zai-org-glm-4-7-flash:0.12.0-preview", + "image_digest": "sha256:b94130be1f12eaab28bdce274a63b3c10a24b8e292dc91f4c563b7a42246c1ab", + "supported_gfx_families": ["gfx110X-all", "gfx120X-all"], + "min_gpu_counts": {"gfx110X-all": 2, "gfx120X-all": 4}, + "supported_boards": ["W7900", "R9700"], + "profile_ids": {"gfx110X-all": "vllm-w7900-bf16-tp2-latency", "gfx120X-all": "vllm-r9700-bf16-tp4-latency"}, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "zai-org/GLM-4.7", + "aliases": [ + "amdenterpriseai/aim-zai-org-glm-4-7" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-zai-org-glm-4-7:0.12.0", + "image_digest": "sha256:6555005fe59ad9334ce292ca3df4b619106656a731e7319114c2b3a43a1b1219", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 2}, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" + ], + "warnings": [ + "AIM (AMD Inference Microservices) image; requires Docker" + ] + }, + { + "canonical_model_id": "zai-org/GLM-5.2", + "aliases": [ + "amdenterpriseai/aim-zai-org-glm-5-2" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" + ], + "model_id_override": "amdenterpriseai/aim-zai-org-glm-5-2:2026.9.0", + "image_digest": "sha256:1a0ad5bda56eb0784085583068d56e1cfc03f4b28875fe968d9477b30b9f5f5b", + "supported_gfx_families": ["gfx94X-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 8}, + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim", + "vllm" ], "warnings": [ - "Radeon-targeted AIM image; requires Docker and an explicit AIM_PROFILE_ID" + "AIM (AMD Inference Microservices) image; requires Docker" ] } ] diff --git a/crates/rocm-core/src/model_catalog.rs b/crates/rocm-core/src/model_catalog.rs index 18c4126041..574be7bf4a 100644 --- a/crates/rocm-core/src/model_catalog.rs +++ b/crates/rocm-core/src/model_catalog.rs @@ -92,6 +92,99 @@ pub struct ModelRecipeEngineRecord { /// (currently `aim`); catalog validation rejects it elsewhere. #[serde(default)] pub image_digest: Option, + /// When a maintainer last checked this entry against the upstream image + /// registry (unix ms). `rocm model --verbose` warns once it is older than + /// [`MODEL_RECIPE_STALE_AFTER_MS`], since the pinned image may be gone. + #[serde(default)] + pub generated_at_unix_ms: Option, + /// TheRock gfx families (for example `gfx94X-dcgpu`) this engine recipe has a + /// profile for. Empty means unrestricted. Serving on a detected GPU outside + /// this list is refused, and it ties the recipe to the platforms covering + /// these families in `rocm model`. Only valid on `aim`. + #[serde(default)] + pub supported_gfx_families: Vec, + /// AIM profile id (`AIM_PROFILE_ID`) to pin per gfx family, for images whose + /// profile auto-selection is unreliable (Radeon). Keys must be listed in + /// `supported_gfx_families`. An explicit `--engine-env AIM_PROFILE_ID=...` + /// wins over this. Only valid on `aim`. + #[serde(default)] + pub profile_ids: BTreeMap, + /// The image needs a Hugging Face token (a gated model). `rocm serve` refuses + /// to start it until `HF_TOKEN` is passed with `--engine-env`. Only valid on + /// `aim`. + #[serde(default)] + pub requires_hf_token: bool, + /// Board names (for example `W7900`) the image's pinned profiles were built + /// for. `rocm serve` warns when the detected GPU name matches none of them, + /// since a gfx family can cover boards the profile does not suit. Only valid + /// on `aim`, and only checked when `profile_ids` pins a profile. + #[serde(default)] + pub supported_boards: Vec, + /// Minimum number of GPUs the image needs per gfx family (the smallest + /// tensor-parallel size across that family's profiles). A family left out + /// needs one. Keys must be listed in `supported_gfx_families`. `rocm serve` + /// gives an engine one GPU, so it refuses to start an image listed here for + /// the detected family, however many GPUs the host has. Only valid on `aim`. + #[serde(default)] + pub min_gpu_counts: BTreeMap, +} + +/// Outcome of matching an engine recipe against the detected GPU family. +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum GfxSupport<'a> { + /// No restriction, or no GPU family known to check against. + Unrestricted, + /// Supported; carries the profile id pinned for this family, if any. + Supported(Option<&'a str>), + /// The recipe lists families and this one is not among them. + Unsupported, +} + +/// Checks `engine_recipe` against `gfx_family` (a normalized TheRock family). +pub fn engine_recipe_gfx_support<'a>( + engine_recipe: &'a ModelRecipeEngineRecord, + gfx_family: Option<&str>, +) -> GfxSupport<'a> { + if engine_recipe.supported_gfx_families.is_empty() { + return GfxSupport::Unrestricted; + } + let Some(family) = gfx_family else { + return GfxSupport::Unrestricted; + }; + if !engine_recipe + .supported_gfx_families + .iter() + .any(|f| f.eq_ignore_ascii_case(family)) + { + return GfxSupport::Unsupported; + } + GfxSupport::Supported( + engine_recipe + .profile_ids + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(family)) + .map(|(_, id)| id.as_str()), + ) +} + +/// How long an engine recipe's `generated_at_unix_ms` stays trusted (180 days). +pub const MODEL_RECIPE_STALE_AFTER_MS: u128 = 180 * 24 * 60 * 60 * 1000; + +/// Returns a warning when `engine_recipe` was last verified more than +/// [`MODEL_RECIPE_STALE_AFTER_MS`] before `now_ms`. Entries without a +/// timestamp are never flagged. +pub fn model_recipe_staleness_warning( + engine_recipe: &ModelRecipeEngineRecord, + now_ms: u128, +) -> Option { + let age_ms = now_ms.checked_sub(engine_recipe.generated_at_unix_ms?)?; + (age_ms > MODEL_RECIPE_STALE_AFTER_MS).then(|| { + format!( + "{} entry was last verified {} days ago; the pinned image may no longer exist; entries are refreshed by hand, so a newer rocm-cli release or recipe index may carry a current one", + engine_recipe.engine, + age_ms / (24 * 60 * 60 * 1000) + ) + }) } #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] @@ -709,6 +802,52 @@ fn validate_model_recipe_engine_record( ); } } + let known = crate::known_therock_families(); + for family in &engine_recipe.supported_gfx_families { + if !known.contains(&family.as_str()) { + bail!( + "engine recipe for `{}` on `{canonical_model_id}` lists unknown gfx family `{family}`", + engine_recipe.engine + ); + } + } + for (family, profile) in &engine_recipe.profile_ids { + require_nonempty(profile, "engine profile id")?; + if !engine_recipe.supported_gfx_families.contains(family) { + bail!( + "engine recipe profile_ids for `{family}` on `{canonical_model_id}` must also \ + be listed in supported_gfx_families" + ); + } + if engine_recipe.extra_env.contains_key("AIM_PROFILE_ID") { + bail!( + "engine recipe on `{canonical_model_id}` sets both profile_ids and \ + extra_env AIM_PROFILE_ID" + ); + } + } + for (family, count) in &engine_recipe.min_gpu_counts { + if !engine_recipe.supported_gfx_families.contains(family) || *count < 2 { + bail!( + "engine recipe min_gpu_counts for `{family}` on `{canonical_model_id}` must be \ + a listed supported_gfx_families entry with a count of at least 2" + ); + } + } + if (!engine_recipe.supported_gfx_families.is_empty() + || !engine_recipe.min_gpu_counts.is_empty() + || !engine_recipe.profile_ids.is_empty() + || engine_recipe.requires_hf_token + || !engine_recipe.supported_boards.is_empty()) + && !engine_recipe.engine.eq_ignore_ascii_case("aim") + { + bail!( + "engine recipe for `{}` on `{canonical_model_id}` sets supported_gfx_families, \ + profile_ids, min_gpu_counts, requires_hf_token or supported_boards, which only \ + apply to aim", + engine_recipe.engine + ); + } if engine_recipe.required_flags.is_empty() && engine_recipe.parser_settings.is_empty() && engine_recipe.preferred_endpoint.is_none() @@ -717,6 +856,12 @@ fn validate_model_recipe_engine_record( && engine_recipe.model_id_override.is_none() && engine_recipe.image_digest.is_none() && engine_recipe.extra_env.is_empty() + && engine_recipe.supported_gfx_families.is_empty() + && engine_recipe.generated_at_unix_ms.is_none() + && engine_recipe.profile_ids.is_empty() + && !engine_recipe.requires_hf_token + && engine_recipe.supported_boards.is_empty() + && engine_recipe.min_gpu_counts.is_empty() { bail!( "engine recipe for `{}` on `{canonical_model_id}` must not be empty", @@ -764,15 +909,47 @@ pub fn model_recipe_target_platform_label( recipe: &ModelRecipeRecord, platforms: &[ModelCatalogPlatform], ) -> String { - let engine = recipe - .preferred_engines - .first() - .map(|e| e.trim().to_ascii_lowercase()) - .unwrap_or_default(); platforms .iter() - .find(|p| p.engines.iter().any(|e| e.eq_ignore_ascii_case(&engine))) - .map_or_else(|| engine.clone(), |p| p.label.clone()) + .find(|p| model_recipe_targets_platform(recipe, p)) + .map_or_else( + || { + recipe + .preferred_engines + .first() + .map(|e| e.trim().to_ascii_lowercase()) + .unwrap_or_default() + }, + |p| p.label.clone(), + ) +} + +/// Whether `recipe` belongs under `platform`: its first preferred engine runs +/// there and, when that engine's recipe restricts gfx families, the platform +/// covers at least one of them. +pub fn model_recipe_targets_platform( + recipe: &ModelRecipeRecord, + platform: &ModelCatalogPlatform, +) -> bool { + let Some(engine) = recipe.preferred_engines.first() else { + return false; + }; + if !platform + .engines + .iter() + .any(|e| e.eq_ignore_ascii_case(engine)) + { + return false; + } + let families = recipe + .engine_recipes + .iter() + .find(|r| r.engine.eq_ignore_ascii_case(engine)) + .map_or(&[][..], |r| r.supported_gfx_families.as_slice()); + families.is_empty() + || families + .iter() + .any(|family| platform_matches_gfx_family(platform, family)) } /// Whether the given normalized TheRock family matches a platform's gfx targets. @@ -878,18 +1055,18 @@ mod tests { let strix = resolve_builtin_model_recipe("qwen").expect("qwen assistant"); assert_eq!( model_recipe_target_platform_label(&strix, &platforms), - "AMD Ryzen AI - Strix Halo (Lemonade / llama.cpp / AIM)" + "AMD Ryzen AI - Strix Halo (Lemonade / llama.cpp)" ); let mi300x = resolve_builtin_model_recipe("qwen3.6-27b").expect("qwen3.6-27b"); assert_eq!( model_recipe_target_platform_label(&mi300x, &platforms), - "AMD Instinct - MI300X, MI350X, MI355X (vLLM)" + "AMD Instinct - MI300X, MI350X, MI355X (vLLM / AIM)" ); // vLLM recipes land on the Instinct platform. let llama = resolve_builtin_model_recipe("llama-3.2-3b-instruct").expect("llama"); assert_eq!( model_recipe_target_platform_label(&llama, &platforms), - "AMD Instinct - MI300X, MI350X, MI355X (vLLM)" + "AMD Instinct - MI300X, MI350X, MI355X (vLLM / AIM)" ); } @@ -950,6 +1127,340 @@ mod tests { assert_eq!(tiny.min_gpu_mem_gb, Some(2)); } + #[test] + fn staleness_warning_fires_only_past_the_threshold() { + let mut aim = ModelRecipeEngineRecord { + engine: "aim".to_owned(), + ..Default::default() + }; + let now = 10 * MODEL_RECIPE_STALE_AFTER_MS; + assert_eq!(model_recipe_staleness_warning(&aim, now), None); + aim.generated_at_unix_ms = Some(now - MODEL_RECIPE_STALE_AFTER_MS); + assert_eq!(model_recipe_staleness_warning(&aim, now), None); + aim.generated_at_unix_ms = Some(now - MODEL_RECIPE_STALE_AFTER_MS - 1); + assert!( + model_recipe_staleness_warning(&aim, now) + .expect("stale entry warns") + .contains("180 days ago") + ); + // A timestamp in the future (clock skew) is not stale. + aim.generated_at_unix_ms = Some(now + 1); + assert_eq!(model_recipe_staleness_warning(&aim, now), None); + } + + #[test] + fn gated_aim_models_say_to_pass_hf_token() { + // Gated per the AIM catalog's model pages ("requires a Hugging Face + // authentication"); `--engine-env HF_TOKEN` forwarding is covered in + // engines/aim (run_args_never_puts_env_values_on_the_command_line). + for id in [ + "meta-llama/Llama-3.2-3B-Instruct", + "meta-llama/Llama-3.3-70B-Instruct", + "google/gemma-3-27b-it", + ] { + let recipe = resolve_builtin_model_recipe(id).expect("gated aim recipe"); + let aim = recipe + .engine_recipes + .iter() + .find(|e| e.engine == "aim") + .expect("aim engine"); + assert!(aim.requires_hf_token, "{id}"); + assert!( + aim.notes + .iter() + .any(|n| n.contains("--engine-env HF_TOKEN")), + "{id}" + ); + } + // The flag and the note never drift apart across the whole catalog. + for recipe in builtin_model_recipes() { + for aim in recipe.engine_recipes.iter().filter(|e| e.engine == "aim") { + assert_eq!( + aim.requires_hf_token, + aim.notes + .iter() + .any(|n| n.contains("--engine-env HF_TOKEN")), + "{}", + recipe.canonical_model_id + ); + } + } + let open = resolve_builtin_model_recipe("Qwen/Qwen3-32B").expect("ungated aim recipe"); + assert!( + !open.engine_recipes[0] + .notes + .iter() + .any(|n| n.contains("HF_TOKEN")) + ); + } + + #[test] + fn aim_recipes_are_matched_to_the_platforms_their_profiles_support() { + let registry = builtin_model_recipe_registry(); + let platforms = model_catalog_platforms(®istry); + let label = |id: &str| { + model_recipe_target_platform_label( + &resolve_builtin_model_recipe(id).expect("recipe"), + &platforms, + ) + }; + assert!(label("Qwen/Qwen3.5-9B").starts_with("AMD Radeon")); + assert!(label("Qwen/Qwen3-32B").starts_with("AMD Instinct")); + let strix = platforms + .iter() + .find(|p| p.label.contains("Strix Halo")) + .expect("strix halo platform"); + assert!(!strix.engines.iter().any(|e| e == "aim")); + for recipe in ®istry.recipes { + for aim in recipe.engine_recipes.iter().filter(|e| e.engine == "aim") { + assert!( + !aim.supported_gfx_families.is_empty(), + "{} aim entry must say which GPUs its profiles cover", + recipe.canonical_model_id + ); + } + } + } + + #[test] + fn gfx_support_pins_profiles_and_rejects_other_families() { + let aim = ModelRecipeEngineRecord { + engine: "aim".to_owned(), + supported_gfx_families: vec!["gfx110X-all".to_owned()], + profile_ids: BTreeMap::from([("gfx110X-all".to_owned(), "p".to_owned())]), + ..Default::default() + }; + assert_eq!( + engine_recipe_gfx_support(&aim, Some("gfx110X-all")), + GfxSupport::Supported(Some("p")) + ); + assert_eq!( + engine_recipe_gfx_support(&aim, Some("gfx1151")), + GfxSupport::Unsupported + ); + assert_eq!( + engine_recipe_gfx_support(&aim, None), + GfxSupport::Unrestricted + ); + let mut bad = aim.clone(); + bad.profile_ids + .insert("gfx120X-all".to_owned(), "q".to_owned()); + assert!(validate_model_recipe_engine_record(&bad, "m").is_err()); + let mut bad_count = bad.clone(); + bad_count.profile_ids.clear(); + bad_count.min_gpu_counts.insert("gfx110X-all".to_owned(), 1); + assert!(validate_model_recipe_engine_record(&bad_count, "m").is_err()); + let mut wrong_engine = aim; + wrong_engine.engine = "vllm".to_owned(); + assert!(validate_model_recipe_engine_record(&wrong_engine, "m").is_err()); + } + + /// One case per branch of `validate_model_recipe_engine_record` added for the + /// AIM host-fit fields; each is the only thing wrong (or, for the `ok` rows, + /// the only content) in its record, so removing a branch turns its row red. + #[test] + fn engine_record_validation_covers_each_aim_field_branch() { + let aim = |edit: fn(&mut ModelRecipeEngineRecord)| { + let mut record = ModelRecipeEngineRecord { + engine: "aim".to_owned(), + ..Default::default() + }; + edit(&mut record); + record + }; + let cases: Vec<(&str, ModelRecipeEngineRecord, bool)> = vec![ + ( + "unknown gfx family", + aim(|r| r.supported_gfx_families = vec!["gfx0000".to_owned()]), + false, + ), + ( + "profile pin and AIM_PROFILE_ID env", + aim(|r| { + r.supported_gfx_families = vec!["gfx110X-all".to_owned()]; + r.profile_ids + .insert("gfx110X-all".to_owned(), "p".to_owned()); + r.extra_env + .insert("AIM_PROFILE_ID".to_owned(), "q".to_owned()); + }), + false, + ), + ( + "min_gpu_counts family not in supported_gfx_families", + aim(|r| { + r.supported_gfx_families = vec!["gfx110X-all".to_owned()]; + r.min_gpu_counts.insert("gfx120X-all".to_owned(), 2); + }), + false, + ), + ( + "requires_hf_token on a non-aim engine", + ModelRecipeEngineRecord { + engine: "vllm".to_owned(), + requires_hf_token: true, + ..Default::default() + }, + false, + ), + ( + "supported_boards on a non-aim engine", + ModelRecipeEngineRecord { + engine: "vllm".to_owned(), + supported_boards: vec!["W7900".to_owned()], + ..Default::default() + }, + false, + ), + ("empty aim record", aim(|_| {}), false), + ( + "aim record carrying only supported_gfx_families", + aim(|r| r.supported_gfx_families = vec!["gfx110X-all".to_owned()]), + true, + ), + ( + "aim record carrying only requires_hf_token", + aim(|r| r.requires_hf_token = true), + true, + ), + ( + "aim record carrying only supported_boards", + aim(|r| r.supported_boards = vec!["W7900".to_owned()]), + true, + ), + ( + "aim record carrying only generated_at_unix_ms", + aim(|r| r.generated_at_unix_ms = Some(1)), + true, + ), + ]; + for (name, record, valid) in cases { + assert_eq!( + validate_model_recipe_engine_record(&record, "m").is_ok(), + valid, + "{name}" + ); + } + } + + #[test] + fn every_builtin_aim_recipe_carries_a_verification_timestamp() { + let mut checked = 0; + for recipe in builtin_model_recipes() { + for aim in recipe.engine_recipes.iter().filter(|e| e.engine == "aim") { + checked += 1; + assert!( + aim.generated_at_unix_ms.is_some(), + "{} aim entry has no generated_at_unix_ms, so it can never be flagged stale", + recipe.canonical_model_id + ); + } + } + assert!(checked > 0, "the built-in catalog lists no aim entries"); + } + + #[test] + fn engine_recipe_gfx_support_matches_families_without_regard_to_case() { + let unrestricted = ModelRecipeEngineRecord { + engine: "aim".to_owned(), + ..Default::default() + }; + // No families listed: any host, detected or not, is unrestricted. + for family in [None, Some("gfx1151"), Some("gfx94X-dcgpu")] { + assert_eq!( + engine_recipe_gfx_support(&unrestricted, family), + GfxSupport::Unrestricted, + "{family:?}" + ); + } + let aim = ModelRecipeEngineRecord { + engine: "aim".to_owned(), + supported_gfx_families: vec!["gfx94X-dcgpu".to_owned(), "gfx110X-all".to_owned()], + profile_ids: BTreeMap::from([("gfx110X-all".to_owned(), "p".to_owned())]), + ..Default::default() + }; + // Matching ignores case on both the family list and the pinned profile key. + assert_eq!( + engine_recipe_gfx_support(&aim, Some("GFX110X-ALL")), + GfxSupport::Supported(Some("p")) + ); + assert_eq!( + engine_recipe_gfx_support(&aim, Some("gfx94x-DCGPU")), + GfxSupport::Supported(None) + ); + assert_eq!( + engine_recipe_gfx_support(&aim, Some("gfx120X-all")), + GfxSupport::Unsupported + ); + assert_eq!( + engine_recipe_gfx_support(&aim, None), + GfxSupport::Unrestricted + ); + } + + #[test] + fn model_recipe_targets_platform_needs_the_engine_and_a_covered_family() { + let platform = |engines: &[&str], families: &[&str]| ModelCatalogPlatform { + label: "p".to_owned(), + engines: engines.iter().map(|e| (*e).to_owned()).collect(), + gfx_families: families.iter().map(|f| (*f).to_owned()).collect(), + }; + let recipe = |preferred: &[&str], families: &[&str]| { + let mut recipe = resolve_builtin_model_recipe("qwen").expect("recipe"); + recipe.preferred_engines = preferred.iter().map(|e| (*e).to_owned()).collect(); + recipe.engine_recipes = vec![ModelRecipeEngineRecord { + engine: "aim".to_owned(), + supported_gfx_families: families.iter().map(|f| (*f).to_owned()).collect(), + ..Default::default() + }]; + recipe + }; + let two_families = recipe(&["aim"], &["gfx94X-dcgpu", "gfx110X-all"]); + for (recipe, platform, expected, why) in [ + ( + &two_families, + platform(&["aim"], &["gfx110X-all"]), + true, + "covering just one of the recipe's families is enough", + ), + ( + &two_families, + platform(&["aim"], &["gfx1151"]), + false, + "no family in common", + ), + ( + &two_families, + platform(&["vllm"], &["gfx94X-dcgpu"]), + false, + "the engine does not run on the platform", + ), + ( + &two_families, + platform(&["AIM"], &["GFX94X-DCGPU"]), + true, + "engine and family names compare without regard to case", + ), + ( + &recipe(&["aim"], &[]), + platform(&["aim"], &["gfx1151"]), + true, + "an unrestricted recipe fits any platform running the engine", + ), + ( + &recipe(&[], &["gfx94X-dcgpu"]), + platform(&["aim"], &["gfx94X-dcgpu"]), + false, + "no preferred engine, no platform", + ), + ] { + assert_eq!( + model_recipe_targets_platform(recipe, &platform), + expected, + "{why}" + ); + } + } + #[test] fn builtin_recipe_records_remote_code_policy() { let glm = resolve_builtin_model_recipe("glm-5").expect("glm alias should resolve"); @@ -976,6 +1487,12 @@ mod tests { notes: Vec::new(), model_id_override: None, image_digest: Some(digest.to_owned()), + generated_at_unix_ms: None, + supported_gfx_families: Vec::new(), + profile_ids: BTreeMap::new(), + requires_hf_token: false, + supported_boards: Vec::new(), + min_gpu_counts: BTreeMap::new(), }); ModelRecipeIndexDocument { schema_version: 1, @@ -1029,6 +1546,12 @@ mod tests { notes: vec!["metadata only; adapter protocol does not consume this yet".to_owned()], model_id_override: None, image_digest: None, + generated_at_unix_ms: None, + supported_gfx_families: Vec::new(), + profile_ids: BTreeMap::new(), + requires_hf_token: false, + supported_boards: Vec::new(), + min_gpu_counts: BTreeMap::new(), }); let index = ModelRecipeIndexDocument { schema_version: 1, diff --git a/docs/aim-catalog.md b/docs/aim-catalog.md new file mode 100644 index 0000000000..d693deb713 --- /dev/null +++ b/docs/aim-catalog.md @@ -0,0 +1,92 @@ + + +# AIM catalog: freshness and updates + +AIM entries live in [model_catalog.json](../crates/rocm-core/src/model_catalog.json) +alongside every other model recipe (`"loader": "aim"`). They are hand-maintained +and ship on the normal CLI release cadence, through the same PR review as any +other recipe. There is no network fetch of the catalog: a hosted, signed +recipe index is blocked on production signing keys (see +[release-trust.md](release-trust.md)) and is out of scope here. + +## Freshness + +Each AIM `engine_recipes` entry pins an image by `image_digest` and records +`generated_at_unix_ms`, the time a maintainer last confirmed the image and +digest still exist in the registry. `rocm model --verbose` prints a warning for any +engine recipe whose timestamp is older than 180 days +(`MODEL_RECIPE_STALE_AFTER_MS` in +[model_catalog.rs](../crates/rocm-core/src/model_catalog.rs)), because the +pinned image may have been withdrawn. Only `rocm model --verbose` shows the +warning; `rocm serve` does not check it. Every built-in AIM entry must carry a +timestamp (a unit test enforces it); an entry without one, such as one from a +configured recipe index, is never flagged. + +## What `rocm serve` checks, and its limit + +For an AIM catalog entry `rocm serve` checks, before pulling anything, that a +gated image has `HF_TOKEN`, that the detected GPU family has a profile, and that +the image does not need several GPUs. The family comes from one host-level +reading (the lowest KFD node), not from the GPU `--gpu` selects or the +visibility mask, so on a host with mixed GPUs, such as an APU plus a discrete +card, `--gpu 1` is judged by the host's first GPU. The board-name note has the +same limit. + +## Refreshing an entry + +Whoever owns the AIM images does this; it does not need a rocm-cli maintainer. + +1. Watch for new AIM releases. A version bump is the practical signal that AIM + images were added or changed. +2. For each AIM entry, find the newest stable tag + (`oras repo tags docker.io/amdenterpriseai/`) and copy its digest + (`oras resolve docker.io/amdenterpriseai/:`) into `image_digest` (and `model_id_override` if the tag moved). + Check each `profile_ids` entry is still a valid profile. +3. Set `generated_at_unix_ms` to now (`date +%s%3N`). +4. Add entries for new AIM images the same way as the existing ones. The list + of images is the AMD Inference Microservices (AIM) catalog + (). + CPU-only (`aim-epyc-*`) images are left out, since rocm-cli never falls back + to CPU, as are non-chat images (speech, video, bio) until the serve path + supports their tasks. +5. Check each image's model page in the AIM catalog for the line "requires a + Hugging Face authentication". For those, set `requires_hf_token: true` and + add the `HF_TOKEN` note to the entry's `notes` (the pattern used by the + existing gated entries). `rocm serve` then refuses to start the image until + `HF_TOKEN` is passed with `--engine-env`. +6. Record which GPUs the image has profiles for. From the model page's profile + table, set `supported_gfx_families` on the entry (MI250X `gfx90a`; MI300X and + MI325X `gfx94X-dcgpu`; MI350X and MI355X `gfx950-dcgpu`; W7900 `gfx110X-all`; + R9700 `gfx120X-all`). `rocm serve` refuses a GPU outside that list, and + `rocm model` groups the recipe under the platform covering those families. + Instinct images auto-select a profile, so leave `profile_ids` unset. Radeon + images do not select reliably: add `profile_ids` mapping each family to a + profile (the lowest-GPU-count optimized latency profile, noting in `notes` if + it needs more than one GPU). An explicit `--engine-env AIM_PROFILE_ID=...` + still wins. +7. On Radeon entries, set `supported_boards` to the board names in the profile + table (for example `W7900`, `R9700`). `rocm serve` warns, without refusing, + when the detected GPU name matches none of them, because a gfx family can + cover boards the pinned profile was not built for. +8. Set `min_gpu_mem_gb`. The AIM docs publish GPU counts but no VRAM, so the + values in the catalog are estimates: total parameters times 1 byte (2 when + the image has no fp8 profile) plus 20% headroom, rounded up. Leave it unset + for other dtypes (for example MXFP4), including when only some supported + families have an MXFP4 single-GPU profile (Llama-3.1-405B-Instruct and GLM-4.7 + on `gfx950-dcgpu`), and cap it at the entry's + `min_gpu_counts` times the smallest supported GPU's memory so it never + contradicts the GPU count. Entries whose + parameter count is not published (DeepSeek-V4-Flash, MiniMax-M3, GLM-5.2) are + left unset rather than guessed. Replace an estimate with a measured figure + when one is available. +9. Set `min_gpu_counts` for any family whose smallest profile uses more than one + GPU (the smallest tensor-parallel size in that family's profiles; leave the + family out when a single-GPU profile exists). `rocm serve` refuses such an + image up front, because `rocm serve` does not yet select several GPUs; that + is tracked as ROCMAI-751. +10. Open a PR; `cargo test -p rocm-core model_catalog` validates the catalog, + including `builtin_model_catalog_json_parses_and_validates`. diff --git a/docs/architecture.md b/docs/architecture.md index 73fc9cf3b9..07b76456c6 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -55,7 +55,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) — full domain extraction, crate-level (split out of `therock.rs`): the installed-runtime manifest model (`InstalledRuntimeManifest`, `WheelRuntimeComposition`, `RocmSdkPythonProbe`, `RocmSdkPackageProbe`, `RocmSdkLibraryProbe`, whose serde shape is the on-disk contract of the registry files and each install's `.rocm-cli-runtime.json`), where those files live (`runtime_registry_dir`, `runtime_manifest_path`), writing and loading them (`save_runtime_manifest`, `load_runtime_manifests`, `load_runtime_manifests_reporting_unparsed`, `sort_manifests_newest_install_first`), the composition readers (`wheel_composition_includes_devel`, `wheel_composition_device_target`), the probe validator (`validate_rocm_sdk_runtime_probe`), and registry selection and validation (`current_runtime_manifest`, `default_runtime_id_matches`, `runtime_keys_text`, `select_runtime_manifest`, `runtime_usability_status`, `validate_runtime_manifest_for_activation`, and the wheel/tarball validators behind it). Lives at the crate level and depends on neither `therock.rs` nor `runtimes_cmd.rs`, so both build on it. [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) still depends on `therock.rs` (it reaches the manifest types and loaders through `therock.rs`'s re-exports), but `therock.rs` no longer depends on `runtimes_cmd.rs` in production code (one of its tests still calls into it). `runtimes_cmd.rs` re-exports the selection and path-helper names that other modules import from it. Its own tests cover the manifest round-trip, the `devel`/composition readers, and runtime selector resolution. - [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) — full domain extraction; reads the live local servers at activation time, classifies each against the runtime being activated, renders the report, and moves stale ones with `--restart-services`. `fn runtimes()` lives in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) and `RuntimesCommand` in [`main.rs`](../apps/rocm/src/main.rs); this file owns `ServiceRuntimeState`/`RuntimeServiceEntry`/`FailedServiceRestart`/`UnconfirmedServiceRestart`/`ServiceRestartOutcome`/`RuntimeServiceReconciliation`/`ServiceRuntimePin`, reached from `runtimes_cmd.rs` via `crate::runtime_services`. - [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) — mechanical relocation, owns types (Phase 6c, ROCMAI-89): `runtimes()` and its activate/rollback/uninstall/import/adopt handlers, plus the runtime list rendering; registry selection and manifest validation moved to [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), which this module re-exports for the callers that import them from here. `RuntimesCommand` stayed in [`main.rs`](../apps/rocm/src/main.rs) (part of the `Command`/`Cli` clap definition, the same way `EnginesCommand` did in Phase 6a). `RuntimeActivationResult` and `ActiveRuntimeMarker` are `pub(crate)`, read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs)'s `apply_runtime_update`/`append_update_activate_summary` and from [`storage.rs`](../apps/rocm/src/storage.rs) respectively; `normalize_path_for_compare`/`recover_setup_runtime_registration` are also `pub(crate)`, read from other modules such as [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) and [`comfyui.rs`](../apps/rocm/src/comfyui.rs). The dependency on the sibling [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) above runs the other way: that module calls into this one (`activate_runtime`, `newest_installed_runtime_manifest`, `recover_setup_runtime_registration`), while this module reaches it only from its test module. -- [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) — mechanical relocation, owns types (Phase 6a): `serve()`, plus engine-recipe overrides. +- [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) — mechanical relocation, owns types (Phase 6a): `serve()`, plus engine-recipe overrides and the AIM pre-flight that runs inside it: the Hugging Face token check, the GPU-family profile check and pin, the multi-GPU refusal (`aim_gpu_preflight`), and AIM-aware default engine selection (`recipe_preferred_engine`). - [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs) — full domain extraction. - [`service_hints.rs`](../apps/rocm/src/service_hints.rs) — shared CLI-output component: the `rocm services stop|restart|remove --yes` hint text, shared by [`main.rs`](../apps/rocm/src/main.rs), [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs), [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs), [`service_launch.rs`](../apps/rocm/src/service_launch.rs), and [`services_text.rs`](../apps/rocm/src/services_text.rs) so they cannot go out of sync with each other, as `serve_summary.rs`'s stop row did from the day it was added — #417. - [`service_launch.rs`](../apps/rocm/src/service_launch.rs) — mechanical relocation, owns types (Phase 6b, ROCMAI-91): spawning/attaching to a managed service process and streaming its log, plus the endpoint-key/auth checks that gate a public bind — `AttachOutcome`/`RawModeGuard`/`ManagedSpawn`/`AutostartClaim` are its private types; `ManagedLaunchReport` is `pub(crate)`, since [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) reads its fields directly for the interactive summary. Most of this module's functions are `pub(crate)` for the same reason — other modules, including [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`automations.rs`](../apps/rocm/src/automations.rs), and [`services_text.rs`](../apps/rocm/src/services_text.rs), call into it; `main.rs` does not. `ServicesCommand`/`services()` stayed in [`main.rs`](../apps/rocm/src/main.rs) rather than moving alongside — the service-record-management helpers `services()` dispatches to (stop/restart/remove/prune) are entangled with each other; only stop/restart are also reachable from the chat/LLM sandbox tool-calling path, out of scope for Phase 6b. diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 48b3c2d948..aedfea80ed 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -938,7 +938,13 @@ Expected result: ## 10. AIM (Docker) Engine Verification Linux/WSL2 only, and requires a reachable Docker daemon (`docker info` must -succeed). +succeed). The catalog entry used below has profiles for Radeon PRO W7900 and +Radeon AI PRO R9700 (`gfx110X-all`, `gfx120X-all`) only; on a GPU of another +family, such as Strix Halo or an Instinct card, `rocm serve` refuses it, so use +the raw image form further down. Other boards of a supported family (for +example an RX 7900 XTX) are not refused: the CLI prints a note that the pinned +profile targets the W7900 / R9700 and names `--engine-env AIM_PROFILE_ID` to +override it. ```bash rocm serve aim-qwen3.5-9b --engine aim --managed @@ -954,9 +960,15 @@ the engine's default 5-minute readiness wait. Raise it with environment of `rocm serve`. Stopping or failing a start removes the container and its downloaded weights, so a retry downloads them again. -`aim-qwen3.5-9b` is the catalog entry (`rocm model --verbose`). It supplies the -pinned image and `AIM_PROFILE_ID`; to serve a raw image reference instead, pass -the profile yourself: +`aim-qwen3.5-9b` (canonical id `Qwen/Qwen3.5-9B`) is the catalog entry +(`rocm model --verbose`). It supplies the pinned image and, for the detected GPU +family, `AIM_PROFILE_ID`; on a GPU the image has no profile for (for example +Strix Halo or an Instinct card) `rocm serve` refuses with the supported +families. An image whose smallest profile needs several GPUs on the detected +family is refused, because `rocm serve` gives an engine one GPU. None of these +GPU checks run when the host's GPU family cannot be detected or with +`--device cpu_only`. To serve a raw image +reference instead, which skips these catalog checks, pass the profile yourself: ```bash rocm serve amdenterpriseai/aim-radeon-qwen-qwen3-5-9b:0.12.0-preview --engine aim \ @@ -966,7 +978,9 @@ rocm serve amdenterpriseai/aim-radeon-qwen-qwen3-5-9b:0.12.0-preview --engine ai `--engine-env KEY=VALUE` is stored with the service record, so use it for non-secret values only. For a secret such as a gated model's `HF_TOKEN`, export it and pass the bare name (`export HF_TOKEN=...`, then `--engine-env HF_TOKEN`): -it is forwarded from your environment and never stored. `--engine-env` is +it is forwarded from your environment and never stored. A gated model +(`rocm model --verbose` shows the `HF_TOKEN` note) is refused before +anything is pulled if no token was passed. `--engine-env` is ignored by the other engines (the command prints a note saying so). Expected result: diff --git a/docs/testing.md b/docs/testing.md index 7dd8a30c8a..586834d822 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -845,6 +845,7 @@ cargo test -p rocm --bin rocm render_model_registry_text_reports_host_ram_fit cargo test -p rocm --bin rocm model_recipe cargo test -p rocm --bin rocm logs cargo test -p rocm-core model_recipe +cargo test -p rocm-core staleness_warning cargo test -p rocm-core load_model_recipe_index cargo test -p rocm --bin rocm update_report_policy cargo test -p rocm-engine-lemonade stdio_protocol_routes_all_methods_without_side_effects diff --git a/tests/e2e-cucumber/features/model_catalog.feature b/tests/e2e-cucumber/features/model_catalog.feature index a7acdee31e..458dd80d7e 100644 --- a/tests/e2e-cucumber/features/model_catalog.feature +++ b/tests/e2e-cucumber/features/model_catalog.feature @@ -1,6 +1,6 @@ Feature: Model catalog listing - # ROCMAI-555: the AIM engine recipe (aim-qwen3.5-9b) is curated out of the + # ROCMAI-555: the AIM engine recipe (Qwen/Qwen3.5-9B) is curated out of the # default `rocm model` view (it is not `featured`), so only `--verbose` # actually lists it. `model` and its `models` alias share one parser branch, # so exercising `model --verbose` covers the epic's "rocm model or rocm @@ -13,8 +13,9 @@ Feature: Model catalog listing # The epic names two reasons a listed AIM image can be unusable: an # unsupported GPU family, or no Docker. Only the Docker check runs at the # catalog layer today (see `engine_ruled_out_by_platform` in main.rs); - # GPU-family fit is a `rocm serve`/`rocm diagnose --model` readiness check, - # not a catalog property. This plants a `docker` that always fails so the + # GPU-family fit is a property of the entry (`supported_gfx_families`, shown by + # `--verbose`) but is only enforced by `rocm serve`, not by the listing. + # This plants a `docker` that always fails so the # unavailable branch is exercised deterministically on every host, including # ones where Docker really is reachable (an earlier variant compared against # the host's real Docker state, which asserted a different outcome per host). @@ -25,3 +26,14 @@ Feature: Model catalog listing Given no Docker daemon is reachable When the user asks for the model catalog in verbose detail Then the catalog's AIM entry is flagged unavailable for lack of Docker + + # ROCMAI-556: an AIM entry pins an image that may since have been withdrawn, so + # `rocm model --verbose` warns once the entry was last verified more than 180 + # days ago. The entry is shipped with a verification timestamp; this plants a + # signed catalog whose timestamp is decades old, so the warning is exercised + # without depending on the date the suite runs. + @id:model-catalog-warns-on-stale-aim-entry + Scenario: model-catalog-03 - A stale AIM catalog entry is flagged in rocm model --verbose + Given a recipe catalog whose AIM entry was last verified long ago + When the user asks for the model catalog in verbose detail + Then the catalog warns that the AIM entry may no longer be current diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 13c6a353f1..2536900ffa 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -345,7 +345,9 @@ Feature: Model serving # AIM serve + inference canary (ROCMAI-554). This scenario deliberately still # serves via the raw AIM Docker image reference rather than the catalog # recipe added in ROCMAI-555 — it's a hardware-verified canary pinned to one - # exact image/profile, not a catalog-discovery test. `@requires-engine:aim` + # exact image/profile, not a catalog-discovery test. Because it names the raw + # image, the catalog's GPU-family refusal (no profile for Strix Halo) does not + # apply to it. `@requires-engine:aim` # skips here when Docker is unreachable OR the gfx family isn't one aim is # verified on (see `HostCapability::engine_available`'s "aim" arm — Radeon # gfx110X/120X and Strix Halo gfx1151, not Instinct); a Strix Halo @@ -561,3 +563,59 @@ Feature: Model serving Then the install is refused And no vLLM install was started And the user is told vLLM needs WSL or Linux, or the Lemonade engine to install + + # ROCMAI-556: a gated AIM image is refused before anything is pulled when no + # Hugging Face token was passed. The check runs ahead of GPU detection and + # Docker, so it needs neither and gates every PR (ungated). The refusal names + # the remedy and leaves no service record behind; serve-37 proves the named + # remedy clears the condition. A Docker that logs its calls shows that nothing + # was pulled. serve-38 and serve-39 cover the GPU-dependent refusals. + @id:serve-gated-aim-refused-without-hf-token + Scenario: serve-36 - A gated AIM image is refused without a Hugging Face token + Given Docker is watched for any call + When the user serves a gated AIM model without passing a Hugging Face token + Then serving is refused before any engine starts + And the CLI explains that the model needs a Hugging Face token + And no service was started + And Docker was never called + + # The remedy printed by serve-36 (`export HF_TOKEN`, then `--engine-env + # HF_TOKEN`) must actually clear that refusal. The host's one GPU is one the + # image supports and every GPU is masked, so the serve passes the token and + # GPU-family gates and stops at the GPU pre-flight, still before anything is + # pulled. The GPU is planted, so the outcome does not depend on the runner. + # Linux only: the empty `HIP_VISIBLE_DEVICES` mask and the planted kernel + # topology are Linux mechanisms. + @id:serve-gated-aim-token-remedy-clears-refusal @requires-os:linux + Scenario: serve-37 - Passing the exported Hugging Face token clears the gated AIM refusal + Given Docker is watched for any call + And a host whose only GPU is an AMD Radeon PRO W7900 + When the user serves a gated AIM model with an exported Hugging Face token while every GPU is masked + Then serving is refused before any engine starts + And the refusal is that no usable GPU is visible + And Docker was never called + + # An AIM image only carries profiles for some GPUs. On a GPU it has none for, + # `rocm serve --engine aim` is refused up front and names what the image does + # support, rather than pulling a container that fails inside. The detected GPU + # is planted (an Instinct card against a Radeon-only image), so this needs no + # AMD hardware and runs on every Linux lane. + @id:serve-aim-refused-on-unsupported-gpu @requires-os:linux + Scenario: serve-38 - An AIM image is refused on a GPU it has no profile for + Given Docker is watched for any call + And a host whose only GPU is an AMD Instinct MI300X + When the user serves a Radeon-only AIM image + Then serving is refused before any engine starts + And the CLI says the image has no profile for this GPU and names the GPUs it supports + And Docker was never called + + # `rocm serve` gives an engine one GPU, so an AIM image whose smallest profile + # needs several is refused before launch instead of starting with too few. + @id:serve-aim-refused-when-it-needs-several-gpus @requires-os:linux + Scenario: serve-39 - An AIM image that needs several GPUs is refused + Given Docker is watched for any call + And a host whose only GPU is an AMD Instinct MI300X + When the user serves an AIM image that needs eight GPUs + Then serving is refused before any engine starts + And the CLI says the image needs eight GPUs and multi-GPU serving is not supported + And Docker was never called diff --git a/tests/e2e-cucumber/src/bin/fake-docker.rs b/tests/e2e-cucumber/src/bin/fake-docker.rs index d6e55cf181..a43f2f9eac 100644 --- a/tests/e2e-cucumber/src/bin/fake-docker.rs +++ b/tests/e2e-cucumber/src/bin/fake-docker.rs @@ -11,9 +11,24 @@ //! simulates "no reachable daemon" regardless of whether the host running the //! suite actually has Docker installed. //! +//! When `FAKE_DOCKER_LOG` names a file, every invocation's arguments are +//! appended to it first (one line per call), so a scenario can prove a refused +//! `rocm serve` never reached Docker, not even for `docker info`. +//! //! A Rust binary rather than a shell script so the scenarios run on Windows //! too. fn main() { + if let Some(log) = std::env::var_os("FAKE_DOCKER_LOG") { + use std::io::Write as _; + let args: Vec = std::env::args().skip(1).collect(); + if let Ok(mut file) = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(log) + { + let _ = writeln!(file, "{}", args.join(" ")); + } + } std::process::exit(1); } diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index abe4d87efe..71f877b571 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -175,7 +175,10 @@ impl HostCapability { // Strix Halo (gfx1151) — exactly the families // `family_prefers_vllm` excludes. Without this, a // Docker-equipped Instinct lane would run the Radeon aim - // image with a W7900 profile. + // image with a W7900 profile. Strix Halo is verified for the + // raw image reference serve-26 uses; the catalog entry + // itself is refused on gfx1151 (no profile for it), so + // this is not a statement about the catalog. && !family_prefers_vllm(self.gfx_target.as_deref()) } _ => self.available_engines.iter().any(|e| e == engine), diff --git a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs index aebf589ae1..a12eae28a4 100644 --- a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs @@ -2238,7 +2238,7 @@ async fn assert_reason_is_not_curated(world: &mut E2eWorld) { ); } -fn workspace_root() -> std::path::PathBuf { +pub(crate) fn workspace_root() -> std::path::PathBuf { std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) .parent() .and_then(std::path::Path::parent) @@ -2246,7 +2246,7 @@ fn workspace_root() -> std::path::PathBuf { .to_path_buf() } -fn xtask_command() -> std::process::Command { +pub(crate) fn xtask_command() -> std::process::Command { if let Some(binary) = std::env::var_os("ROCM_XTASK_BINARY") { std::process::Command::new(binary) } else { diff --git a/tests/e2e-cucumber/tests/e2e/model_catalog_steps.rs b/tests/e2e-cucumber/tests/e2e/model_catalog_steps.rs index 3e0e077d5b..b79c66ff17 100644 --- a/tests/e2e-cucumber/tests/e2e/model_catalog_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/model_catalog_steps.rs @@ -24,10 +24,23 @@ async fn catalog_lists_aim_recipe(world: &mut E2eWorld) { Some(0), "listing the catalog is not a failure:\n{output}" ); + // The recipe is listed by its model name (`aim-qwen3.5-9b` is an alias) and + // shares the entry with vLLM, so match the AIM engine within that one line. assert!( - output.contains("aim-qwen3.5-9b") && output.contains("engines=[aim]"), + output + .lines() + .any(|line| line.contains("Qwen/Qwen3.5-9B") && line.contains("engines=[aim")), "expected the AIM recipe listed by canonical id and engine:\n{output}" ); + // The host-fit fields of the same entry: which GPU families it covers, that + // it needs no Hugging Face token, and the profile pinned for a family. + assert!( + output.lines().any(|line| line.contains("engine_recipe aim") + && line.contains("supported_gfx_families=[gfx110X-all,gfx120X-all]") + && line.contains("requires_hf_token=no") + && line.contains("gfx110X-all=vllm-w7900-bf16-tp1-latency")), + "expected the AIM recipe's host-fit line in the verbose listing:\n{output}" + ); } /// Plant a `docker` on `PATH` that always fails, so the catalog's @@ -60,3 +73,90 @@ async fn catalog_aim_entry_flagged_unavailable(world: &mut E2eWorld) { "expected the AIM entry to name the Docker-unavailable reason:\n{output}" ); } + +/// Unix ms for 2001-09-09, far more than 180 days before any run of this suite. +const LONG_AGO_MS: u64 = 1_000_000_000_000; + +/// Point the CLI at a signed recipe index holding the built-in `Qwen/Qwen3.5-9B` +/// AIM entry with `generated_at_unix_ms` set `LONG_AGO_MS`, the only difference +/// from the shipped entry. Signed because that is the only way the CLI loads a +/// recipe index (same xtask keygen/sign path as `diagnose_steps`). +#[given("a recipe catalog whose AIM entry was last verified long ago")] +async fn catalog_with_stale_aim_entry(world: &mut E2eWorld) { + let mut recipe = rocm_core::resolve_builtin_model_recipe("Qwen/Qwen3.5-9B") + .expect("the built-in AIM recipe"); + for engine_recipe in &mut recipe.engine_recipes { + if engine_recipe.engine == "aim" { + engine_recipe.generated_at_unix_ms = Some(u128::from(LONG_AGO_MS)); + } + } + let root = world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .to_path_buf(); + let index = root.join("stale-aim-recipes.json"); + let signature = root.join("stale-aim-recipes.json.sig"); + let public_key = root.join("stale-aim-public.pem"); + let private_key = root.join("stale-aim-private.pem"); + let document = serde_json::json!({ + "schema_version": 1, + "source": "e2e-stale-aim", + "recipes": [recipe], + }); + std::fs::write( + &index, + serde_json::to_vec_pretty(&document).expect("failed to serialize the recipe fixture"), + ) + .expect("failed to write the recipe fixture"); + + let keygen = crate::e2e::diagnose_steps::xtask_command() + .args(["keygen", "--private-out"]) + .arg(&private_key) + .arg("--public-out") + .arg(&public_key) + .current_dir(crate::e2e::diagnose_steps::workspace_root()) + .status() + .expect("failed to run xtask keygen"); + assert!(keygen.success(), "xtask keygen failed"); + let sign = crate::e2e::diagnose_steps::xtask_command() + .args(["sign", "--private-key"]) + .arg(&private_key) + .arg("--in") + .arg(&index) + .arg("--out") + .arg(&signature) + .current_dir(crate::e2e::diagnose_steps::workspace_root()) + .status() + .expect("failed to run xtask sign"); + assert!(sign.success(), "xtask sign failed"); + + world + .command_env + .push(("ROCM_CLI_MODEL_RECIPE_INDEX_PATH", index.into_os_string())); + world.command_env.push(( + "ROCM_CLI_MODEL_RECIPE_INDEX_SIGNATURE_PATH", + signature.into_os_string(), + )); + world.command_env.push(( + "ROCM_CLI_MODEL_RECIPE_INDEX_PUBLIC_KEY_PATH", + public_key.into_os_string(), + )); +} + +#[then("the catalog warns that the AIM entry may no longer be current")] +async fn catalog_warns_aim_entry_is_stale(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no model catalog output"); + assert_eq!( + world.cli_rc, + Some(0), + "listing the catalog failed:\n{output}" + ); + assert!( + output.contains("warning: aim entry was last verified") + && output.contains("days ago") + && output.contains("entries are refreshed by hand"), + "expected a staleness warning for the AIM entry, got:\n{output}" + ); +} diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 26a1328b37..49ab899ce3 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1094,6 +1094,123 @@ async fn user_serves_negative_temperature(world: &mut E2eWorld) { world.cli_rc = Some(rc); } +/// A gated AIM catalog entry (`requires_hf_token`), with a profile for a Radeon +/// PRO W7900 (`gfx110X-all`). The token check runs before GPU detection and +/// Docker, so no engine or GPU is needed to reach it. +const GATED_AIM_MODEL: &str = "meta-llama/Llama-3.1-8B-Instruct"; + +/// An AIM image with profiles for Radeon (`gfx110X-all`, `gfx120X-all`) only. +const RADEON_AIM_MODEL: &str = "Qwen/Qwen3.5-9B"; + +/// An AIM image that needs 8 GPUs on an Instinct MI300X. +const EIGHT_GPU_AIM_MODEL: &str = "deepseek-ai/DeepSeek-R1"; + +/// Put a `docker` that always fails first on `PATH` and have it log every call to +/// a file in the scenario root, so a refused serve can be shown never to have +/// reached Docker (not even `docker info`). +#[given("Docker is watched for any call")] +async fn docker_is_watched(world: &mut E2eWorld) { + let root = world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .to_path_buf(); + let bin_dir = root.join("watched-docker-bin"); + crate::install_fixture_binary( + "fake-docker", + &bin_dir.join(e2e_cucumber::fixture_binary::fixture_target_name("docker")), + ); + let mut path = std::ffi::OsString::from(&bin_dir); + path.push(if cfg!(windows) { ";" } else { ":" }); + path.push(std::env::var_os("PATH").unwrap_or_default()); + world.command_env.push(("PATH", path)); + world + .command_env + .push(("FAKE_DOCKER_LOG", docker_call_log(world).into_os_string())); +} + +fn docker_call_log(world: &E2eWorld) -> std::path::PathBuf { + world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .join("docker-calls.log") +} + +/// Plant one GPU in the simulated kernel topology (`ROCM_CLI_TEST_HOST_ROOT`, honoured +/// by `e2e-test-hooks` builds) so the family the CLI detects does not depend on the +/// runner's hardware. `gfx_target_version` is the kernel's decimal encoding. +fn plant_single_gpu(world: &mut E2eWorld, gfx_target_version: u32) { + let host_root = world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .join("host-root"); + let node = host_root.join("sys/class/kfd/kfd/topology/nodes/1"); + std::fs::create_dir_all(&node).expect("failed to create the planted KFD node"); + // `cpu_cores_count 0` with a nonzero `simd_count` makes the node a GPU. + std::fs::write( + node.join("properties"), + format!("cpu_cores_count 0\nsimd_count 56\ngfx_target_version {gfx_target_version}\n"), + ) + .expect("failed to write the planted KFD node properties"); + world + .command_env + .push(("ROCM_CLI_TEST_HOST_ROOT", host_root.into_os_string())); +} + +/// gfx942, TheRock family `gfx94X-dcgpu`. +#[given("a host whose only GPU is an AMD Instinct MI300X")] +async fn host_with_an_mi300x(world: &mut E2eWorld) { + plant_single_gpu(world, 90402); +} + +/// gfx1100, TheRock family `gfx110X-all`. +#[given("a host whose only GPU is an AMD Radeon PRO W7900")] +async fn host_with_a_w7900(world: &mut E2eWorld) { + plant_single_gpu(world, 110_000); +} + +fn run_serve_aim(world: &mut E2eWorld, args: &[&str]) { + let mut full = vec!["serve"]; + full.extend_from_slice(args); + full.extend_from_slice(&["--engine", "aim"]); + let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env(world, &full); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[when("the user serves a gated AIM model without passing a Hugging Face token")] +async fn user_serves_gated_aim_without_token(world: &mut E2eWorld) { + run_serve_aim(world, &[GATED_AIM_MODEL]); +} + +#[when( + "the user serves a gated AIM model with an exported Hugging Face token while every GPU is masked" +)] +async fn user_serves_gated_aim_with_token(world: &mut E2eWorld) { + world + .command_env + .push(("HF_TOKEN", "hf_e2e_placeholder".into())); + // An empty mask hides every device, so a GPU host presents as having none usable. + world.command_env.push(("HIP_VISIBLE_DEVICES", "".into())); + run_serve_aim(world, &[GATED_AIM_MODEL, "--engine-env", "HF_TOKEN"]); +} + +#[when("the user serves a Radeon-only AIM image")] +async fn user_serves_radeon_only_aim(world: &mut E2eWorld) { + run_serve_aim(world, &[RADEON_AIM_MODEL]); +} + +#[when("the user serves an AIM image that needs eight GPUs")] +async fn user_serves_eight_gpu_aim(world: &mut E2eWorld) { + run_serve_aim(world, &[EIGHT_GPU_AIM_MODEL]); +} + /// Serve with every GPU hidden from the CLI and engine via an empty /// `HIP_VISIBLE_DEVICES`, so a GPU host presents as having no usable device. #[when("the user serves a model with every GPU masked from view")] @@ -1858,6 +1975,71 @@ async fn assert_no_engine_preparation(world: &mut E2eWorld) { ); } +#[then("the CLI explains that the model needs a Hugging Face token")] +async fn assert_hf_token_message(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + output.contains("needs a Hugging Face token") && output.contains("--engine-env HF_TOKEN"), + "expected the gated-model refusal naming the remedy, got:\n{output}" + ); +} + +#[then("no service was started")] +async fn assert_no_service_started(world: &mut E2eWorld) { + let services_dir = world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .join("data") + .join("services"); + let records = std::fs::read_dir(&services_dir).map_or(0, Iterator::count); + assert_eq!(records, 0, "a refused serve left service records behind"); +} + +#[then("the refusal is that no usable GPU is visible")] +async fn assert_refusal_is_no_usable_gpu(world: &mut E2eWorld) { + // The token gate and the GPU-family gate were both passed: the serve got as + // far as the GPU-required pre-flight, which the empty mask fails. + let output = serve_output(world); + assert!( + output.contains("no usable AMD GPU") && !output.contains("needs a Hugging Face token"), + "expected the exported token to clear its gate and the serve to stop at the \ + GPU pre-flight, got:\n{output}" + ); +} + +#[then("the CLI says the image has no profile for this GPU and names the GPUs it supports")] +async fn assert_no_profile_for_gpu(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + output.contains("has no profile for this GPU (gfx94X-dcgpu)") + && output.contains("it supports: gfx110X-all, gfx120X-all"), + "expected the unsupported-GPU refusal naming the detected and supported families, \ + got:\n{output}" + ); +} + +#[then("the CLI says the image needs eight GPUs and multi-GPU serving is not supported")] +async fn assert_needs_eight_gpus(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + output.contains("needs 8 GPUs") && output.contains("does not support multi-GPU serving"), + "expected the multi-GPU refusal, got:\n{output}" + ); +} + +#[then("Docker was never called")] +async fn assert_docker_never_called(world: &mut E2eWorld) { + let log = docker_call_log(world); + let calls = std::fs::read_to_string(&log).unwrap_or_default(); + assert!( + calls.is_empty(), + "a refused serve reached Docker ({}):\n{calls}", + log.display() + ); +} + #[then("the CLI explains that temperature cannot be negative")] async fn assert_negative_temperature_message(world: &mut E2eWorld) { let output = serve_output(world);