From 80ceb084d82dda917beaa07fe3a86d6f87bb7f69 Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Thu, 29 Oct 2026 12:21:27 +0200 Subject: [PATCH 01/21] ROCMAI-556: warn when an AIM catalog entry hasn't been verified recently Add a per-engine-recipe generated_at_unix_ms, warn in `rocm model` once it is over 180 days old, and document the hand-maintained AIM refresh process. Signed-off-by: Juho Vainio --- CONTRIBUTING.md | 5 +++ README.md | 4 +- apps/rocm/src/examine_report_text.rs | 9 ++++ apps/rocm/src/serve_cmd.rs | 2 + crates/rocm-core/src/lib.rs | 20 +++++---- crates/rocm-core/src/model_catalog.json | 1 + crates/rocm-core/src/model_catalog.rs | 59 +++++++++++++++++++++++++ docs/aim-catalog.md | 39 ++++++++++++++++ docs/testing.md | 1 + 9 files changed, 130 insertions(+), 10 deletions(-) create mode 100644 docs/aim-catalog.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3e6442a63a..fc9b88922c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -91,6 +91,11 @@ Every local (relative-path) Markdown link and `#anchor` fragment in the reposito Since only links are checked, cite a specific repository file as a link relative to the citing file, `[fix.rs](../crates/rocm-core/src/fix.rs)` from `docs/`, rather than a bare backtick path, which nothing checks. The full rule, including the link forms that fail the check and the places it doesn't apply (`.github/`, links leaving a `skills/` folder, and `docs/rocm-docs/` along with the files it includes, which are this file and README.md), is in AGENTS.md section 5 ("Investigate rocm-cli Before Editing"); it applies to citations you add or edit. +### Updating the AIM catalog + +AIM model entries are hand-maintained in `crates/rocm-core/src/model_catalog.json`; +see `docs/aim-catalog.md` for the refresh process and staleness warning. + ### Test commands | Component | Command | diff --git a/README.md b/README.md index 71e8da0eba..0b9f33ea52 100644 --- a/README.md +++ b/README.md @@ -821,7 +821,9 @@ rocm model [--verbose] `rocm model` (alias `rocm models`) lists the recommended local models, grouped by the hardware they target. Pass `--verbose` to list every recipe, including ones hidden from the default list, with details such as aliases, data type, minimum -GPU memory, and preferred engines. +GPU memory, and preferred engines. AIM entries are hand-maintained and record +when they were last verified; `rocm model` warns when one is over 180 days old +(see [docs/aim-catalog.md](docs/aim-catalog.md)). ### Model serving diff --git a/apps/rocm/src/examine_report_text.rs b/apps/rocm/src/examine_report_text.rs index c401189c0d..e886a15af5 100644 --- a/apps/rocm/src/examine_report_text.rs +++ b/apps/rocm/src/examine_report_text.rs @@ -700,6 +700,14 @@ pub(crate) fn render_model_registry_verbose_text_with_context_and_host( for warning in &recipe.warnings { let _ = writeln!(output, " warning: {warning}"); } + for engine_recipe in &recipe.engine_recipes { + if let Some(warning) = rocm_core::model_recipe_staleness_warning( + engine_recipe, + rocm_core::unix_time_millis(), + ) { + let _ = writeln!(output, " warning: {warning}"); + } + } } append_model_recipe_registry_source(&mut output, ®istry); output @@ -2028,6 +2036,7 @@ mod tests { notes: vec!["metadata only; not applied to launches yet".to_owned()], model_id_override: None, image_digest: None, + generated_at_unix_ms: None, extra_env: BTreeMap::new(), }]; let mut output = String::new(); diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 29beb0ec97..6c56a3af79 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -1617,6 +1617,7 @@ mod tests { notes: vec!["adapter hint".to_owned()], model_id_override: None, image_digest: None, + generated_at_unix_ms: None, extra_env: BTreeMap::new(), }, rocm_core::ModelRecipeEngineRecord { @@ -1628,6 +1629,7 @@ mod tests { notes: Vec::new(), model_id_override: None, image_digest: None, + generated_at_unix_ms: None, extra_env: BTreeMap::new(), }, ]; diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 1685a9fc1e..606d7f3341 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -124,15 +124,17 @@ use managed_runtime::{ TheRockFamilyManifest, managed_sdk_tool_path, sort_records_newest_install_first, }; pub use model_catalog::{ - ModelArtifactCacheStatus, ModelCatalogPlatform, ModelRecipeArtifactRecord, - ModelRecipeArtifactSourcePolicyRecord, ModelRecipeEndpointRecord, ModelRecipeEngineRecord, - ModelRecipeIndexDocument, ModelRecipeRecord, ModelRecipeRegistry, ModelRecipeRegistrySource, - ModelRecipeUnsupportedCombinationRecord, builtin_model_recipe_registry, builtin_model_recipes, - load_model_recipe_index, load_model_recipe_registry, load_signed_model_recipe_index, - model_artifact_cache_marker_path, model_artifact_cache_status, model_catalog_platforms, - model_recipe_featured, model_recipe_index_signature_path, model_recipe_target_platform_label, - normalize_model_ref, platform_matches_gfx_family, resolve_builtin_model_recipe, - resolve_model_recipe, resolve_model_recipe_artifact, verify_model_recipe_index_signature, + MODEL_RECIPE_STALE_AFTER_MS, ModelArtifactCacheStatus, ModelCatalogPlatform, + ModelRecipeArtifactRecord, ModelRecipeArtifactSourcePolicyRecord, ModelRecipeEndpointRecord, + ModelRecipeEngineRecord, ModelRecipeIndexDocument, ModelRecipeRecord, ModelRecipeRegistry, + ModelRecipeRegistrySource, ModelRecipeUnsupportedCombinationRecord, + builtin_model_recipe_registry, builtin_model_recipes, load_model_recipe_index, + load_model_recipe_registry, load_signed_model_recipe_index, model_artifact_cache_marker_path, + model_artifact_cache_status, model_catalog_platforms, model_recipe_featured, + model_recipe_index_signature_path, model_recipe_staleness_warning, + model_recipe_target_platform_label, normalize_model_ref, platform_matches_gfx_family, + resolve_builtin_model_recipe, resolve_model_recipe, resolve_model_recipe_artifact, + verify_model_recipe_index_signature, }; pub use net::{ Backoff, DEFAULT_LOCAL_HOST, DEFAULT_LOCAL_PORT, DOWNLOAD_MAX_ATTEMPTS, DownloadOutcome, diff --git a/crates/rocm-core/src/model_catalog.json b/crates/rocm-core/src/model_catalog.json index c107637f3a..b39fa225fd 100644 --- a/crates/rocm-core/src/model_catalog.json +++ b/crates/rocm-core/src/model_catalog.json @@ -539,6 +539,7 @@ ], "model_id_override": "amdenterpriseai/aim-radeon-qwen-qwen3-5-9b:0.12.0-preview", "image_digest": "sha256:ee1201261f358be881318de5930ce9505d17309486f747a83147d7d042fc95ce", + "generated_at_unix_ms": 1793203564000, "extra_env": { "AIM_PROFILE_ID": "vllm-w7900-bf16-tp1-latency" } diff --git a/crates/rocm-core/src/model_catalog.rs b/crates/rocm-core/src/model_catalog.rs index 18c4126041..e551584730 100644 --- a/crates/rocm-core/src/model_catalog.rs +++ b/crates/rocm-core/src/model_catalog.rs @@ -92,6 +92,31 @@ pub struct ModelRecipeEngineRecord { /// (currently `aim`); catalog validation rejects it elsewhere. #[serde(default)] pub image_digest: Option, + /// When a maintainer last checked this entry against the upstream image + /// registry (unix ms). `rocm model` warns once it is older than + /// [`MODEL_RECIPE_STALE_AFTER_MS`], since the pinned image may be gone. + #[serde(default)] + pub generated_at_unix_ms: Option, +} + +/// How long an engine recipe's `generated_at_unix_ms` stays trusted (180 days). +pub const MODEL_RECIPE_STALE_AFTER_MS: u128 = 180 * 24 * 60 * 60 * 1000; + +/// Returns a warning when `engine_recipe` was last verified more than +/// [`MODEL_RECIPE_STALE_AFTER_MS`] before `now_ms`. Entries without a +/// timestamp are never flagged. +pub fn model_recipe_staleness_warning( + engine_recipe: &ModelRecipeEngineRecord, + now_ms: u128, +) -> Option { + let age_ms = now_ms.checked_sub(engine_recipe.generated_at_unix_ms?)?; + (age_ms > MODEL_RECIPE_STALE_AFTER_MS).then(|| { + format!( + "{} entry was last verified {} days ago; the pinned image may no longer exist (see docs/aim-catalog.md to refresh)", + engine_recipe.engine, + age_ms / (24 * 60 * 60 * 1000) + ) + }) } #[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)] @@ -950,6 +975,38 @@ mod tests { assert_eq!(tiny.min_gpu_mem_gb, Some(2)); } + #[test] + fn staleness_warning_fires_only_past_the_threshold() { + let mut aim = ModelRecipeEngineRecord { + engine: "aim".to_owned(), + ..Default::default() + }; + let now = 10 * MODEL_RECIPE_STALE_AFTER_MS; + assert_eq!(model_recipe_staleness_warning(&aim, now), None); + aim.generated_at_unix_ms = Some(now - MODEL_RECIPE_STALE_AFTER_MS); + assert_eq!(model_recipe_staleness_warning(&aim, now), None); + aim.generated_at_unix_ms = Some(now - MODEL_RECIPE_STALE_AFTER_MS - 1); + assert!( + model_recipe_staleness_warning(&aim, now) + .expect("stale entry warns") + .contains("180 days ago") + ); + // A timestamp in the future (clock skew) is not stale. + aim.generated_at_unix_ms = Some(now + 1); + assert_eq!(model_recipe_staleness_warning(&aim, now), None); + } + + #[test] + fn builtin_aim_recipe_carries_a_verification_timestamp() { + let recipe = resolve_builtin_model_recipe("aim-qwen3.5-9b").expect("aim recipe"); + let aim = recipe + .engine_recipes + .iter() + .find(|e| e.engine == "aim") + .expect("aim engine"); + assert!(aim.generated_at_unix_ms.is_some()); + } + #[test] fn builtin_recipe_records_remote_code_policy() { let glm = resolve_builtin_model_recipe("glm-5").expect("glm alias should resolve"); @@ -976,6 +1033,7 @@ mod tests { notes: Vec::new(), model_id_override: None, image_digest: Some(digest.to_owned()), + generated_at_unix_ms: None, }); ModelRecipeIndexDocument { schema_version: 1, @@ -1029,6 +1087,7 @@ mod tests { notes: vec!["metadata only; adapter protocol does not consume this yet".to_owned()], model_id_override: None, image_digest: None, + generated_at_unix_ms: None, }); let index = ModelRecipeIndexDocument { schema_version: 1, diff --git a/docs/aim-catalog.md b/docs/aim-catalog.md new file mode 100644 index 0000000000..c433a0c0a8 --- /dev/null +++ b/docs/aim-catalog.md @@ -0,0 +1,39 @@ + + +# AIM catalog: freshness and updates + +AIM entries live in [model_catalog.json](../crates/rocm-core/src/model_catalog.json) +alongside every other model recipe (`"loader": "aim"`). They are hand-maintained +and ship on the normal CLI release cadence, through the same PR review as any +other recipe. There is no network fetch of the catalog: a hosted, signed +recipe index is blocked on production signing keys (see +[release-trust.md](release-trust.md)) and is out of scope here. + +## Freshness + +Each AIM `engine_recipes` entry pins an image by `image_digest` and records +`generated_at_unix_ms`, the time a maintainer last confirmed the image and +digest still exist in the registry. `rocm model` prints a warning for any +engine recipe whose timestamp is older than 180 days +(`MODEL_RECIPE_STALE_AFTER_MS` in +[model_catalog.rs](../crates/rocm-core/src/model_catalog.rs)), because the +pinned image may have been withdrawn. Entries with no timestamp are never +flagged. + +## Refreshing an entry + +The owner of the AIM relationship (likely the AMD EAI team) does this; it does +not need a rocm-cli maintainer. + +1. Watch for AMD Enterprise AI Suite releases. A version bump is the practical + signal that AIM images were added or changed. +2. For each AIM entry, confirm the image tag still exists and copy its current + digest into `image_digest` (and `model_id_override` if the tag moved). + Check `AIM_PROFILE_ID` in `extra_env` is still a valid profile. +3. Set `generated_at_unix_ms` to now (`date +%s%3N`). +4. Add entries for new AIM images the same way as the existing one. +5. Open a PR; `cargo test -p rocm-core model_recipe` validates the catalog. diff --git a/docs/testing.md b/docs/testing.md index 1892bf2482..376ae3126e 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -845,6 +845,7 @@ cargo test -p rocm --bin rocm render_model_registry_text_reports_host_ram_fit cargo test -p rocm --bin rocm model_recipe cargo test -p rocm --bin rocm logs cargo test -p rocm-core model_recipe +cargo test -p rocm-core staleness_warning cargo test -p rocm-core load_model_recipe_index cargo test -p rocm --bin rocm update_report_policy cargo test -p rocm-engine-lemonade stdio_protocol_routes_all_methods_without_side_effects From f1732f92dd638debec2514f55c517651c81576f5 Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Thu, 29 Oct 2026 13:22:47 +0200 Subject: [PATCH 02/21] ROCMAI-556: add the AMD Enterprise AI AIM catalog to the model catalog Adds 38 GPU chat AIM images, each pinned to the newest stable tag's digest and stamped with generated_at_unix_ms. All are hidden from the default list. Signed-off-by: Juho Vainio --- crates/rocm-core/src/model_catalog.json | 1558 +++++++++++++++++++++++ docs/aim-catalog.md | 12 +- 2 files changed, 1567 insertions(+), 3 deletions(-) diff --git a/crates/rocm-core/src/model_catalog.json b/crates/rocm-core/src/model_catalog.json index b39fa225fd..2e5795e5e6 100644 --- a/crates/rocm-core/src/model_catalog.json +++ b/crates/rocm-core/src/model_catalog.json @@ -553,6 +553,1564 @@ "warnings": [ "Radeon-targeted AIM image; requires Docker and an explicit AIM_PROFILE_ID" ] + }, + { + "canonical_model_id": "aim-coherelabs-command-a-reasoning-08-2025", + "aliases": [ + "amdenterpriseai/aim-coherelabs-command-a-reasoning-08-2025" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-coherelabs-command-a-reasoning-08-2025:0.11.1", + "image_digest": "sha256:6da329641e3ea98e945f43b1df17b7efb3ba0a284dfb123297ace72d4084b4d3", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-deepseek-ai-deepseek-r1-0528", + "aliases": [ + "amdenterpriseai/aim-deepseek-ai-deepseek-r1-0528" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-r1-0528:0.11.1", + "image_digest": "sha256:348f6ee92b07bf150b01c7b37adb8365336ffc98a2a6f747ed73754cd41f61bc", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-deepseek-ai-deepseek-r1", + "aliases": [ + "amdenterpriseai/aim-deepseek-ai-deepseek-r1" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-r1:0.11.1", + "image_digest": "sha256:4bbea192e4c3fd11865652119c584e9bad27b07634d5d56da2ca1bcf1c74473d", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-deepseek-ai-deepseek-v3-1-terminus", + "aliases": [ + "amdenterpriseai/aim-deepseek-ai-deepseek-v3-1-terminus" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-v3-1-terminus:0.11.1", + "image_digest": "sha256:5f0335322a88bd4a8abe0865227287f419b906b47d9913cf8de7a35df80142f4", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-deepseek-ai-deepseek-v3-1", + "aliases": [ + "amdenterpriseai/aim-deepseek-ai-deepseek-v3-1" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-v3-1:0.11.1", + "image_digest": "sha256:9fc6f3c6e5b165352ed0873d60bc1acfc1aeaebc048da4630d3166e0b498fb33", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-deepseek-ai-deepseek-v4-flash-0731", + "aliases": [ + "amdenterpriseai/aim-deepseek-ai-deepseek-v4-flash-0731" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-v4-flash-0731:2026.11.0", + "image_digest": "sha256:bb4e2c8b9ecfcfac886d7c5ebf2629c62ded2debac804d8bf2c11483f7ac6d8c", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-google-gemma-3-1b-it", + "aliases": [ + "amdenterpriseai/aim-google-gemma-3-1b-it" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-google-gemma-3-1b-it:0.12.0", + "image_digest": "sha256:fb49e69fe72ca40efaef97173d1532e09982a141900905015d41267ced1654cf", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-google-gemma-3-27b-it", + "aliases": [ + "amdenterpriseai/aim-google-gemma-3-27b-it" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-google-gemma-3-27b-it:0.13.0", + "image_digest": "sha256:2340abdb1c7cf34a84c5398607b57e1a7cf73b7099ba3fdbde8fbb810e849e33", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-google-gemma-4-31b-it", + "aliases": [ + "amdenterpriseai/aim-google-gemma-4-31b-it" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-google-gemma-4-31b-it:0.13.0-preview", + "image_digest": "sha256:ec2c6940ebd7088f112d09c4c6f554e4d3efaa88b61c89be9f1ecd347beaef9a", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-google-medgemma-27b-it", + "aliases": [ + "amdenterpriseai/aim-google-medgemma-27b-it" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-google-medgemma-27b-it:0.13.0", + "image_digest": "sha256:a89823487230a8cea1f9d49f881e64f20375ee793739c223ab2c09f7bc30a088", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-meta-llama-llama-3-1-405b-instruct", + "aliases": [ + "amdenterpriseai/aim-meta-llama-llama-3-1-405b-instruct" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-1-405b-instruct:0.11.1", + "image_digest": "sha256:bfe038184d0f0d8a415ee057f4fe64b984dbb743874d0907448be9ceed432743", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-meta-llama-llama-3-1-8b-instruct", + "aliases": [ + "amdenterpriseai/aim-meta-llama-llama-3-1-8b-instruct" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-1-8b-instruct:0.11.1", + "image_digest": "sha256:dd386ba89444bf7b16016a622127f58a58bb875d47d528b0f95186cf170c1df7", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-meta-llama-llama-3-2-1b-instruct", + "aliases": [ + "amdenterpriseai/aim-meta-llama-llama-3-2-1b-instruct" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-2-1b-instruct:0.11.1", + "image_digest": "sha256:92e87333a235d5af0494d3148a5dea537da041de671b8b9a9db05bfbd2bfb164", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-meta-llama-llama-3-2-3b-instruct", + "aliases": [ + "amdenterpriseai/aim-meta-llama-llama-3-2-3b-instruct" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-2-3b-instruct:0.11.1", + "image_digest": "sha256:12e146a90d3ad26bafc102ac0217f5d49b3107c9e7abc476d16ad3f1ae7de4ae", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-meta-llama-llama-3-3-70b-instruct", + "aliases": [ + "amdenterpriseai/aim-meta-llama-llama-3-3-70b-instruct" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-3-70b-instruct:0.11.1", + "image_digest": "sha256:c8d1bb99119241592d5749e7e707224154a7acae081558d5a30caefc75c32338", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-minimaxai-minimax-m2-5", + "aliases": [ + "amdenterpriseai/aim-minimaxai-minimax-m2-5" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-minimaxai-minimax-m2-5:0.12.0", + "image_digest": "sha256:0a20a6dd756dc3ebc9cc0ada23f95e8edf49fbe33354652702f6d878380323ed", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-minimaxai-minimax-m3", + "aliases": [ + "amdenterpriseai/aim-minimaxai-minimax-m3" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-minimaxai-minimax-m3:2026.9.0", + "image_digest": "sha256:2b7221c5fa3592afefb0e904d10b063126ebe76f71ca7a5b8cd2c2a8a26c10c9", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-mistralai-ministral-3-14b-instruct-2512", + "aliases": [ + "amdenterpriseai/aim-mistralai-ministral-3-14b-instruct-2512" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-mistralai-ministral-3-14b-instruct-2512:0.11.1", + "image_digest": "sha256:8fa94cb5351554ca32c15543b48d74b79ef214a29f8066501c841d38798f6504", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-mistralai-ministral-3-14b-reasoning-2512", + "aliases": [ + "amdenterpriseai/aim-mistralai-ministral-3-14b-reasoning-2512" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-mistralai-ministral-3-14b-reasoning-2512:0.11.1", + "image_digest": "sha256:50cb5fc4f2a2a6973f63adcf0f8621d091dd825f67f3808cb6d743ded9d2a00a", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-mistralai-mistral-large-3-675b-instruct-2512", + "aliases": [ + "amdenterpriseai/aim-mistralai-mistral-large-3-675b-instruct-2512" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-mistralai-mistral-large-3-675b-instruct-2512:0.10.0", + "image_digest": "sha256:6119b9c717b44ff4bec3183318623738d2d95a99ecd85e0fe5bdca98bf24dbdd", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-mistralai-mistral-small-24b-instruct-2501", + "aliases": [ + "amdenterpriseai/aim-mistralai-mistral-small-24b-instruct-2501" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-mistralai-mistral-small-24b-instruct-2501:0.13.0", + "image_digest": "sha256:976a1e5a9880756058dad68dd3f543d76f01f1955a436f1bf5b03832704e2b20", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-mistralai-mistral-small-3-2-24b-instruct-2506", + "aliases": [ + "amdenterpriseai/aim-mistralai-mistral-small-3-2-24b-instruct-2506" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-mistralai-mistral-small-3-2-24b-instruct-2506:0.11.1", + "image_digest": "sha256:7883d07cc867795a307073855a716b0c7e623e7f1327a6c347a338f95760f719", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-mistralai-mixtral-8x22b-instruct-v0-1", + "aliases": [ + "amdenterpriseai/aim-mistralai-mixtral-8x22b-instruct-v0-1" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-mistralai-mixtral-8x22b-instruct-v0-1:0.11.1", + "image_digest": "sha256:bcbe47ea00037f1d3b203a47c95bcfaa2dc76b5739a94bfdf0d681be1385c8a7", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-mistralai-mixtral-8x7b-instruct-v0-1", + "aliases": [ + "amdenterpriseai/aim-mistralai-mixtral-8x7b-instruct-v0-1" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-mistralai-mixtral-8x7b-instruct-v0-1:0.11.1", + "image_digest": "sha256:9f2abb10dc3a21169d7cd0e6f96294df3683dd42ea49f1d5230e62722f4ae738", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-openai-gpt-oss-120b", + "aliases": [ + "amdenterpriseai/aim-openai-gpt-oss-120b" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-openai-gpt-oss-120b:0.11.1", + "image_digest": "sha256:a3052d6230d7a414837903c31be0122f6615819448598e9caea973c084dc3ea7", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-openai-gpt-oss-20b", + "aliases": [ + "amdenterpriseai/aim-openai-gpt-oss-20b" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-openai-gpt-oss-20b:0.11.1", + "image_digest": "sha256:67cee29f3f9036eb3918190c76bc4442dd48254657e680d0256a5018c2164274", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-qwen-qwen3-235b-a22b", + "aliases": [ + "amdenterpriseai/aim-qwen-qwen3-235b-a22b" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-qwen-qwen3-235b-a22b:0.11.1", + "image_digest": "sha256:427d25b93cce97b499f9c1e9268b33eea25a8da8a0f7ef7904f4cdeef5f52f5a", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-qwen-qwen3-32b", + "aliases": [ + "amdenterpriseai/aim-qwen-qwen3-32b" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-qwen-qwen3-32b:0.13.0", + "image_digest": "sha256:3e8d47db7c88ca1b7bb210b7bfd5a66f0439e3b768f200ab4aa51fb058d2871c", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-qwen-qwen3-coder-next", + "aliases": [ + "amdenterpriseai/aim-qwen-qwen3-coder-next" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-qwen-qwen3-coder-next:0.11.1", + "image_digest": "sha256:0874c97e547a59df1893813e3f011c7e5bc1d035b68e99361f4175c0e15010c0", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-qwen-qwen3-vl-235b-a22b-instruct", + "aliases": [ + "amdenterpriseai/aim-qwen-qwen3-vl-235b-a22b-instruct" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-qwen-qwen3-vl-235b-a22b-instruct:0.11.1", + "image_digest": "sha256:4353ab9fb67d30b54d4c724429c0401490b42aa96045a050faea7b46a9e2d3d6", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-qwen-qwen3-vl-235b-a22b-thinking", + "aliases": [ + "amdenterpriseai/aim-qwen-qwen3-vl-235b-a22b-thinking" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-qwen-qwen3-vl-235b-a22b-thinking:0.11.1", + "image_digest": "sha256:c10ee1a4189708e101802b4b0b3d292dfc4802ad08c6da70a0f56ae2094eeccb", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-radeon-google-gemma-3n-e4b-it", + "aliases": [ + "amdenterpriseai/aim-radeon-google-gemma-3n-e4b-it" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Radeon-targeted image; if profile auto-selection fails, set AIM_PROFILE_ID with --engine-env" + ], + "model_id_override": "amdenterpriseai/aim-radeon-google-gemma-3n-e4b-it:0.12.0-preview", + "image_digest": "sha256:f5dd9116c6714a8e51e6bdbf94f5a2a8b06db62fc95994e6b331f92931416697", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-radeon-meta-llama-llama-3-1-8b-instruct", + "aliases": [ + "amdenterpriseai/aim-radeon-meta-llama-llama-3-1-8b-instruct" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Radeon-targeted image; if profile auto-selection fails, set AIM_PROFILE_ID with --engine-env" + ], + "model_id_override": "amdenterpriseai/aim-radeon-meta-llama-llama-3-1-8b-instruct:2026.11.0", + "image_digest": "sha256:a300b53f5f919a59d0cdc4bc177a6db5ef1660719cb50c128be6f8ec4a7c79d5", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-radeon-qwen-qwen3-6-35b-a3b", + "aliases": [ + "amdenterpriseai/aim-radeon-qwen-qwen3-6-35b-a3b" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Radeon-targeted image; if profile auto-selection fails, set AIM_PROFILE_ID with --engine-env" + ], + "model_id_override": "amdenterpriseai/aim-radeon-qwen-qwen3-6-35b-a3b:2026.11.0", + "image_digest": "sha256:4df926af5c9e792675577cf881fb12b4964b6dac18f6d3f7c1e2cf460834eee2", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-radeon-qwen-qwen3-vl-8b-instruct", + "aliases": [ + "amdenterpriseai/aim-radeon-qwen-qwen3-vl-8b-instruct" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Radeon-targeted image; if profile auto-selection fails, set AIM_PROFILE_ID with --engine-env" + ], + "model_id_override": "amdenterpriseai/aim-radeon-qwen-qwen3-vl-8b-instruct:2026.11.0", + "image_digest": "sha256:89775113626743d77023e15b595d4218c1e46e89e932b3f3ee5c0350914add55", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-radeon-zai-org-glm-4-7-flash", + "aliases": [ + "amdenterpriseai/aim-radeon-zai-org-glm-4-7-flash" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "Radeon-targeted image; if profile auto-selection fails, set AIM_PROFILE_ID with --engine-env" + ], + "model_id_override": "amdenterpriseai/aim-radeon-zai-org-glm-4-7-flash:0.12.0-preview", + "image_digest": "sha256:b94130be1f12eaab28bdce274a63b3c10a24b8e292dc91f4c563b7a42246c1ab", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-zai-org-glm-4-7", + "aliases": [ + "amdenterpriseai/aim-zai-org-glm-4-7" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-zai-org-glm-4-7:0.12.0", + "image_digest": "sha256:6555005fe59ad9334ce292ca3df4b619106656a731e7319114c2b3a43a1b1219", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] + }, + { + "canonical_model_id": "aim-zai-org-glm-5-2", + "aliases": [ + "amdenterpriseai/aim-zai-org-glm-5-2" + ], + "task": "chat", + "source": "recipe_index", + "revision": "main", + "loader": "aim", + "trust_remote_code": false, + "dtype": "auto", + "device_policy": "gpu_required", + "quantization": "none; served as-is by the AIM image's bundled vLLM", + "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifacts": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-zai-org-glm-5-2:2026.9.0", + "image_digest": "sha256:1a0ad5bda56eb0784085583068d56e1cfc03f4b28875fe968d9477b30b9f5f5b", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], + "manual_alternatives": [], + "chat_template_mode": "auto", + "preferred_engines": [ + "aim" + ], + "warnings": [ + "AIM image from the AMD Enterprise AI catalog; requires Docker" + ] } ] } diff --git a/docs/aim-catalog.md b/docs/aim-catalog.md index c433a0c0a8..cefbc7e4a2 100644 --- a/docs/aim-catalog.md +++ b/docs/aim-catalog.md @@ -31,9 +31,15 @@ not need a rocm-cli maintainer. 1. Watch for AMD Enterprise AI Suite releases. A version bump is the practical signal that AIM images were added or changed. -2. For each AIM entry, confirm the image tag still exists and copy its current - digest into `image_digest` (and `model_id_override` if the tag moved). +2. For each AIM entry, find the newest stable tag + (`oras repo tags docker.io/amdenterpriseai/`) and copy its digest + (`oras resolve docker.io/amdenterpriseai/:`) into `image_digest` (and `model_id_override` if the tag moved). Check `AIM_PROFILE_ID` in `extra_env` is still a valid profile. 3. Set `generated_at_unix_ms` to now (`date +%s%3N`). -4. Add entries for new AIM images the same way as the existing one. +4. Add entries for new AIM images the same way as the existing ones. The list + of images is the AMD Enterprise AI AIM catalog + (). + CPU-only (`aim-epyc-*`) images are left out, since rocm-cli never falls back + to CPU, as are non-chat images (speech, video, bio) until the serve path + supports their tasks. 5. Open a PR; `cargo test -p rocm-core model_recipe` validates the catalog. From 4282f16af140debf8004908fa7ecee75a1f2b747 Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Thu, 29 Oct 2026 14:10:03 +0200 Subject: [PATCH 03/21] ROCMAI-556: use the real model names as AIM canonical ids Name each AIM recipe by its model (e.g. Qwen/Qwen3-VL-8B-Instruct) as the catalog page lists it, with the image as an alias. Models that already have a recipe (Llama-3.2-3B, gemma-4-31B) gain an aim engine record instead of a duplicate recipe. aim-qwen3.5-9b is now Qwen/Qwen3.5-9B and keeps its old id as an alias. Signed-off-by: Juho Vainio --- crates/rocm-core/src/model_catalog.json | 237 ++++++++---------------- crates/rocm-core/src/model_catalog.rs | 2 +- 2 files changed, 74 insertions(+), 165 deletions(-) diff --git a/crates/rocm-core/src/model_catalog.json b/crates/rocm-core/src/model_catalog.json index 2e5795e5e6..3ee05c3751 100644 --- a/crates/rocm-core/src/model_catalog.json +++ b/crates/rocm-core/src/model_catalog.json @@ -330,6 +330,21 @@ "unsupported_combinations": [], "notes": [], "model_id_override": null + }, + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-2-3b-instruct:0.11.1", + "image_digest": "sha256:12e146a90d3ad26bafc102ac0217f5d49b3107c9e7abc476d16ad3f1ae7de4ae", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} } ], "manual_alternatives": [ @@ -429,7 +444,23 @@ "quantization": "BF16, ~62 GiB (1x MI300X)", "artifact_hint": "Hugging Face model id; vLLM ROCm serving validates artifacts during serve", "artifacts": [], - "engine_recipes": [], + "engine_recipes": [ + { + "engine": "aim", + "required_flags": [], + "parser_settings": {}, + "preferred_endpoint": null, + "unsupported_combinations": [], + "notes": [ + "requires a reachable Docker daemon; the image is pulled on first serve", + "datacenter-class image; check the AIM catalog for the GPU memory it needs" + ], + "model_id_override": "amdenterpriseai/aim-google-gemma-4-31b-it:0.13.0-preview", + "image_digest": "sha256:ec2c6940ebd7088f112d09c4c6f554e4d3efaa88b61c89be9f1ecd347beaef9a", + "generated_at_unix_ms": 1793272421113, + "extra_env": {} + } + ], "manual_alternatives": [ "qwen3-32b-fp8" ], @@ -510,8 +541,9 @@ ] }, { - "canonical_model_id": "aim-qwen3.5-9b", + "canonical_model_id": "Qwen/Qwen3.5-9B", "aliases": [ + "aim-qwen3.5-9b", "amdenterpriseai/aim-radeon-qwen-qwen3-5-9b" ], "task": "chat", @@ -555,7 +587,7 @@ ] }, { - "canonical_model_id": "aim-coherelabs-command-a-reasoning-08-2025", + "canonical_model_id": "CohereLabs/command-a-reasoning-08-2025", "aliases": [ "amdenterpriseai/aim-coherelabs-command-a-reasoning-08-2025" ], @@ -596,7 +628,7 @@ ] }, { - "canonical_model_id": "aim-deepseek-ai-deepseek-r1-0528", + "canonical_model_id": "deepseek-ai/DeepSeek-R1-0528", "aliases": [ "amdenterpriseai/aim-deepseek-ai-deepseek-r1-0528" ], @@ -637,7 +669,7 @@ ] }, { - "canonical_model_id": "aim-deepseek-ai-deepseek-r1", + "canonical_model_id": "deepseek-ai/DeepSeek-R1", "aliases": [ "amdenterpriseai/aim-deepseek-ai-deepseek-r1" ], @@ -678,7 +710,7 @@ ] }, { - "canonical_model_id": "aim-deepseek-ai-deepseek-v3-1-terminus", + "canonical_model_id": "deepseek-ai/DeepSeek-V3.1-Terminus", "aliases": [ "amdenterpriseai/aim-deepseek-ai-deepseek-v3-1-terminus" ], @@ -719,7 +751,7 @@ ] }, { - "canonical_model_id": "aim-deepseek-ai-deepseek-v3-1", + "canonical_model_id": "deepseek-ai/DeepSeek-V3.1", "aliases": [ "amdenterpriseai/aim-deepseek-ai-deepseek-v3-1" ], @@ -760,7 +792,7 @@ ] }, { - "canonical_model_id": "aim-deepseek-ai-deepseek-v4-flash-0731", + "canonical_model_id": "deepseek-ai/DeepSeek-V4-Flash-0731", "aliases": [ "amdenterpriseai/aim-deepseek-ai-deepseek-v4-flash-0731" ], @@ -801,7 +833,7 @@ ] }, { - "canonical_model_id": "aim-google-gemma-3-1b-it", + "canonical_model_id": "google/gemma-3-1b-it", "aliases": [ "amdenterpriseai/aim-google-gemma-3-1b-it" ], @@ -842,7 +874,7 @@ ] }, { - "canonical_model_id": "aim-google-gemma-3-27b-it", + "canonical_model_id": "google/gemma-3-27b-it", "aliases": [ "amdenterpriseai/aim-google-gemma-3-27b-it" ], @@ -883,48 +915,7 @@ ] }, { - "canonical_model_id": "aim-google-gemma-4-31b-it", - "aliases": [ - "amdenterpriseai/aim-google-gemma-4-31b-it" - ], - "task": "chat", - "source": "recipe_index", - "revision": "main", - "loader": "aim", - "trust_remote_code": false, - "dtype": "auto", - "device_policy": "gpu_required", - "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", - "artifacts": [], - "engine_recipes": [ - { - "engine": "aim", - "required_flags": [], - "parser_settings": {}, - "preferred_endpoint": null, - "unsupported_combinations": [], - "notes": [ - "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" - ], - "model_id_override": "amdenterpriseai/aim-google-gemma-4-31b-it:0.13.0-preview", - "image_digest": "sha256:ec2c6940ebd7088f112d09c4c6f554e4d3efaa88b61c89be9f1ecd347beaef9a", - "generated_at_unix_ms": 1793272421113, - "extra_env": {} - } - ], - "manual_alternatives": [], - "chat_template_mode": "auto", - "preferred_engines": [ - "aim" - ], - "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" - ] - }, - { - "canonical_model_id": "aim-google-medgemma-27b-it", + "canonical_model_id": "google/medgemma-27b-it", "aliases": [ "amdenterpriseai/aim-google-medgemma-27b-it" ], @@ -965,7 +956,7 @@ ] }, { - "canonical_model_id": "aim-meta-llama-llama-3-1-405b-instruct", + "canonical_model_id": "meta-llama/Llama-3.1-405B-Instruct", "aliases": [ "amdenterpriseai/aim-meta-llama-llama-3-1-405b-instruct" ], @@ -1006,9 +997,9 @@ ] }, { - "canonical_model_id": "aim-meta-llama-llama-3-1-8b-instruct", + "canonical_model_id": "meta-llama/Llama-3.1-8B-Instruct", "aliases": [ - "amdenterpriseai/aim-meta-llama-llama-3-1-8b-instruct" + "amdenterpriseai/aim-radeon-meta-llama-llama-3-1-8b-instruct" ], "task": "chat", "source": "recipe_index", @@ -1029,10 +1020,10 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Radeon-targeted image; if profile auto-selection fails, set AIM_PROFILE_ID with --engine-env" ], - "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-1-8b-instruct:0.11.1", - "image_digest": "sha256:dd386ba89444bf7b16016a622127f58a58bb875d47d528b0f95186cf170c1df7", + "model_id_override": "amdenterpriseai/aim-radeon-meta-llama-llama-3-1-8b-instruct:2026.11.0", + "image_digest": "sha256:a300b53f5f919a59d0cdc4bc177a6db5ef1660719cb50c128be6f8ec4a7c79d5", "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1047,7 +1038,7 @@ ] }, { - "canonical_model_id": "aim-meta-llama-llama-3-2-1b-instruct", + "canonical_model_id": "meta-llama/Llama-3.2-1B-Instruct", "aliases": [ "amdenterpriseai/aim-meta-llama-llama-3-2-1b-instruct" ], @@ -1088,48 +1079,7 @@ ] }, { - "canonical_model_id": "aim-meta-llama-llama-3-2-3b-instruct", - "aliases": [ - "amdenterpriseai/aim-meta-llama-llama-3-2-3b-instruct" - ], - "task": "chat", - "source": "recipe_index", - "revision": "main", - "loader": "aim", - "trust_remote_code": false, - "dtype": "auto", - "device_policy": "gpu_required", - "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", - "artifacts": [], - "engine_recipes": [ - { - "engine": "aim", - "required_flags": [], - "parser_settings": {}, - "preferred_endpoint": null, - "unsupported_combinations": [], - "notes": [ - "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" - ], - "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-2-3b-instruct:0.11.1", - "image_digest": "sha256:12e146a90d3ad26bafc102ac0217f5d49b3107c9e7abc476d16ad3f1ae7de4ae", - "generated_at_unix_ms": 1793272421113, - "extra_env": {} - } - ], - "manual_alternatives": [], - "chat_template_mode": "auto", - "preferred_engines": [ - "aim" - ], - "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" - ] - }, - { - "canonical_model_id": "aim-meta-llama-llama-3-3-70b-instruct", + "canonical_model_id": "meta-llama/Llama-3.3-70B-Instruct", "aliases": [ "amdenterpriseai/aim-meta-llama-llama-3-3-70b-instruct" ], @@ -1170,7 +1120,7 @@ ] }, { - "canonical_model_id": "aim-minimaxai-minimax-m2-5", + "canonical_model_id": "MiniMaxAI/MiniMax-M2.5", "aliases": [ "amdenterpriseai/aim-minimaxai-minimax-m2-5" ], @@ -1211,7 +1161,7 @@ ] }, { - "canonical_model_id": "aim-minimaxai-minimax-m3", + "canonical_model_id": "MiniMaxAI/MiniMax-M3", "aliases": [ "amdenterpriseai/aim-minimaxai-minimax-m3" ], @@ -1252,7 +1202,7 @@ ] }, { - "canonical_model_id": "aim-mistralai-ministral-3-14b-instruct-2512", + "canonical_model_id": "mistralai/Ministral-3-14B-Instruct-2512", "aliases": [ "amdenterpriseai/aim-mistralai-ministral-3-14b-instruct-2512" ], @@ -1293,7 +1243,7 @@ ] }, { - "canonical_model_id": "aim-mistralai-ministral-3-14b-reasoning-2512", + "canonical_model_id": "mistralai/Ministral-3-14B-Reasoning-2512", "aliases": [ "amdenterpriseai/aim-mistralai-ministral-3-14b-reasoning-2512" ], @@ -1334,7 +1284,7 @@ ] }, { - "canonical_model_id": "aim-mistralai-mistral-large-3-675b-instruct-2512", + "canonical_model_id": "mistralai/Mistral-Large-3-675B-Instruct-2512", "aliases": [ "amdenterpriseai/aim-mistralai-mistral-large-3-675b-instruct-2512" ], @@ -1375,7 +1325,7 @@ ] }, { - "canonical_model_id": "aim-mistralai-mistral-small-24b-instruct-2501", + "canonical_model_id": "mistralai/Mistral-Small-24B-Instruct-2501", "aliases": [ "amdenterpriseai/aim-mistralai-mistral-small-24b-instruct-2501" ], @@ -1416,7 +1366,7 @@ ] }, { - "canonical_model_id": "aim-mistralai-mistral-small-3-2-24b-instruct-2506", + "canonical_model_id": "mistralai/Mistral-Small-3.2-24B-Instruct-2506", "aliases": [ "amdenterpriseai/aim-mistralai-mistral-small-3-2-24b-instruct-2506" ], @@ -1457,7 +1407,7 @@ ] }, { - "canonical_model_id": "aim-mistralai-mixtral-8x22b-instruct-v0-1", + "canonical_model_id": "mistralai/Mixtral-8x22B-Instruct-v0.1", "aliases": [ "amdenterpriseai/aim-mistralai-mixtral-8x22b-instruct-v0-1" ], @@ -1498,7 +1448,7 @@ ] }, { - "canonical_model_id": "aim-mistralai-mixtral-8x7b-instruct-v0-1", + "canonical_model_id": "mistralai/Mixtral-8x7B-Instruct-v0.1", "aliases": [ "amdenterpriseai/aim-mistralai-mixtral-8x7b-instruct-v0-1" ], @@ -1539,7 +1489,7 @@ ] }, { - "canonical_model_id": "aim-openai-gpt-oss-120b", + "canonical_model_id": "openai/gpt-oss-120b", "aliases": [ "amdenterpriseai/aim-openai-gpt-oss-120b" ], @@ -1580,7 +1530,7 @@ ] }, { - "canonical_model_id": "aim-openai-gpt-oss-20b", + "canonical_model_id": "openai/gpt-oss-20b", "aliases": [ "amdenterpriseai/aim-openai-gpt-oss-20b" ], @@ -1621,7 +1571,7 @@ ] }, { - "canonical_model_id": "aim-qwen-qwen3-235b-a22b", + "canonical_model_id": "Qwen/Qwen3-235B-A22B", "aliases": [ "amdenterpriseai/aim-qwen-qwen3-235b-a22b" ], @@ -1662,7 +1612,7 @@ ] }, { - "canonical_model_id": "aim-qwen-qwen3-32b", + "canonical_model_id": "Qwen/Qwen3-32B", "aliases": [ "amdenterpriseai/aim-qwen-qwen3-32b" ], @@ -1703,7 +1653,7 @@ ] }, { - "canonical_model_id": "aim-qwen-qwen3-coder-next", + "canonical_model_id": "Qwen/Qwen3-Coder-Next", "aliases": [ "amdenterpriseai/aim-qwen-qwen3-coder-next" ], @@ -1744,7 +1694,7 @@ ] }, { - "canonical_model_id": "aim-qwen-qwen3-vl-235b-a22b-instruct", + "canonical_model_id": "Qwen/Qwen3-VL-235B-A22B-Instruct", "aliases": [ "amdenterpriseai/aim-qwen-qwen3-vl-235b-a22b-instruct" ], @@ -1785,7 +1735,7 @@ ] }, { - "canonical_model_id": "aim-qwen-qwen3-vl-235b-a22b-thinking", + "canonical_model_id": "Qwen/Qwen3-VL-235B-A22B-Thinking", "aliases": [ "amdenterpriseai/aim-qwen-qwen3-vl-235b-a22b-thinking" ], @@ -1826,7 +1776,7 @@ ] }, { - "canonical_model_id": "aim-radeon-google-gemma-3n-e4b-it", + "canonical_model_id": "google/gemma-3n-E4B-it", "aliases": [ "amdenterpriseai/aim-radeon-google-gemma-3n-e4b-it" ], @@ -1867,48 +1817,7 @@ ] }, { - "canonical_model_id": "aim-radeon-meta-llama-llama-3-1-8b-instruct", - "aliases": [ - "amdenterpriseai/aim-radeon-meta-llama-llama-3-1-8b-instruct" - ], - "task": "chat", - "source": "recipe_index", - "revision": "main", - "loader": "aim", - "trust_remote_code": false, - "dtype": "auto", - "device_policy": "gpu_required", - "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", - "artifacts": [], - "engine_recipes": [ - { - "engine": "aim", - "required_flags": [], - "parser_settings": {}, - "preferred_endpoint": null, - "unsupported_combinations": [], - "notes": [ - "requires a reachable Docker daemon; the image is pulled on first serve", - "Radeon-targeted image; if profile auto-selection fails, set AIM_PROFILE_ID with --engine-env" - ], - "model_id_override": "amdenterpriseai/aim-radeon-meta-llama-llama-3-1-8b-instruct:2026.11.0", - "image_digest": "sha256:a300b53f5f919a59d0cdc4bc177a6db5ef1660719cb50c128be6f8ec4a7c79d5", - "generated_at_unix_ms": 1793272421113, - "extra_env": {} - } - ], - "manual_alternatives": [], - "chat_template_mode": "auto", - "preferred_engines": [ - "aim" - ], - "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" - ] - }, - { - "canonical_model_id": "aim-radeon-qwen-qwen3-6-35b-a3b", + "canonical_model_id": "Qwen/Qwen3.6-35B-A3B", "aliases": [ "amdenterpriseai/aim-radeon-qwen-qwen3-6-35b-a3b" ], @@ -1949,7 +1858,7 @@ ] }, { - "canonical_model_id": "aim-radeon-qwen-qwen3-vl-8b-instruct", + "canonical_model_id": "Qwen/Qwen3-VL-8B-Instruct", "aliases": [ "amdenterpriseai/aim-radeon-qwen-qwen3-vl-8b-instruct" ], @@ -1990,7 +1899,7 @@ ] }, { - "canonical_model_id": "aim-radeon-zai-org-glm-4-7-flash", + "canonical_model_id": "zai-org/GLM-4.7-Flash", "aliases": [ "amdenterpriseai/aim-radeon-zai-org-glm-4-7-flash" ], @@ -2031,7 +1940,7 @@ ] }, { - "canonical_model_id": "aim-zai-org-glm-4-7", + "canonical_model_id": "zai-org/GLM-4.7", "aliases": [ "amdenterpriseai/aim-zai-org-glm-4-7" ], @@ -2072,7 +1981,7 @@ ] }, { - "canonical_model_id": "aim-zai-org-glm-5-2", + "canonical_model_id": "zai-org/GLM-5.2", "aliases": [ "amdenterpriseai/aim-zai-org-glm-5-2" ], diff --git a/crates/rocm-core/src/model_catalog.rs b/crates/rocm-core/src/model_catalog.rs index e551584730..84a5068fdb 100644 --- a/crates/rocm-core/src/model_catalog.rs +++ b/crates/rocm-core/src/model_catalog.rs @@ -998,7 +998,7 @@ mod tests { #[test] fn builtin_aim_recipe_carries_a_verification_timestamp() { - let recipe = resolve_builtin_model_recipe("aim-qwen3.5-9b").expect("aim recipe"); + let recipe = resolve_builtin_model_recipe("Qwen/Qwen3.5-9B").expect("aim recipe"); let aim = recipe .engine_recipes .iter() From ce59ace1a5ed846861aa31a85047a46dda63b31d Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Thu, 29 Oct 2026 14:52:50 +0200 Subject: [PATCH 04/21] ROCMAI-556: flag AIM models that need a Hugging Face token Ten catalog images are gated per the AIM catalog's model pages. Their engine recipe notes now say to export HF_TOKEN and pass --engine-env HF_TOKEN. Signed-off-by: Juho Vainio --- crates/rocm-core/src/model_catalog.json | 10 ++++++++ crates/rocm-core/src/model_catalog.rs | 32 +++++++++++++++++++++++++ docs/aim-catalog.md | 3 +++ 3 files changed, 45 insertions(+) diff --git a/crates/rocm-core/src/model_catalog.json b/crates/rocm-core/src/model_catalog.json index 3ee05c3751..9efd07935b 100644 --- a/crates/rocm-core/src/model_catalog.json +++ b/crates/rocm-core/src/model_catalog.json @@ -338,6 +338,7 @@ "preferred_endpoint": null, "unsupported_combinations": [], "notes": [ + "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", "datacenter-class image; check the AIM catalog for the GPU memory it needs" ], @@ -609,6 +610,7 @@ "preferred_endpoint": null, "unsupported_combinations": [], "notes": [ + "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", "datacenter-class image; check the AIM catalog for the GPU memory it needs" ], @@ -855,6 +857,7 @@ "preferred_endpoint": null, "unsupported_combinations": [], "notes": [ + "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", "datacenter-class image; check the AIM catalog for the GPU memory it needs" ], @@ -896,6 +899,7 @@ "preferred_endpoint": null, "unsupported_combinations": [], "notes": [ + "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", "datacenter-class image; check the AIM catalog for the GPU memory it needs" ], @@ -937,6 +941,7 @@ "preferred_endpoint": null, "unsupported_combinations": [], "notes": [ + "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", "datacenter-class image; check the AIM catalog for the GPU memory it needs" ], @@ -978,6 +983,7 @@ "preferred_endpoint": null, "unsupported_combinations": [], "notes": [ + "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", "datacenter-class image; check the AIM catalog for the GPU memory it needs" ], @@ -1019,6 +1025,7 @@ "preferred_endpoint": null, "unsupported_combinations": [], "notes": [ + "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", "Radeon-targeted image; if profile auto-selection fails, set AIM_PROFILE_ID with --engine-env" ], @@ -1060,6 +1067,7 @@ "preferred_endpoint": null, "unsupported_combinations": [], "notes": [ + "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", "datacenter-class image; check the AIM catalog for the GPU memory it needs" ], @@ -1101,6 +1109,7 @@ "preferred_endpoint": null, "unsupported_combinations": [], "notes": [ + "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", "datacenter-class image; check the AIM catalog for the GPU memory it needs" ], @@ -1798,6 +1807,7 @@ "preferred_endpoint": null, "unsupported_combinations": [], "notes": [ + "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", "Radeon-targeted image; if profile auto-selection fails, set AIM_PROFILE_ID with --engine-env" ], diff --git a/crates/rocm-core/src/model_catalog.rs b/crates/rocm-core/src/model_catalog.rs index 84a5068fdb..7db311653a 100644 --- a/crates/rocm-core/src/model_catalog.rs +++ b/crates/rocm-core/src/model_catalog.rs @@ -996,6 +996,38 @@ mod tests { assert_eq!(model_recipe_staleness_warning(&aim, now), None); } + #[test] + fn gated_aim_models_say_to_pass_hf_token() { + // Gated per the AIM catalog's model pages ("requires a Hugging Face + // authentication"); `--engine-env HF_TOKEN` forwarding is covered in + // engines/aim (run_args_never_puts_env_values_on_the_command_line). + for id in [ + "meta-llama/Llama-3.2-3B-Instruct", + "meta-llama/Llama-3.3-70B-Instruct", + "google/gemma-3-27b-it", + ] { + let recipe = resolve_builtin_model_recipe(id).expect("gated aim recipe"); + let aim = recipe + .engine_recipes + .iter() + .find(|e| e.engine == "aim") + .expect("aim engine"); + assert!( + aim.notes + .iter() + .any(|n| n.contains("--engine-env HF_TOKEN")), + "{id}" + ); + } + let open = resolve_builtin_model_recipe("Qwen/Qwen3-32B").expect("ungated aim recipe"); + assert!( + !open.engine_recipes[0] + .notes + .iter() + .any(|n| n.contains("HF_TOKEN")) + ); + } + #[test] fn builtin_aim_recipe_carries_a_verification_timestamp() { let recipe = resolve_builtin_model_recipe("Qwen/Qwen3.5-9B").expect("aim recipe"); diff --git a/docs/aim-catalog.md b/docs/aim-catalog.md index cefbc7e4a2..3987e63663 100644 --- a/docs/aim-catalog.md +++ b/docs/aim-catalog.md @@ -43,3 +43,6 @@ not need a rocm-cli maintainer. to CPU, as are non-chat images (speech, video, bio) until the serve path supports their tasks. 5. Open a PR; `cargo test -p rocm-core model_recipe` validates the catalog. +6. Check each image's model page in the AIM catalog for the line "requires a + Hugging Face authentication". For those, add the `HF_TOKEN` note to the + entry's `notes` (the pattern used by the existing gated entries). From b22835ba30bacb6604e2a84c5de8495bd03fdb6a Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Thu, 29 Oct 2026 15:08:20 +0200 Subject: [PATCH 05/21] ROCMAI-556: list AIM as an engine on the AMD Instinct catalog platform The platform list predates AIM, and the only AIM image at the time was Radeon-targeted. The catalog now carries Instinct-class images. Signed-off-by: Juho Vainio --- apps/rocm/src/examine_report_text.rs | 4 ++-- crates/rocm-core/src/model_catalog.json | 4 ++-- crates/rocm-core/src/model_catalog.rs | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/rocm/src/examine_report_text.rs b/apps/rocm/src/examine_report_text.rs index e886a15af5..9bccd79694 100644 --- a/apps/rocm/src/examine_report_text.rs +++ b/apps/rocm/src/examine_report_text.rs @@ -1896,7 +1896,7 @@ mod tests { // Featured current models are grouped under their hardware target, each // with the quant that fits a single GPU. assert!(rendered.contains("AMD Ryzen AI - Strix Halo (Lemonade / llama.cpp / AIM)")); - assert!(rendered.contains("AMD Instinct - MI300X, MI350X, MI355X (vLLM)")); + assert!(rendered.contains("AMD Instinct - MI300X, MI350X, MI355X (vLLM / AIM)")); // Strix Halo GGUF entries carry the servable owner/repo:variant ref. assert!(rendered.contains("unsloth/Qwen3.6-35B-A3B-GGUF:Q4_K_M")); assert!(rendered.contains("Q4_K_M GGUF")); @@ -1988,7 +1988,7 @@ mod tests { assert!(rendered.contains("source: built-in recipe registry")); // Verbose keeps every recipe (including hidden ones) and annotates each // with its curated catalog placement. - assert!(rendered.contains("catalog: AMD Instinct - MI300X, MI350X, MI355X (vLLM)")); + assert!(rendered.contains("catalog: AMD Instinct - MI300X, MI350X, MI355X (vLLM / AIM)")); assert!(rendered.contains("catalog: hidden (resolvable via rocm serve")); assert!(rendered.contains("sshleifer/tiny-gpt2")); // ROCMAI-543: `rocm model --verbose` output must stick to ASCII too, not diff --git a/crates/rocm-core/src/model_catalog.json b/crates/rocm-core/src/model_catalog.json index 9efd07935b..f720cfaf1e 100644 --- a/crates/rocm-core/src/model_catalog.json +++ b/crates/rocm-core/src/model_catalog.json @@ -14,8 +14,8 @@ "gfx_families": ["gfx110X-all", "gfx120X-all"] }, { - "label": "AMD Instinct - MI300X, MI350X, MI355X (vLLM)", - "engines": ["vllm"], + "label": "AMD Instinct - MI300X, MI350X, MI355X (vLLM / AIM)", + "engines": ["vllm", "aim"], "gfx_families": ["gfx906", "gfx908", "gfx90a", "gfx94X-dcgpu", "gfx950-dcgpu"] } ], diff --git a/crates/rocm-core/src/model_catalog.rs b/crates/rocm-core/src/model_catalog.rs index 7db311653a..16ee56cab7 100644 --- a/crates/rocm-core/src/model_catalog.rs +++ b/crates/rocm-core/src/model_catalog.rs @@ -908,13 +908,13 @@ mod tests { let mi300x = resolve_builtin_model_recipe("qwen3.6-27b").expect("qwen3.6-27b"); assert_eq!( model_recipe_target_platform_label(&mi300x, &platforms), - "AMD Instinct - MI300X, MI350X, MI355X (vLLM)" + "AMD Instinct - MI300X, MI350X, MI355X (vLLM / AIM)" ); // vLLM recipes land on the Instinct platform. let llama = resolve_builtin_model_recipe("llama-3.2-3b-instruct").expect("llama"); assert_eq!( model_recipe_target_platform_label(&llama, &platforms), - "AMD Instinct - MI300X, MI350X, MI355X (vLLM)" + "AMD Instinct - MI300X, MI350X, MI355X (vLLM / AIM)" ); } From 86e80b803f1acf4e25d7aef452baed962499a862 Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Thu, 29 Oct 2026 15:51:45 +0200 Subject: [PATCH 06/21] ROCMAI-556: match AIM images to the GPUs their profiles support Record supported_gfx_families per AIM engine recipe from the image's profile table and pin AIM_PROFILE_ID per gfx family for the Radeon images, whose auto-selection is unreliable. rocm serve refuses a GPU with no profile, rocm model places recipes by the platform covering those families, and Strix Halo no longer lists aim since no profile supports gfx1151. Signed-off-by: Juho Vainio --- apps/rocm/src/examine_report_text.rs | 22 +-- apps/rocm/src/serve_cmd.rs | 144 +++++++++++++++++- crates/rocm-core/src/lib.rs | 16 +- crates/rocm-core/src/model_catalog.json | 133 ++++++++++------ crates/rocm-core/src/model_catalog.rs | 193 +++++++++++++++++++++++- docs/aim-catalog.md | 10 ++ docs/manual-testing.md | 6 +- 7 files changed, 446 insertions(+), 78 deletions(-) diff --git a/apps/rocm/src/examine_report_text.rs b/apps/rocm/src/examine_report_text.rs index 9bccd79694..57125cb218 100644 --- a/apps/rocm/src/examine_report_text.rs +++ b/apps/rocm/src/examine_report_text.rs @@ -561,14 +561,7 @@ pub(crate) fn render_model_registry_text_with_context_and_host( for platform in &platforms { let mut group = visible .iter() - .filter(|recipe| { - recipe.preferred_engines.first().is_some_and(|engine| { - platform - .engines - .iter() - .any(|e| e.eq_ignore_ascii_case(engine)) - }) - }) + .filter(|recipe| rocm_core::model_recipe_targets_platform(recipe, platform)) .peekable(); if group.peek().is_none() { continue; @@ -808,6 +801,15 @@ fn append_model_engine_recipe_settings_lines(output: &mut String, recipe: &Model .join("; ") }; let notes = format_list_or_none(&engine_recipe.notes); + if !engine_recipe.supported_gfx_families.is_empty() { + let _ = writeln!( + output, + " engine_recipe {} supported_gfx_families=[{}] profile_ids=[{}]", + engine_recipe.engine, + engine_recipe.supported_gfx_families.join(","), + format_string_map_or_none(&engine_recipe.profile_ids) + ); + } let _ = writeln!( output, " engine_recipe {} required_flags=[{}] parser_settings=[{}] preferred_endpoint={} unsupported_combinations=[{}] notes=[{}]", @@ -1895,7 +1897,7 @@ mod tests { assert!(rendered.contains("Recommended models - run one with `rocm serve `")); // Featured current models are grouped under their hardware target, each // with the quant that fits a single GPU. - assert!(rendered.contains("AMD Ryzen AI - Strix Halo (Lemonade / llama.cpp / AIM)")); + assert!(rendered.contains("AMD Ryzen AI - Strix Halo (Lemonade / llama.cpp)")); assert!(rendered.contains("AMD Instinct - MI300X, MI350X, MI355X (vLLM / AIM)")); // Strix Halo GGUF entries carry the servable owner/repo:variant ref. assert!(rendered.contains("unsloth/Qwen3.6-35B-A3B-GGUF:Q4_K_M")); @@ -2037,6 +2039,8 @@ mod tests { model_id_override: None, image_digest: None, generated_at_unix_ms: None, + supported_gfx_families: Vec::new(), + profile_ids: BTreeMap::new(), extra_env: BTreeMap::new(), }]; let mut output = String::new(); diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 6c56a3af79..dcc7201037 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -38,8 +38,9 @@ use std::fmt::Write as _; use anyhow::{Context as _, Result, bail}; use rocm_core::{ - AppPaths, ModelRecipeRecord, RocmCliConfig, default_engine_for_platform, - detect_host_gpu_summary, generate_service_id, preferred_serve_engine_for_host_gpu_summary, + AppPaths, GfxSupport, ModelRecipeRecord, RocmCliConfig, default_engine_for_platform, + detect_host_gfx_target, detect_host_gpu_summary, engine_recipe_gfx_support, + generate_service_id, normalize_therock_family, preferred_serve_engine_for_host_gpu_summary, resolve_model_recipe, }; use rocm_engine_protocol::{ @@ -190,6 +191,50 @@ fn render_serve_engine_recipe_lines(engine_recipe: &EngineRecipeHint) -> String output } +/// For AIM, refuses a GPU the catalog has no profile for and pins the catalog's +/// `AIM_PROFILE_ID` for the detected GPU family. `host_family` is a normalized +/// TheRock family, `None` when it could not be detected (nothing is enforced +/// then). Runs before `--engine-env`, so an explicit override still wins. +fn engine_recipe_with_aim_gfx_profile( + engine: &str, + recipe: Option<&ModelRecipeRecord>, + hint: Option, + host_family: impl FnOnce() -> Option, +) -> Result> { + let Some(aim) = recipe + .filter(|_| engine.eq_ignore_ascii_case("aim")) + .and_then(|recipe| { + recipe + .engine_recipes + .iter() + .find(|record| record.engine.eq_ignore_ascii_case("aim")) + .map(|record| (recipe, record)) + }) + .filter(|(_, record)| !record.supported_gfx_families.is_empty()) + else { + return Ok(hint); + }; + let (recipe, record) = aim; + let family = host_family(); + match engine_recipe_gfx_support(record, family.as_deref()) { + GfxSupport::Unrestricted => Ok(hint), + GfxSupport::Unsupported => bail!( + "the AIM image for `{}` has no profile for this GPU ({}); it supports: {}", + recipe.canonical_model_id, + family.as_deref().unwrap_or("unknown"), + record.supported_gfx_families.join(", ") + ), + GfxSupport::Supported(profile) => Ok(match (hint, profile) { + (Some(mut hint), Some(profile)) => { + hint.extra_env + .insert("AIM_PROFILE_ID".to_owned(), profile.to_owned()); + Some(hint) + } + (hint, _) => hint, + }), + } +} + fn protocol_engine_recipe_hint( recipe: &ModelRecipeRecord, engine: &str, @@ -728,6 +773,16 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { .as_ref() .filter(|recipe| model_recipe_supports_engine(recipe, &selected_engine)) .and_then(|recipe| protocol_engine_recipe_hint(recipe, &selected_engine)); + let recipe_hint = engine_recipe_with_aim_gfx_profile( + &selected_engine, + shared_recipe.as_ref(), + recipe_hint, + || { + detect_host_gfx_target() + .as_deref() + .and_then(normalize_therock_family) + }, + )?; // vLLM rejects the TUI chat tab's tool-bearing requests with HTTP 400 unless it // is launched with `--enable-auto-tool-choice`/`--tool-call-parser`. The parser // is model-specific and vLLM does not auto-detect it, so it is never guessed: it @@ -1508,17 +1563,32 @@ mod tests { } #[test] - fn catalog_extra_env_reaches_the_aim_hint_and_engine_env_overrides_it() { + fn catalog_profile_pin_reaches_the_aim_hint_and_engine_env_overrides_it() { let recipe = resolve_builtin_model_recipe("aim-qwen3.5-9b").expect("aim catalog entry"); - let hint = protocol_engine_recipe_hint(&recipe, "aim").expect("aim hint"); + let hint = protocol_engine_recipe_hint(&recipe, "aim"); + let pinned = |family: &'static str| { + engine_recipe_with_aim_gfx_profile("aim", Some(&recipe), hint.clone(), || { + Some(family.to_owned()) + }) + .unwrap() + .unwrap() + }; + let w7900 = pinned("gfx110X-all"); assert_eq!( - hint.extra_env.get("AIM_PROFILE_ID").map(String::as_str), + w7900.extra_env.get("AIM_PROFILE_ID").map(String::as_str), Some("vllm-w7900-bf16-tp1-latency") ); + assert_eq!( + pinned("gfx120X-all") + .extra_env + .get("AIM_PROFILE_ID") + .map(String::as_str), + Some("vllm-r9700-bf16-tp1-latency") + ); let overridden = engine_recipe_with_extra_env( "aim", - Some(hint), + Some(w7900), &[ "AIM_PROFILE_ID=vllm-other".to_owned(), "HF_TOKEN".to_owned(), @@ -1537,6 +1607,64 @@ mod tests { assert_eq!(overridden.extra_env.get("HF_TOKEN"), Some(&String::new())); } + #[test] + fn aim_serve_refuses_a_gpu_the_image_has_no_profile_for() { + let radeon = resolve_builtin_model_recipe("aim-qwen3.5-9b").expect("radeon aim entry"); + let hint = protocol_engine_recipe_hint(&radeon, "aim"); + let err = engine_recipe_with_aim_gfx_profile("aim", Some(&radeon), hint, || { + Some("gfx94X-dcgpu".to_owned()) + }) + .unwrap_err() + .to_string(); + assert!( + err.contains("no profile for this GPU (gfx94X-dcgpu)"), + "{err}" + ); + assert!(err.contains("gfx110X-all, gfx120X-all"), "{err}"); + + // Strix Halo has no AIM profile either. + let hint = protocol_engine_recipe_hint(&radeon, "aim"); + assert!( + engine_recipe_with_aim_gfx_profile("aim", Some(&radeon), hint, || Some( + "gfx1151".to_owned() + )) + .is_err() + ); + + // Instinct images serve on Instinct without pinning a profile (AIM + // auto-selects), and are refused on a Radeon. + let instinct = resolve_builtin_model_recipe("Qwen/Qwen3-32B").expect("instinct aim entry"); + let hint = protocol_engine_recipe_hint(&instinct, "aim"); + let served = + engine_recipe_with_aim_gfx_profile("aim", Some(&instinct), hint.clone(), || { + Some("gfx94X-dcgpu".to_owned()) + }) + .unwrap() + .unwrap(); + assert!(!served.extra_env.contains_key("AIM_PROFILE_ID")); + assert!( + engine_recipe_with_aim_gfx_profile("aim", Some(&instinct), hint, || Some( + "gfx110X-all".to_owned() + )) + .is_err() + ); + } + + #[test] + fn aim_gfx_check_is_skipped_when_the_gpu_is_unknown_or_the_engine_is_not_aim() { + let radeon = resolve_builtin_model_recipe("aim-qwen3.5-9b").expect("radeon aim entry"); + let hint = protocol_engine_recipe_hint(&radeon, "aim"); + let unchanged = + engine_recipe_with_aim_gfx_profile("aim", Some(&radeon), hint.clone(), || None) + .unwrap(); + assert_eq!(unchanged, hint); + let vllm = engine_recipe_with_aim_gfx_profile("vllm", Some(&radeon), None, || { + panic!("host GPU must not be probed for other engines") + }) + .unwrap(); + assert!(vllm.is_none()); + } + #[test] fn explicit_engine_override_keeps_alias_when_shared_recipe_is_for_another_engine() { // `qwen-smoke` is a Lemonade-only GGUF recipe (no vLLM engine recipe). @@ -1618,6 +1746,8 @@ mod tests { model_id_override: None, image_digest: None, generated_at_unix_ms: None, + supported_gfx_families: Vec::new(), + profile_ids: BTreeMap::new(), extra_env: BTreeMap::new(), }, rocm_core::ModelRecipeEngineRecord { @@ -1630,6 +1760,8 @@ mod tests { model_id_override: None, image_digest: None, generated_at_unix_ms: None, + supported_gfx_families: Vec::new(), + profile_ids: BTreeMap::new(), extra_env: BTreeMap::new(), }, ]; diff --git a/crates/rocm-core/src/lib.rs b/crates/rocm-core/src/lib.rs index 606d7f3341..947a80d438 100644 --- a/crates/rocm-core/src/lib.rs +++ b/crates/rocm-core/src/lib.rs @@ -124,17 +124,17 @@ use managed_runtime::{ TheRockFamilyManifest, managed_sdk_tool_path, sort_records_newest_install_first, }; pub use model_catalog::{ - MODEL_RECIPE_STALE_AFTER_MS, ModelArtifactCacheStatus, ModelCatalogPlatform, + GfxSupport, MODEL_RECIPE_STALE_AFTER_MS, ModelArtifactCacheStatus, ModelCatalogPlatform, ModelRecipeArtifactRecord, ModelRecipeArtifactSourcePolicyRecord, ModelRecipeEndpointRecord, ModelRecipeEngineRecord, ModelRecipeIndexDocument, ModelRecipeRecord, ModelRecipeRegistry, ModelRecipeRegistrySource, ModelRecipeUnsupportedCombinationRecord, - builtin_model_recipe_registry, builtin_model_recipes, load_model_recipe_index, - load_model_recipe_registry, load_signed_model_recipe_index, model_artifact_cache_marker_path, - model_artifact_cache_status, model_catalog_platforms, model_recipe_featured, - model_recipe_index_signature_path, model_recipe_staleness_warning, - model_recipe_target_platform_label, normalize_model_ref, platform_matches_gfx_family, - resolve_builtin_model_recipe, resolve_model_recipe, resolve_model_recipe_artifact, - verify_model_recipe_index_signature, + builtin_model_recipe_registry, builtin_model_recipes, engine_recipe_gfx_support, + load_model_recipe_index, load_model_recipe_registry, load_signed_model_recipe_index, + model_artifact_cache_marker_path, model_artifact_cache_status, model_catalog_platforms, + model_recipe_featured, model_recipe_index_signature_path, model_recipe_staleness_warning, + model_recipe_target_platform_label, model_recipe_targets_platform, normalize_model_ref, + platform_matches_gfx_family, resolve_builtin_model_recipe, resolve_model_recipe, + resolve_model_recipe_artifact, verify_model_recipe_index_signature, }; pub use net::{ Backoff, DEFAULT_LOCAL_HOST, DEFAULT_LOCAL_PORT, DOWNLOAD_MAX_ATTEMPTS, DownloadOutcome, diff --git a/crates/rocm-core/src/model_catalog.json b/crates/rocm-core/src/model_catalog.json index f720cfaf1e..788e6e32b5 100644 --- a/crates/rocm-core/src/model_catalog.json +++ b/crates/rocm-core/src/model_catalog.json @@ -4,8 +4,8 @@ "generated_at_unix_ms": null, "platforms": [ { - "label": "AMD Ryzen AI - Strix Halo (Lemonade / llama.cpp / AIM)", - "engines": ["lemonade", "llama.cpp", "llamacpp", "aim"], + "label": "AMD Ryzen AI - Strix Halo (Lemonade / llama.cpp)", + "engines": ["lemonade", "llama.cpp", "llamacpp"], "gfx_families": ["gfx1151"] }, { @@ -340,10 +340,11 @@ "notes": [ "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-2-3b-instruct:0.11.1", "image_digest": "sha256:12e146a90d3ad26bafc102ac0217f5d49b3107c9e7abc476d16ad3f1ae7de4ae", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -454,10 +455,11 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-google-gemma-4-31b-it:0.13.0-preview", "image_digest": "sha256:ec2c6940ebd7088f112d09c4c6f554e4d3efaa88b61c89be9f1ecd347beaef9a", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -567,15 +569,14 @@ "preferred_endpoint": null, "unsupported_combinations": [], "notes": [ - "pins AIM_PROFILE_ID=vllm-w7900-bf16-tp1-latency via extra_env; Radeon profile auto-selection isn't reliable yet, override with --engine-env if needed", + "pins AIM_PROFILE_ID per GPU family from profile_ids because Radeon profile auto-selection isn't reliable yet; override with --engine-env AIM_PROFILE_ID= if needed", "requires a reachable Docker daemon; the image is pulled on first serve" ], "model_id_override": "amdenterpriseai/aim-radeon-qwen-qwen3-5-9b:0.12.0-preview", "image_digest": "sha256:ee1201261f358be881318de5930ce9505d17309486f747a83147d7d042fc95ce", - "generated_at_unix_ms": 1793203564000, - "extra_env": { - "AIM_PROFILE_ID": "vllm-w7900-bf16-tp1-latency" - } + "supported_gfx_families": ["gfx110X-all", "gfx120X-all"], + "profile_ids": {"gfx110X-all": "vllm-w7900-bf16-tp1-latency", "gfx120X-all": "vllm-r9700-bf16-tp1-latency"}, + "generated_at_unix_ms": 1793203564000 } ], "manual_alternatives": [], @@ -584,7 +585,7 @@ "aim" ], "warnings": [ - "Radeon-targeted AIM image; requires Docker and an explicit AIM_PROFILE_ID" + "Radeon-targeted AIM image (W7900 / R9700 profiles only); requires Docker" ] }, { @@ -612,10 +613,11 @@ "notes": [ "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-coherelabs-command-a-reasoning-08-2025:0.11.1", "image_digest": "sha256:6da329641e3ea98e945f43b1df17b7efb3ba0a284dfb123297ace72d4084b4d3", + "supported_gfx_families": ["gfx90a", "gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -653,10 +655,11 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-r1-0528:0.11.1", "image_digest": "sha256:348f6ee92b07bf150b01c7b37adb8365336ffc98a2a6f747ed73754cd41f61bc", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -694,10 +697,11 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-r1:0.11.1", "image_digest": "sha256:4bbea192e4c3fd11865652119c584e9bad27b07634d5d56da2ca1bcf1c74473d", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -735,10 +739,11 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-v3-1-terminus:0.11.1", "image_digest": "sha256:5f0335322a88bd4a8abe0865227287f419b906b47d9913cf8de7a35df80142f4", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -776,10 +781,11 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-v3-1:0.11.1", "image_digest": "sha256:9fc6f3c6e5b165352ed0873d60bc1acfc1aeaebc048da4630d3166e0b498fb33", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -817,10 +823,11 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-v4-flash-0731:2026.11.0", "image_digest": "sha256:bb4e2c8b9ecfcfac886d7c5ebf2629c62ded2debac804d8bf2c11483f7ac6d8c", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -859,10 +866,11 @@ "notes": [ "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-google-gemma-3-1b-it:0.12.0", "image_digest": "sha256:fb49e69fe72ca40efaef97173d1532e09982a141900905015d41267ced1654cf", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -901,10 +909,11 @@ "notes": [ "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-google-gemma-3-27b-it:0.13.0", "image_digest": "sha256:2340abdb1c7cf34a84c5398607b57e1a7cf73b7099ba3fdbde8fbb810e849e33", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -943,10 +952,11 @@ "notes": [ "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-google-medgemma-27b-it:0.13.0", "image_digest": "sha256:a89823487230a8cea1f9d49f881e64f20375ee793739c223ab2c09f7bc30a088", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -985,10 +995,11 @@ "notes": [ "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-1-405b-instruct:0.11.1", "image_digest": "sha256:bfe038184d0f0d8a415ee057f4fe64b984dbb743874d0907448be9ceed432743", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1027,10 +1038,12 @@ "notes": [ "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", - "Radeon-targeted image; if profile auto-selection fails, set AIM_PROFILE_ID with --engine-env" + "Radeon-targeted image; the AIM profile is pinned per GPU family from profile_ids, override with --engine-env AIM_PROFILE_ID=" ], "model_id_override": "amdenterpriseai/aim-radeon-meta-llama-llama-3-1-8b-instruct:2026.11.0", "image_digest": "sha256:a300b53f5f919a59d0cdc4bc177a6db5ef1660719cb50c128be6f8ec4a7c79d5", + "supported_gfx_families": ["gfx110X-all", "gfx120X-all"], + "profile_ids": {"gfx110X-all": "vllm-w7900-bf16-tp1-latency", "gfx120X-all": "vllm-r9700-bf16-tp1-latency"}, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1069,10 +1082,11 @@ "notes": [ "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-2-1b-instruct:0.11.1", "image_digest": "sha256:92e87333a235d5af0494d3148a5dea537da041de671b8b9a9db05bfbd2bfb164", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1111,10 +1125,11 @@ "notes": [ "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-3-70b-instruct:0.11.1", "image_digest": "sha256:c8d1bb99119241592d5749e7e707224154a7acae081558d5a30caefc75c32338", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1152,10 +1167,11 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-minimaxai-minimax-m2-5:0.12.0", "image_digest": "sha256:0a20a6dd756dc3ebc9cc0ada23f95e8edf49fbe33354652702f6d878380323ed", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1193,10 +1209,11 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-minimaxai-minimax-m3:2026.9.0", "image_digest": "sha256:2b7221c5fa3592afefb0e904d10b063126ebe76f71ca7a5b8cd2c2a8a26c10c9", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1234,10 +1251,11 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-mistralai-ministral-3-14b-instruct-2512:0.11.1", "image_digest": "sha256:8fa94cb5351554ca32c15543b48d74b79ef214a29f8066501c841d38798f6504", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1275,10 +1293,11 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-mistralai-ministral-3-14b-reasoning-2512:0.11.1", "image_digest": "sha256:50cb5fc4f2a2a6973f63adcf0f8621d091dd825f67f3808cb6d743ded9d2a00a", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1316,10 +1335,11 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-mistralai-mistral-large-3-675b-instruct-2512:0.10.0", "image_digest": "sha256:6119b9c717b44ff4bec3183318623738d2d95a99ecd85e0fe5bdca98bf24dbdd", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1357,10 +1377,11 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-mistralai-mistral-small-24b-instruct-2501:0.13.0", "image_digest": "sha256:976a1e5a9880756058dad68dd3f543d76f01f1955a436f1bf5b03832704e2b20", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1398,10 +1419,11 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-mistralai-mistral-small-3-2-24b-instruct-2506:0.11.1", "image_digest": "sha256:7883d07cc867795a307073855a716b0c7e623e7f1327a6c347a338f95760f719", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1439,10 +1461,11 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-mistralai-mixtral-8x22b-instruct-v0-1:0.11.1", "image_digest": "sha256:bcbe47ea00037f1d3b203a47c95bcfaa2dc76b5739a94bfdf0d681be1385c8a7", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1480,10 +1503,11 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-mistralai-mixtral-8x7b-instruct-v0-1:0.11.1", "image_digest": "sha256:9f2abb10dc3a21169d7cd0e6f96294df3683dd42ea49f1d5230e62722f4ae738", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1521,10 +1545,11 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-openai-gpt-oss-120b:0.11.1", "image_digest": "sha256:a3052d6230d7a414837903c31be0122f6615819448598e9caea973c084dc3ea7", + "supported_gfx_families": ["gfx90a", "gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1562,10 +1587,11 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-openai-gpt-oss-20b:0.11.1", "image_digest": "sha256:67cee29f3f9036eb3918190c76bc4442dd48254657e680d0256a5018c2164274", + "supported_gfx_families": ["gfx90a", "gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1603,10 +1629,11 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-qwen-qwen3-235b-a22b:0.11.1", "image_digest": "sha256:427d25b93cce97b499f9c1e9268b33eea25a8da8a0f7ef7904f4cdeef5f52f5a", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1644,10 +1671,11 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-qwen-qwen3-32b:0.13.0", "image_digest": "sha256:3e8d47db7c88ca1b7bb210b7bfd5a66f0439e3b768f200ab4aa51fb058d2871c", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1685,10 +1713,11 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-qwen-qwen3-coder-next:0.11.1", "image_digest": "sha256:0874c97e547a59df1893813e3f011c7e5bc1d035b68e99361f4175c0e15010c0", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1726,10 +1755,11 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-qwen-qwen3-vl-235b-a22b-instruct:0.11.1", "image_digest": "sha256:4353ab9fb67d30b54d4c724429c0401490b42aa96045a050faea7b46a9e2d3d6", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1767,10 +1797,11 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-qwen-qwen3-vl-235b-a22b-thinking:0.11.1", "image_digest": "sha256:c10ee1a4189708e101802b4b0b3d292dfc4802ad08c6da70a0f56ae2094eeccb", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1809,10 +1840,12 @@ "notes": [ "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", - "Radeon-targeted image; if profile auto-selection fails, set AIM_PROFILE_ID with --engine-env" + "Radeon-targeted image; the AIM profile is pinned per GPU family from profile_ids, override with --engine-env AIM_PROFILE_ID=" ], "model_id_override": "amdenterpriseai/aim-radeon-google-gemma-3n-e4b-it:0.12.0-preview", "image_digest": "sha256:f5dd9116c6714a8e51e6bdbf94f5a2a8b06db62fc95994e6b331f92931416697", + "supported_gfx_families": ["gfx110X-all", "gfx120X-all"], + "profile_ids": {"gfx110X-all": "vllm-w7900-bf16-tp1-latency", "gfx120X-all": "vllm-r9700-bf16-tp1-latency"}, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1849,11 +1882,14 @@ "preferred_endpoint": null, "unsupported_combinations": [], "notes": [ + "the pinned profiles use tensor parallelism and need 4 GPUs on gfx110X-all / 4 GPUs on gfx120X-all", "requires a reachable Docker daemon; the image is pulled on first serve", - "Radeon-targeted image; if profile auto-selection fails, set AIM_PROFILE_ID with --engine-env" + "Radeon-targeted image; the AIM profile is pinned per GPU family from profile_ids, override with --engine-env AIM_PROFILE_ID=" ], "model_id_override": "amdenterpriseai/aim-radeon-qwen-qwen3-6-35b-a3b:2026.11.0", "image_digest": "sha256:4df926af5c9e792675577cf881fb12b4964b6dac18f6d3f7c1e2cf460834eee2", + "supported_gfx_families": ["gfx110X-all", "gfx120X-all"], + "profile_ids": {"gfx110X-all": "vllm-w7900-bf16-tp4-latency", "gfx120X-all": "vllm-r9700-bf16-tp4-latency"}, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1891,10 +1927,12 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Radeon-targeted image; if profile auto-selection fails, set AIM_PROFILE_ID with --engine-env" + "Radeon-targeted image; the AIM profile is pinned per GPU family from profile_ids, override with --engine-env AIM_PROFILE_ID=" ], "model_id_override": "amdenterpriseai/aim-radeon-qwen-qwen3-vl-8b-instruct:2026.11.0", "image_digest": "sha256:89775113626743d77023e15b595d4218c1e46e89e932b3f3ee5c0350914add55", + "supported_gfx_families": ["gfx110X-all", "gfx120X-all"], + "profile_ids": {"gfx110X-all": "vllm-w7900-bf16-tp1-latency", "gfx120X-all": "vllm-r9700-bf16-tp1-latency"}, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1931,11 +1969,14 @@ "preferred_endpoint": null, "unsupported_combinations": [], "notes": [ + "the pinned profiles use tensor parallelism and need 2 GPUs on gfx110X-all / 4 GPUs on gfx120X-all", "requires a reachable Docker daemon; the image is pulled on first serve", - "Radeon-targeted image; if profile auto-selection fails, set AIM_PROFILE_ID with --engine-env" + "Radeon-targeted image; the AIM profile is pinned per GPU family from profile_ids, override with --engine-env AIM_PROFILE_ID=" ], "model_id_override": "amdenterpriseai/aim-radeon-zai-org-glm-4-7-flash:0.12.0-preview", "image_digest": "sha256:b94130be1f12eaab28bdce274a63b3c10a24b8e292dc91f4c563b7a42246c1ab", + "supported_gfx_families": ["gfx110X-all", "gfx120X-all"], + "profile_ids": {"gfx110X-all": "vllm-w7900-bf16-tp2-latency", "gfx120X-all": "vllm-r9700-bf16-tp4-latency"}, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1973,10 +2014,11 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-zai-org-glm-4-7:0.12.0", "image_digest": "sha256:6555005fe59ad9334ce292ca3df4b619106656a731e7319114c2b3a43a1b1219", + "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -2014,10 +2056,11 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "datacenter-class image; check the AIM catalog for the GPU memory it needs" + "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" ], "model_id_override": "amdenterpriseai/aim-zai-org-glm-5-2:2026.9.0", "image_digest": "sha256:1a0ad5bda56eb0784085583068d56e1cfc03f4b28875fe968d9477b30b9f5f5b", + "supported_gfx_families": ["gfx94X-dcgpu"], "generated_at_unix_ms": 1793272421113, "extra_env": {} } diff --git a/crates/rocm-core/src/model_catalog.rs b/crates/rocm-core/src/model_catalog.rs index 16ee56cab7..39263f27de 100644 --- a/crates/rocm-core/src/model_catalog.rs +++ b/crates/rocm-core/src/model_catalog.rs @@ -97,6 +97,56 @@ pub struct ModelRecipeEngineRecord { /// [`MODEL_RECIPE_STALE_AFTER_MS`], since the pinned image may be gone. #[serde(default)] pub generated_at_unix_ms: Option, + /// TheRock gfx families (for example `gfx94X-dcgpu`) this engine recipe has a + /// profile for. Empty means unrestricted. Serving on a detected GPU outside + /// this list is refused, and it ties the recipe to the platforms covering + /// these families in `rocm model`. Only valid on `aim`. + #[serde(default)] + pub supported_gfx_families: Vec, + /// AIM profile id (`AIM_PROFILE_ID`) to pin per gfx family, for images whose + /// profile auto-selection is unreliable (Radeon). Keys must be listed in + /// `supported_gfx_families`. An explicit `--engine-env AIM_PROFILE_ID=...` + /// wins over this. Only valid on `aim`. + #[serde(default)] + pub profile_ids: BTreeMap, +} + +/// Outcome of matching an engine recipe against the detected GPU family. +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum GfxSupport<'a> { + /// No restriction, or no GPU family known to check against. + Unrestricted, + /// Supported; carries the profile id pinned for this family, if any. + Supported(Option<&'a str>), + /// The recipe lists families and this one is not among them. + Unsupported, +} + +/// Checks `engine_recipe` against `gfx_family` (a normalized TheRock family). +pub fn engine_recipe_gfx_support<'a>( + engine_recipe: &'a ModelRecipeEngineRecord, + gfx_family: Option<&str>, +) -> GfxSupport<'a> { + if engine_recipe.supported_gfx_families.is_empty() { + return GfxSupport::Unrestricted; + } + let Some(family) = gfx_family else { + return GfxSupport::Unrestricted; + }; + if !engine_recipe + .supported_gfx_families + .iter() + .any(|f| f.eq_ignore_ascii_case(family)) + { + return GfxSupport::Unsupported; + } + GfxSupport::Supported( + engine_recipe + .profile_ids + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(family)) + .map(|(_, id)| id.as_str()), + ) } /// How long an engine recipe's `generated_at_unix_ms` stays trusted (180 days). @@ -734,6 +784,39 @@ fn validate_model_recipe_engine_record( ); } } + let known = crate::known_therock_families(); + for family in &engine_recipe.supported_gfx_families { + if !known.contains(&family.as_str()) { + bail!( + "engine recipe for `{}` on `{canonical_model_id}` lists unknown gfx family `{family}`", + engine_recipe.engine + ); + } + } + for (family, profile) in &engine_recipe.profile_ids { + require_nonempty(profile, "engine profile id")?; + if !engine_recipe.supported_gfx_families.contains(family) { + bail!( + "engine recipe profile_ids for `{family}` on `{canonical_model_id}` must also \ + be listed in supported_gfx_families" + ); + } + if engine_recipe.extra_env.contains_key("AIM_PROFILE_ID") { + bail!( + "engine recipe on `{canonical_model_id}` sets both profile_ids and \ + extra_env AIM_PROFILE_ID" + ); + } + } + if (!engine_recipe.supported_gfx_families.is_empty() || !engine_recipe.profile_ids.is_empty()) + && !engine_recipe.engine.eq_ignore_ascii_case("aim") + { + bail!( + "engine recipe for `{}` on `{canonical_model_id}` sets supported_gfx_families or \ + profile_ids, which only apply to aim", + engine_recipe.engine + ); + } if engine_recipe.required_flags.is_empty() && engine_recipe.parser_settings.is_empty() && engine_recipe.preferred_endpoint.is_none() @@ -742,6 +825,7 @@ fn validate_model_recipe_engine_record( && engine_recipe.model_id_override.is_none() && engine_recipe.image_digest.is_none() && engine_recipe.extra_env.is_empty() + && engine_recipe.supported_gfx_families.is_empty() { bail!( "engine recipe for `{}` on `{canonical_model_id}` must not be empty", @@ -789,15 +873,47 @@ pub fn model_recipe_target_platform_label( recipe: &ModelRecipeRecord, platforms: &[ModelCatalogPlatform], ) -> String { - let engine = recipe - .preferred_engines - .first() - .map(|e| e.trim().to_ascii_lowercase()) - .unwrap_or_default(); platforms .iter() - .find(|p| p.engines.iter().any(|e| e.eq_ignore_ascii_case(&engine))) - .map_or_else(|| engine.clone(), |p| p.label.clone()) + .find(|p| model_recipe_targets_platform(recipe, p)) + .map_or_else( + || { + recipe + .preferred_engines + .first() + .map(|e| e.trim().to_ascii_lowercase()) + .unwrap_or_default() + }, + |p| p.label.clone(), + ) +} + +/// Whether `recipe` belongs under `platform`: its first preferred engine runs +/// there and, when that engine's recipe restricts gfx families, the platform +/// covers at least one of them. +pub fn model_recipe_targets_platform( + recipe: &ModelRecipeRecord, + platform: &ModelCatalogPlatform, +) -> bool { + let Some(engine) = recipe.preferred_engines.first() else { + return false; + }; + if !platform + .engines + .iter() + .any(|e| e.eq_ignore_ascii_case(engine)) + { + return false; + } + let families = recipe + .engine_recipes + .iter() + .find(|r| r.engine.eq_ignore_ascii_case(engine)) + .map_or(&[][..], |r| r.supported_gfx_families.as_slice()); + families.is_empty() + || families + .iter() + .any(|family| platform_matches_gfx_family(platform, family)) } /// Whether the given normalized TheRock family matches a platform's gfx targets. @@ -903,7 +1019,7 @@ mod tests { let strix = resolve_builtin_model_recipe("qwen").expect("qwen assistant"); assert_eq!( model_recipe_target_platform_label(&strix, &platforms), - "AMD Ryzen AI - Strix Halo (Lemonade / llama.cpp / AIM)" + "AMD Ryzen AI - Strix Halo (Lemonade / llama.cpp)" ); let mi300x = resolve_builtin_model_recipe("qwen3.6-27b").expect("qwen3.6-27b"); assert_eq!( @@ -1028,6 +1144,63 @@ mod tests { ); } + #[test] + fn aim_recipes_are_matched_to_the_platforms_their_profiles_support() { + let registry = builtin_model_recipe_registry(); + let platforms = model_catalog_platforms(®istry); + let label = |id: &str| { + model_recipe_target_platform_label( + &resolve_builtin_model_recipe(id).expect("recipe"), + &platforms, + ) + }; + assert!(label("Qwen/Qwen3.5-9B").starts_with("AMD Radeon")); + assert!(label("Qwen/Qwen3-32B").starts_with("AMD Instinct")); + let strix = platforms + .iter() + .find(|p| p.label.contains("Strix Halo")) + .expect("strix halo platform"); + assert!(!strix.engines.iter().any(|e| e == "aim")); + for recipe in ®istry.recipes { + for aim in recipe.engine_recipes.iter().filter(|e| e.engine == "aim") { + assert!( + !aim.supported_gfx_families.is_empty(), + "{} aim entry must say which GPUs its profiles cover", + recipe.canonical_model_id + ); + } + } + } + + #[test] + fn gfx_support_pins_profiles_and_rejects_other_families() { + let aim = ModelRecipeEngineRecord { + engine: "aim".to_owned(), + supported_gfx_families: vec!["gfx110X-all".to_owned()], + profile_ids: BTreeMap::from([("gfx110X-all".to_owned(), "p".to_owned())]), + ..Default::default() + }; + assert_eq!( + engine_recipe_gfx_support(&aim, Some("gfx110X-all")), + GfxSupport::Supported(Some("p")) + ); + assert_eq!( + engine_recipe_gfx_support(&aim, Some("gfx1151")), + GfxSupport::Unsupported + ); + assert_eq!( + engine_recipe_gfx_support(&aim, None), + GfxSupport::Unrestricted + ); + let mut bad = aim.clone(); + bad.profile_ids + .insert("gfx120X-all".to_owned(), "q".to_owned()); + assert!(validate_model_recipe_engine_record(&bad, "m").is_err()); + let mut wrong_engine = aim; + wrong_engine.engine = "vllm".to_owned(); + assert!(validate_model_recipe_engine_record(&wrong_engine, "m").is_err()); + } + #[test] fn builtin_aim_recipe_carries_a_verification_timestamp() { let recipe = resolve_builtin_model_recipe("Qwen/Qwen3.5-9B").expect("aim recipe"); @@ -1066,6 +1239,8 @@ mod tests { model_id_override: None, image_digest: Some(digest.to_owned()), generated_at_unix_ms: None, + supported_gfx_families: Vec::new(), + profile_ids: BTreeMap::new(), }); ModelRecipeIndexDocument { schema_version: 1, @@ -1120,6 +1295,8 @@ mod tests { model_id_override: None, image_digest: None, generated_at_unix_ms: None, + supported_gfx_families: Vec::new(), + profile_ids: BTreeMap::new(), }); let index = ModelRecipeIndexDocument { schema_version: 1, diff --git a/docs/aim-catalog.md b/docs/aim-catalog.md index 3987e63663..5f27f16390 100644 --- a/docs/aim-catalog.md +++ b/docs/aim-catalog.md @@ -46,3 +46,13 @@ not need a rocm-cli maintainer. 6. Check each image's model page in the AIM catalog for the line "requires a Hugging Face authentication". For those, add the `HF_TOKEN` note to the entry's `notes` (the pattern used by the existing gated entries). +7. Record which GPUs the image has profiles for. From the model page's profile + table, set `supported_gfx_families` on the entry (MI250X `gfx90a`; MI300X and + MI325X `gfx94X-dcgpu`; MI350X and MI355X `gfx950-dcgpu`; W7900 `gfx110X-all`; + R9700 `gfx120X-all`). `rocm serve` refuses a GPU outside that list, and + `rocm model` groups the recipe under the platform covering those families. + Instinct images auto-select a profile, so leave `AIM_PROFILE_ID` alone. Radeon + images do not select reliably: add `profile_ids` mapping each family to a + profile (the lowest-GPU-count optimized latency profile, noting in `notes` if + it needs more than one GPU). An explicit `--engine-env AIM_PROFILE_ID=...` + still wins. diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 48b3c2d948..e3a30912d0 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -954,8 +954,10 @@ the engine's default 5-minute readiness wait. Raise it with environment of `rocm serve`. Stopping or failing a start removes the container and its downloaded weights, so a retry downloads them again. -`aim-qwen3.5-9b` is the catalog entry (`rocm model --verbose`). It supplies the -pinned image and `AIM_PROFILE_ID`; to serve a raw image reference instead, pass +`aim-qwen3.5-9b` (canonical id `Qwen/Qwen3.5-9B`) is the catalog entry +(`rocm model --verbose`). It supplies the pinned image and, for the detected GPU +family, `AIM_PROFILE_ID`; on a GPU the image has no profile for (for example +Strix Halo or an Instinct card) `rocm serve` refuses with the supported families. To serve a raw image reference instead, pass the profile yourself: ```bash From 8c9407a2738e0f84f7bd434049f8070c275c46c1 Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Thu, 29 Oct 2026 16:18:30 +0200 Subject: [PATCH 07/21] ROCMAI-556: enforce HF tokens, warn on other boards, estimate VRAM for AIM entries Gated AIM images carry requires_hf_token and rocm serve refuses them before launch without HF_TOKEN. Radeon entries record supported_boards and serve warns when the detected GPU is none of them. 32 AIM entries get an estimated min_gpu_mem_gb. Signed-off-by: Juho Vainio --- apps/rocm/src/examine_report_text.rs | 2 + apps/rocm/src/serve_cmd.rs | 172 +++++++++++++++++++++++- crates/rocm-core/src/model_catalog.json | 48 +++++++ crates/rocm-core/src/model_catalog.rs | 38 +++++- docs/aim-catalog.md | 16 ++- docs/manual-testing.md | 4 +- 6 files changed, 269 insertions(+), 11 deletions(-) diff --git a/apps/rocm/src/examine_report_text.rs b/apps/rocm/src/examine_report_text.rs index 57125cb218..19f3c11389 100644 --- a/apps/rocm/src/examine_report_text.rs +++ b/apps/rocm/src/examine_report_text.rs @@ -2041,6 +2041,8 @@ mod tests { generated_at_unix_ms: None, supported_gfx_families: Vec::new(), profile_ids: BTreeMap::new(), + requires_hf_token: false, + supported_boards: Vec::new(), extra_env: BTreeMap::new(), }]; let mut output = String::new(); diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index dcc7201037..5773bb61a0 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -235,6 +235,79 @@ fn engine_recipe_with_aim_gfx_profile( } } +fn aim_engine_record<'a>( + engine: &str, + recipe: Option<&'a ModelRecipeRecord>, +) -> Option<&'a rocm_core::ModelRecipeEngineRecord> { + recipe + .filter(|_| engine.eq_ignore_ascii_case("aim"))? + .engine_recipes + .iter() + .find(|record| record.engine.eq_ignore_ascii_case("aim")) +} + +/// Refuses to launch a gated AIM image before the container is pulled when no +/// Hugging Face token will reach it: `--engine-env HF_TOKEN=`, or the bare +/// `--engine-env HF_TOKEN` with `HF_TOKEN` exported (the form that is never +/// stored). Runs after `--engine-env` is applied to the hint. +fn ensure_aim_hf_token( + engine: &str, + recipe: Option<&ModelRecipeRecord>, + hint: Option<&EngineRecipeHint>, + env: impl Fn(&str) -> Option, +) -> Result<()> { + if !aim_engine_record(engine, recipe).is_some_and(|record| record.requires_hf_token) { + return Ok(()); + } + let supplied = hint + .and_then(|hint| hint.extra_env.get("HF_TOKEN")) + .is_some_and(|value| { + !value.trim().is_empty() || env("HF_TOKEN").is_some_and(|v| !v.trim().is_empty()) + }); + if supplied { + return Ok(()); + } + let model = recipe.map_or("", |r| r.canonical_model_id.as_str()); + bail!( + "`{model}` is a gated model and needs a Hugging Face token. Export it and pass the \ + bare name so it is forwarded and never stored: `export HF_TOKEN=`, then \ + `rocm serve {model} --engine aim --engine-env HF_TOKEN`" + ) +} + +/// Note shown when the pinned AIM profile targets boards other than the detected +/// GPU (a gfx family can cover cards the profile was not built for). +fn aim_board_note( + engine: &str, + recipe: Option<&ModelRecipeRecord>, + gpu_name: Option<&str>, +) -> Option { + let record = aim_engine_record(engine, recipe)?; + let name = gpu_name?; + if record.profile_ids.is_empty() || record.supported_boards.is_empty() { + return None; + } + let squash = |value: &str| { + value + .chars() + .filter(char::is_ascii_alphanumeric) + .collect::() + .to_ascii_lowercase() + }; + let name_key = squash(name); + if record + .supported_boards + .iter() + .any(|board| name_key.contains(&squash(board))) + { + return None; + } + Some(format!( + "the pinned AIM profile targets {}; `{name}` is not one of them, so it may not run well. Override with `--engine-env AIM_PROFILE_ID=`", + record.supported_boards.join(" / ") + )) +} + fn protocol_engine_recipe_hint( recipe: &ModelRecipeRecord, engine: &str, @@ -823,7 +896,31 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { generation_defaults, )?; let engine_recipe = engine_recipe_with_extra_env(&selected_engine, engine_recipe, &engine_env)?; - let engine_env_note = engine_env_ignored_note(&selected_engine, &engine_env); + ensure_aim_hf_token( + &selected_engine, + shared_recipe.as_ref(), + engine_recipe.as_ref(), + |key| std::env::var(key).ok(), + )?; + let aim_gpu_name = aim_engine_record(&selected_engine, shared_recipe.as_ref()) + .filter(|record| !record.profile_ids.is_empty()) + .and_then(|_| { + host_gpu_summary + .clone() + .unwrap_or_else(|| detect_host_gpu_summary(Some(&paths))) + .name + }); + let engine_env_notes: Vec = [ + engine_env_ignored_note(&selected_engine, &engine_env), + aim_board_note( + &selected_engine, + shared_recipe.as_ref(), + aim_gpu_name.as_deref(), + ), + ] + .into_iter() + .flatten() + .collect(); let tool_call_note = if tool_call_parser.is_some() && !engine_serves_vllm { Some(format!( "note: --tool-call-parser applies only to vLLM; ignored for engine '{selected_engine}'" @@ -1233,7 +1330,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { if let Some(note) = &gpu_memory_utilization_note { println!(" note: {note}"); } - if let Some(note) = &engine_env_note { + for note in &engine_env_notes { println!(" note: {note}"); } } @@ -1316,9 +1413,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { host_gpu_summary.as_ref(), engine_serves_vllm, ); - if let Some(note) = &engine_env_note { - notes.push(note.clone()); - } + notes.extend(engine_env_notes.iter().cloned()); let summary = serve_summary::DeploymentSummary { engine: selected_engine.clone(), requested_model: model, @@ -1650,6 +1745,69 @@ mod tests { ); } + #[test] + fn gated_aim_model_needs_an_hf_token_before_launch() { + let gated = resolve_builtin_model_recipe("meta-llama/Llama-3.2-3B-Instruct") + .expect("gated aim entry"); + let hint = |engine_env: &[&str]| { + engine_recipe_with_extra_env( + "aim", + protocol_engine_recipe_hint(&gated, "aim"), + &engine_env + .iter() + .map(|s| (*s).to_owned()) + .collect::>(), + ) + .unwrap() + }; + let no_env = |_: &str| None; + let exported = |_: &str| Some("hf_x".to_owned()); + + // Without the flag, or with the bare flag but nothing exported, refuse and + // name the command that clears it. + for env in [&no_env as &dyn Fn(&str) -> Option, &exported] { + let err = ensure_aim_hf_token("aim", Some(&gated), hint(&[]).as_ref(), env) + .unwrap_err() + .to_string(); + assert!(err.contains("--engine-env HF_TOKEN"), "{err}"); + } + assert!( + ensure_aim_hf_token("aim", Some(&gated), hint(&["HF_TOKEN"]).as_ref(), no_env).is_err() + ); + // The advised form (bare flag with the variable exported) or an explicit + // value clears it, and other engines are never checked. + assert!( + ensure_aim_hf_token("aim", Some(&gated), hint(&["HF_TOKEN"]).as_ref(), exported) + .is_ok() + ); + assert!( + ensure_aim_hf_token( + "aim", + Some(&gated), + hint(&["HF_TOKEN=hf_y"]).as_ref(), + no_env + ) + .is_ok() + ); + assert!(ensure_aim_hf_token("vllm", Some(&gated), None, no_env).is_ok()); + let open = resolve_builtin_model_recipe("Qwen/Qwen3-32B").expect("ungated aim entry"); + assert!(ensure_aim_hf_token("aim", Some(&open), None, no_env).is_ok()); + } + + #[test] + fn aim_board_note_only_fires_for_a_pinned_profile_on_another_board() { + let radeon = resolve_builtin_model_recipe("aim-qwen3.5-9b").expect("radeon aim entry"); + assert!(aim_board_note("aim", Some(&radeon), Some("AMD Radeon PRO W7900")).is_none()); + assert!(aim_board_note("aim", Some(&radeon), Some("AMD Radeon AI PRO R9700")).is_none()); + let note = aim_board_note("aim", Some(&radeon), Some("AMD Radeon RX 7900 XTX")) + .expect("other board warns"); + assert!(note.contains("W7900 / R9700") && note.contains("--engine-env AIM_PROFILE_ID")); + assert!(aim_board_note("aim", Some(&radeon), None).is_none()); + // Instinct images pin no profile, so there is nothing to mismatch. + let instinct = resolve_builtin_model_recipe("Qwen/Qwen3-32B").expect("instinct entry"); + assert!(aim_board_note("aim", Some(&instinct), Some("anything")).is_none()); + } + #[test] fn aim_gfx_check_is_skipped_when_the_gpu_is_unknown_or_the_engine_is_not_aim() { let radeon = resolve_builtin_model_recipe("aim-qwen3.5-9b").expect("radeon aim entry"); @@ -1748,6 +1906,8 @@ mod tests { generated_at_unix_ms: None, supported_gfx_families: Vec::new(), profile_ids: BTreeMap::new(), + requires_hf_token: false, + supported_boards: Vec::new(), extra_env: BTreeMap::new(), }, rocm_core::ModelRecipeEngineRecord { @@ -1762,6 +1922,8 @@ mod tests { generated_at_unix_ms: None, supported_gfx_families: Vec::new(), profile_ids: BTreeMap::new(), + requires_hf_token: false, + supported_boards: Vec::new(), extra_env: BTreeMap::new(), }, ]; diff --git a/crates/rocm-core/src/model_catalog.json b/crates/rocm-core/src/model_catalog.json index 788e6e32b5..4b628e1f19 100644 --- a/crates/rocm-core/src/model_catalog.json +++ b/crates/rocm-core/src/model_catalog.json @@ -345,6 +345,7 @@ "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-2-3b-instruct:0.11.1", "image_digest": "sha256:12e146a90d3ad26bafc102ac0217f5d49b3107c9e7abc476d16ad3f1ae7de4ae", "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "requires_hf_token": true, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -575,6 +576,7 @@ "model_id_override": "amdenterpriseai/aim-radeon-qwen-qwen3-5-9b:0.12.0-preview", "image_digest": "sha256:ee1201261f358be881318de5930ce9505d17309486f747a83147d7d042fc95ce", "supported_gfx_families": ["gfx110X-all", "gfx120X-all"], + "supported_boards": ["W7900", "R9700"], "profile_ids": {"gfx110X-all": "vllm-w7900-bf16-tp1-latency", "gfx120X-all": "vllm-r9700-bf16-tp1-latency"}, "generated_at_unix_ms": 1793203564000 } @@ -600,6 +602,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 267, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -618,6 +621,7 @@ "model_id_override": "amdenterpriseai/aim-coherelabs-command-a-reasoning-08-2025:0.11.1", "image_digest": "sha256:6da329641e3ea98e945f43b1df17b7efb3ba0a284dfb123297ace72d4084b4d3", "supported_gfx_families": ["gfx90a", "gfx94X-dcgpu", "gfx950-dcgpu"], + "requires_hf_token": true, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -643,6 +647,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 806, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -685,6 +690,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 806, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -727,6 +733,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 806, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -769,6 +776,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 806, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -853,6 +861,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 3, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -871,6 +880,7 @@ "model_id_override": "amdenterpriseai/aim-google-gemma-3-1b-it:0.12.0", "image_digest": "sha256:fb49e69fe72ca40efaef97173d1532e09982a141900905015d41267ced1654cf", "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "requires_hf_token": true, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -896,6 +906,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 65, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -914,6 +925,7 @@ "model_id_override": "amdenterpriseai/aim-google-gemma-3-27b-it:0.13.0", "image_digest": "sha256:2340abdb1c7cf34a84c5398607b57e1a7cf73b7099ba3fdbde8fbb810e849e33", "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "requires_hf_token": true, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -939,6 +951,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 65, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -957,6 +970,7 @@ "model_id_override": "amdenterpriseai/aim-google-medgemma-27b-it:0.13.0", "image_digest": "sha256:a89823487230a8cea1f9d49f881e64f20375ee793739c223ab2c09f7bc30a088", "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "requires_hf_token": true, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -982,6 +996,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 486, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1000,6 +1015,7 @@ "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-1-405b-instruct:0.11.1", "image_digest": "sha256:bfe038184d0f0d8a415ee057f4fe64b984dbb743874d0907448be9ceed432743", "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "requires_hf_token": true, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1025,6 +1041,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 20, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1043,6 +1060,8 @@ "model_id_override": "amdenterpriseai/aim-radeon-meta-llama-llama-3-1-8b-instruct:2026.11.0", "image_digest": "sha256:a300b53f5f919a59d0cdc4bc177a6db5ef1660719cb50c128be6f8ec4a7c79d5", "supported_gfx_families": ["gfx110X-all", "gfx120X-all"], + "supported_boards": ["W7900", "R9700"], + "requires_hf_token": true, "profile_ids": {"gfx110X-all": "vllm-w7900-bf16-tp1-latency", "gfx120X-all": "vllm-r9700-bf16-tp1-latency"}, "generated_at_unix_ms": 1793272421113, "extra_env": {} @@ -1069,6 +1088,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 3, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1087,6 +1107,7 @@ "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-2-1b-instruct:0.11.1", "image_digest": "sha256:92e87333a235d5af0494d3148a5dea537da041de671b8b9a9db05bfbd2bfb164", "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "requires_hf_token": true, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1112,6 +1133,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 85, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1130,6 +1152,7 @@ "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-3-70b-instruct:0.11.1", "image_digest": "sha256:c8d1bb99119241592d5749e7e707224154a7acae081558d5a30caefc75c32338", "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "requires_hf_token": true, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1155,6 +1178,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 276, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1239,6 +1263,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 17, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1281,6 +1306,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 17, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1323,6 +1349,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 810, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1365,6 +1392,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 58, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1407,6 +1435,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 58, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1449,6 +1478,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 339, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1491,6 +1521,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 113, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1533,6 +1564,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 281, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1575,6 +1607,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 51, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1617,6 +1650,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 282, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1659,6 +1693,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 40, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1701,6 +1736,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 96, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1743,6 +1779,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 282, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1785,6 +1822,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 282, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1827,6 +1865,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 20, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1845,6 +1884,8 @@ "model_id_override": "amdenterpriseai/aim-radeon-google-gemma-3n-e4b-it:0.12.0-preview", "image_digest": "sha256:f5dd9116c6714a8e51e6bdbf94f5a2a8b06db62fc95994e6b331f92931416697", "supported_gfx_families": ["gfx110X-all", "gfx120X-all"], + "supported_boards": ["W7900", "R9700"], + "requires_hf_token": true, "profile_ids": {"gfx110X-all": "vllm-w7900-bf16-tp1-latency", "gfx120X-all": "vllm-r9700-bf16-tp1-latency"}, "generated_at_unix_ms": 1793272421113, "extra_env": {} @@ -1871,6 +1912,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 84, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1889,6 +1931,7 @@ "model_id_override": "amdenterpriseai/aim-radeon-qwen-qwen3-6-35b-a3b:2026.11.0", "image_digest": "sha256:4df926af5c9e792675577cf881fb12b4964b6dac18f6d3f7c1e2cf460834eee2", "supported_gfx_families": ["gfx110X-all", "gfx120X-all"], + "supported_boards": ["W7900", "R9700"], "profile_ids": {"gfx110X-all": "vllm-w7900-bf16-tp4-latency", "gfx120X-all": "vllm-r9700-bf16-tp4-latency"}, "generated_at_unix_ms": 1793272421113, "extra_env": {} @@ -1915,6 +1958,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 22, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1932,6 +1976,7 @@ "model_id_override": "amdenterpriseai/aim-radeon-qwen-qwen3-vl-8b-instruct:2026.11.0", "image_digest": "sha256:89775113626743d77023e15b595d4218c1e46e89e932b3f3ee5c0350914add55", "supported_gfx_families": ["gfx110X-all", "gfx120X-all"], + "supported_boards": ["W7900", "R9700"], "profile_ids": {"gfx110X-all": "vllm-w7900-bf16-tp1-latency", "gfx120X-all": "vllm-r9700-bf16-tp1-latency"}, "generated_at_unix_ms": 1793272421113, "extra_env": {} @@ -1958,6 +2003,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 72, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1976,6 +2022,7 @@ "model_id_override": "amdenterpriseai/aim-radeon-zai-org-glm-4-7-flash:0.12.0-preview", "image_digest": "sha256:b94130be1f12eaab28bdce274a63b3c10a24b8e292dc91f4c563b7a42246c1ab", "supported_gfx_families": ["gfx110X-all", "gfx120X-all"], + "supported_boards": ["W7900", "R9700"], "profile_ids": {"gfx110X-all": "vllm-w7900-bf16-tp2-latency", "gfx120X-all": "vllm-r9700-bf16-tp4-latency"}, "generated_at_unix_ms": 1793272421113, "extra_env": {} @@ -2002,6 +2049,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", + "min_gpu_mem_gb": 426, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], diff --git a/crates/rocm-core/src/model_catalog.rs b/crates/rocm-core/src/model_catalog.rs index 39263f27de..79d411cddd 100644 --- a/crates/rocm-core/src/model_catalog.rs +++ b/crates/rocm-core/src/model_catalog.rs @@ -109,6 +109,17 @@ pub struct ModelRecipeEngineRecord { /// wins over this. Only valid on `aim`. #[serde(default)] pub profile_ids: BTreeMap, + /// The image needs a Hugging Face token (a gated model). `rocm serve` refuses + /// to start it until `HF_TOKEN` is passed with `--engine-env`. Only valid on + /// `aim`. + #[serde(default)] + pub requires_hf_token: bool, + /// Board names (for example `W7900`) the image's pinned profiles were built + /// for. `rocm serve` warns when the detected GPU name matches none of them, + /// since a gfx family can cover boards the profile does not suit. Only valid + /// on `aim`, and only checked when `profile_ids` pins a profile. + #[serde(default)] + pub supported_boards: Vec, } /// Outcome of matching an engine recipe against the detected GPU family. @@ -808,12 +819,15 @@ fn validate_model_recipe_engine_record( ); } } - if (!engine_recipe.supported_gfx_families.is_empty() || !engine_recipe.profile_ids.is_empty()) + if (!engine_recipe.supported_gfx_families.is_empty() + || !engine_recipe.profile_ids.is_empty() + || engine_recipe.requires_hf_token + || !engine_recipe.supported_boards.is_empty()) && !engine_recipe.engine.eq_ignore_ascii_case("aim") { bail!( - "engine recipe for `{}` on `{canonical_model_id}` sets supported_gfx_families or \ - profile_ids, which only apply to aim", + "engine recipe for `{}` on `{canonical_model_id}` sets supported_gfx_families, \ + profile_ids, requires_hf_token or supported_boards, which only apply to aim", engine_recipe.engine ); } @@ -1128,6 +1142,7 @@ mod tests { .iter() .find(|e| e.engine == "aim") .expect("aim engine"); + assert!(aim.requires_hf_token, "{id}"); assert!( aim.notes .iter() @@ -1135,6 +1150,19 @@ mod tests { "{id}" ); } + // The flag and the note never drift apart across the whole catalog. + for recipe in builtin_model_recipes() { + for aim in recipe.engine_recipes.iter().filter(|e| e.engine == "aim") { + assert_eq!( + aim.requires_hf_token, + aim.notes + .iter() + .any(|n| n.contains("--engine-env HF_TOKEN")), + "{}", + recipe.canonical_model_id + ); + } + } let open = resolve_builtin_model_recipe("Qwen/Qwen3-32B").expect("ungated aim recipe"); assert!( !open.engine_recipes[0] @@ -1241,6 +1269,8 @@ mod tests { generated_at_unix_ms: None, supported_gfx_families: Vec::new(), profile_ids: BTreeMap::new(), + requires_hf_token: false, + supported_boards: Vec::new(), }); ModelRecipeIndexDocument { schema_version: 1, @@ -1297,6 +1327,8 @@ mod tests { generated_at_unix_ms: None, supported_gfx_families: Vec::new(), profile_ids: BTreeMap::new(), + requires_hf_token: false, + supported_boards: Vec::new(), }); let index = ModelRecipeIndexDocument { schema_version: 1, diff --git a/docs/aim-catalog.md b/docs/aim-catalog.md index 5f27f16390..4d6eab43be 100644 --- a/docs/aim-catalog.md +++ b/docs/aim-catalog.md @@ -44,8 +44,10 @@ not need a rocm-cli maintainer. supports their tasks. 5. Open a PR; `cargo test -p rocm-core model_recipe` validates the catalog. 6. Check each image's model page in the AIM catalog for the line "requires a - Hugging Face authentication". For those, add the `HF_TOKEN` note to the - entry's `notes` (the pattern used by the existing gated entries). + Hugging Face authentication". For those, set `requires_hf_token: true` and + add the `HF_TOKEN` note to the entry's `notes` (the pattern used by the + existing gated entries). `rocm serve` then refuses to start the image until + `HF_TOKEN` is passed with `--engine-env`. 7. Record which GPUs the image has profiles for. From the model page's profile table, set `supported_gfx_families` on the entry (MI250X `gfx90a`; MI300X and MI325X `gfx94X-dcgpu`; MI350X and MI355X `gfx950-dcgpu`; W7900 `gfx110X-all`; @@ -56,3 +58,13 @@ not need a rocm-cli maintainer. profile (the lowest-GPU-count optimized latency profile, noting in `notes` if it needs more than one GPU). An explicit `--engine-env AIM_PROFILE_ID=...` still wins. +8. On Radeon entries, set `supported_boards` to the board names in the profile + table (for example `W7900`, `R9700`). `rocm serve` warns, without refusing, + when the detected GPU name matches none of them, because a gfx family can + cover boards the pinned profile was not built for. +9. Set `min_gpu_mem_gb`. The AIM docs publish GPU counts but no VRAM, so the + values in the catalog are estimates: total parameters times 1 byte (2 when + the image has no fp8 profile) plus 20% headroom, rounded up. Entries whose + parameter count is not published (DeepSeek-V4-Flash, MiniMax-M3, GLM-5.2) are + left unset rather than guessed. Replace an estimate with a measured figure + when one is available. diff --git a/docs/manual-testing.md b/docs/manual-testing.md index e3a30912d0..8ea43bfaaf 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -968,7 +968,9 @@ rocm serve amdenterpriseai/aim-radeon-qwen-qwen3-5-9b:0.12.0-preview --engine ai `--engine-env KEY=VALUE` is stored with the service record, so use it for non-secret values only. For a secret such as a gated model's `HF_TOKEN`, export it and pass the bare name (`export HF_TOKEN=...`, then `--engine-env HF_TOKEN`): -it is forwarded from your environment and never stored. `--engine-env` is +it is forwarded from your environment and never stored. A gated model +(`rocm model --verbose` shows `requires_hf_token` in its notes) is refused before +anything is pulled if no token was passed. `--engine-env` is ignored by the other engines (the command prints a note saying so). Expected result: From fd75c3953924e85901435700612ef8e1c86531c5 Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Thu, 29 Oct 2026 17:43:39 +0200 Subject: [PATCH 08/21] ROCMAI-556: refuse multi-GPU AIM images before launch Record min_gpu_counts per gfx family from each image's profile table. rocm serve refuses an image that needs several GPUs when --gpu pins one device, too few GPUs are usable, or serve would select a single GPU, instead of letting the container fail inside the image. Signed-off-by: Juho Vainio --- apps/rocm/src/examine_report_text.rs | 12 ++- apps/rocm/src/serve_cmd.rs | 129 +++++++++++++++++++++++- crates/rocm-core/src/model_catalog.json | 19 ++++ crates/rocm-core/src/model_catalog.rs | 21 ++++ docs/aim-catalog.md | 6 ++ 5 files changed, 180 insertions(+), 7 deletions(-) diff --git a/apps/rocm/src/examine_report_text.rs b/apps/rocm/src/examine_report_text.rs index 19f3c11389..a7ad72e44b 100644 --- a/apps/rocm/src/examine_report_text.rs +++ b/apps/rocm/src/examine_report_text.rs @@ -804,10 +804,17 @@ fn append_model_engine_recipe_settings_lines(output: &mut String, recipe: &Model if !engine_recipe.supported_gfx_families.is_empty() { let _ = writeln!( output, - " engine_recipe {} supported_gfx_families=[{}] profile_ids=[{}]", + " engine_recipe {} supported_gfx_families=[{}] profile_ids=[{}] min_gpu_counts=[{}]", engine_recipe.engine, engine_recipe.supported_gfx_families.join(","), - format_string_map_or_none(&engine_recipe.profile_ids) + format_string_map_or_none(&engine_recipe.profile_ids), + format_string_map_or_none( + &engine_recipe + .min_gpu_counts + .iter() + .map(|(family, count)| (family.clone(), count.to_string())) + .collect() + ) ); } let _ = writeln!( @@ -2043,6 +2050,7 @@ mod tests { profile_ids: BTreeMap::new(), requires_hf_token: false, supported_boards: Vec::new(), + min_gpu_counts: BTreeMap::new(), extra_env: BTreeMap::new(), }]; let mut output = String::new(); diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 5773bb61a0..0364189765 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -275,6 +275,57 @@ fn ensure_aim_hf_token( ) } +/// GPUs an AIM image needs on `family`, when more than one (see +/// `min_gpu_counts`). `None` when it needs one, or the family is unknown. +fn aim_min_gpu_count( + engine: &str, + recipe: Option<&ModelRecipeRecord>, + family: Option<&str>, +) -> Option { + let family = family?; + aim_engine_record(engine, recipe)? + .min_gpu_counts + .iter() + .find(|(key, _)| key.eq_ignore_ascii_case(family)) + .map(|(_, count)| *count) +} + +/// Refuses an AIM image that needs `needed` GPUs when `--gpu` pins one device, +/// fewer usable GPUs exist, or serve selected fewer devices than it needs, +/// before anything is pulled: the container would start with too few devices and +/// fail inside the image. `usable` is `None` when the count could not be probed, +/// and `selected` is `None` before GPU selection or when none was made (the +/// container then sees every GPU); neither is enforced then. +fn ensure_aim_gpu_count( + model: &str, + needed: u32, + pinned_single_gpu: bool, + usable: Option, + selected: Option, +) -> Result<()> { + let needed_usize = needed as usize; + if pinned_single_gpu { + bail!( + "`{model}` needs {needed} GPUs on this hardware, but `--gpu ` pins one device; \ + drop `--gpu` (`rocm serve` does not support multi-GPU serving yet, so this model \ + cannot be started here)" + ); + } + if let Some(usable) = usable.filter(|usable| *usable < needed_usize) { + bail!( + "`{model}` needs {needed} GPUs on this hardware, but only {usable} usable {} found", + if usable == 1 { "GPU was" } else { "GPUs were" } + ); + } + if selected.is_some_and(|selected| selected > 0 && selected < needed_usize) { + bail!( + "`{model}` needs {needed} GPUs on this hardware, but `rocm serve` selects a single \ + GPU and does not support multi-GPU serving yet, so this model cannot be started here" + ); + } + Ok(()) +} + /// Note shown when the pinned AIM profile targets boards other than the detected /// GPU (a gfx family can cover cards the profile was not built for). fn aim_board_note( @@ -846,16 +897,24 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { .as_ref() .filter(|recipe| model_recipe_supports_engine(recipe, &selected_engine)) .and_then(|recipe| protocol_engine_recipe_hint(recipe, &selected_engine)); + let aim_host_family = aim_engine_record(&selected_engine, shared_recipe.as_ref()) + .filter(|record| !record.supported_gfx_families.is_empty()) + .and_then(|_| { + detect_host_gfx_target() + .as_deref() + .and_then(normalize_therock_family) + }); let recipe_hint = engine_recipe_with_aim_gfx_profile( &selected_engine, shared_recipe.as_ref(), recipe_hint, - || { - detect_host_gfx_target() - .as_deref() - .and_then(normalize_therock_family) - }, + || aim_host_family.clone(), )?; + let aim_min_gpus = aim_min_gpu_count( + &selected_engine, + shared_recipe.as_ref(), + aim_host_family.as_deref(), + ); // vLLM rejects the TUI chat tab's tool-bearing requests with HTTP 400 unless it // is launched with `--enable-auto-tool-choice`/`--tool-call-parser`. The parser // is model-specific and vLLM does not auto-detect it, so it is never guessed: it @@ -1172,6 +1231,15 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { } else { None }; + if let (Some(needed), false) = (aim_min_gpus, cpu_only) { + ensure_aim_gpu_count( + &model, + needed, + matches!(gpu_selection, GpuSelection::Index(_)), + visible_gpu_indices.as_ref().map(Vec::len), + None, + )?; + } // Resolve which runtime/env this serve will use, now that the host GPU // pre-condition and `--gpu` have both been checked (or, on the reuse path, // the pre-gate already resolved it under the exemption). This must not move back @@ -1251,6 +1319,9 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { visible_gpu_indices.as_deref(), gpu_vram.as_deref(), )?; + if let (Some(needed), false) = (aim_min_gpus, cpu_only) { + ensure_aim_gpu_count(&model, needed, false, None, Some(gpu_indices.len()))?; + } let service_id = generate_service_id(&selected_engine, &resolve.canonical_model_id); // Attached foreground streaming is the debugging path, selected by `--verbose` @@ -1794,6 +1865,52 @@ mod tests { assert!(ensure_aim_hf_token("aim", Some(&open), None, no_env).is_ok()); } + #[test] + fn multi_gpu_aim_image_is_refused_before_launch() { + let id = "deepseek-ai/DeepSeek-R1"; + let r1 = resolve_builtin_model_recipe(id).expect("multi-gpu aim entry"); + assert_eq!( + aim_min_gpu_count("aim", Some(&r1), Some("gfx94X-dcgpu")), + Some(8) + ); + assert_eq!(aim_min_gpu_count("aim", Some(&r1), Some("gfx1151")), None); + assert_eq!(aim_min_gpu_count("aim", Some(&r1), None), None); + assert_eq!( + aim_min_gpu_count("vllm", Some(&r1), Some("gfx94X-dcgpu")), + None + ); + // A single-GPU image has no requirement. + let small = resolve_builtin_model_recipe("Qwen/Qwen3-32B").expect("single-gpu entry"); + assert_eq!( + aim_min_gpu_count("aim", Some(&small), Some("gfx94X-dcgpu")), + None + ); + // Radeon needs differ per family. + let flash = resolve_builtin_model_recipe("zai-org/GLM-4.7-Flash").expect("flash entry"); + assert_eq!( + aim_min_gpu_count("aim", Some(&flash), Some("gfx110X-all")), + Some(2) + ); + assert_eq!( + aim_min_gpu_count("aim", Some(&flash), Some("gfx120X-all")), + Some(4) + ); + + let err = |pinned, usable, selected| { + ensure_aim_gpu_count(id, 8, pinned, usable, selected) + .unwrap_err() + .to_string() + }; + assert!(err(true, Some(8), None).contains("`--gpu ` pins one device")); + assert!(err(false, Some(2), None).contains("only 2 usable GPUs were found")); + assert!(err(false, Some(1), None).contains("only 1 usable GPU was found")); + assert!(err(false, Some(8), Some(1)).contains("does not support multi-GPU serving yet")); + // Unknown counts, or no selection made (container sees every GPU), pass. + assert!(ensure_aim_gpu_count(id, 8, false, None, None).is_ok()); + assert!(ensure_aim_gpu_count(id, 8, false, Some(8), Some(0)).is_ok()); + assert!(ensure_aim_gpu_count(id, 8, false, Some(8), Some(8)).is_ok()); + } + #[test] fn aim_board_note_only_fires_for_a_pinned_profile_on_another_board() { let radeon = resolve_builtin_model_recipe("aim-qwen3.5-9b").expect("radeon aim entry"); @@ -1908,6 +2025,7 @@ mod tests { profile_ids: BTreeMap::new(), requires_hf_token: false, supported_boards: Vec::new(), + min_gpu_counts: BTreeMap::new(), extra_env: BTreeMap::new(), }, rocm_core::ModelRecipeEngineRecord { @@ -1924,6 +2042,7 @@ mod tests { profile_ids: BTreeMap::new(), requires_hf_token: false, supported_boards: Vec::new(), + min_gpu_counts: BTreeMap::new(), extra_env: BTreeMap::new(), }, ]; diff --git a/crates/rocm-core/src/model_catalog.json b/crates/rocm-core/src/model_catalog.json index 4b628e1f19..1b167a7022 100644 --- a/crates/rocm-core/src/model_catalog.json +++ b/crates/rocm-core/src/model_catalog.json @@ -621,6 +621,7 @@ "model_id_override": "amdenterpriseai/aim-coherelabs-command-a-reasoning-08-2025:0.11.1", "image_digest": "sha256:6da329641e3ea98e945f43b1df17b7efb3ba0a284dfb123297ace72d4084b4d3", "supported_gfx_families": ["gfx90a", "gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx90a": 8, "gfx94X-dcgpu": 2}, "requires_hf_token": true, "generated_at_unix_ms": 1793272421113, "extra_env": {} @@ -665,6 +666,7 @@ "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-r1-0528:0.11.1", "image_digest": "sha256:348f6ee92b07bf150b01c7b37adb8365336ffc98a2a6f747ed73754cd41f61bc", "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 8, "gfx950-dcgpu": 8}, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -708,6 +710,7 @@ "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-r1:0.11.1", "image_digest": "sha256:4bbea192e4c3fd11865652119c584e9bad27b07634d5d56da2ca1bcf1c74473d", "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 8, "gfx950-dcgpu": 8}, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -751,6 +754,7 @@ "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-v3-1-terminus:0.11.1", "image_digest": "sha256:5f0335322a88bd4a8abe0865227287f419b906b47d9913cf8de7a35df80142f4", "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 8, "gfx950-dcgpu": 8}, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -794,6 +798,7 @@ "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-v3-1:0.11.1", "image_digest": "sha256:9fc6f3c6e5b165352ed0873d60bc1acfc1aeaebc048da4630d3166e0b498fb33", "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 8, "gfx950-dcgpu": 8}, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -836,6 +841,7 @@ "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-v4-flash-0731:2026.11.0", "image_digest": "sha256:bb4e2c8b9ecfcfac886d7c5ebf2629c62ded2debac804d8bf2c11483f7ac6d8c", "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 2, "gfx950-dcgpu": 2}, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1015,6 +1021,7 @@ "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-1-405b-instruct:0.11.1", "image_digest": "sha256:bfe038184d0f0d8a415ee057f4fe64b984dbb743874d0907448be9ceed432743", "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 4}, "requires_hf_token": true, "generated_at_unix_ms": 1793272421113, "extra_env": {} @@ -1196,6 +1203,7 @@ "model_id_override": "amdenterpriseai/aim-minimaxai-minimax-m2-5:0.12.0", "image_digest": "sha256:0a20a6dd756dc3ebc9cc0ada23f95e8edf49fbe33354652702f6d878380323ed", "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 2}, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1238,6 +1246,7 @@ "model_id_override": "amdenterpriseai/aim-minimaxai-minimax-m3:2026.9.0", "image_digest": "sha256:2b7221c5fa3592afefb0e904d10b063126ebe76f71ca7a5b8cd2c2a8a26c10c9", "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 4, "gfx950-dcgpu": 4}, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1367,6 +1376,7 @@ "model_id_override": "amdenterpriseai/aim-mistralai-mistral-large-3-675b-instruct-2512:0.10.0", "image_digest": "sha256:6119b9c717b44ff4bec3183318623738d2d95a99ecd85e0fe5bdca98bf24dbdd", "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 8, "gfx950-dcgpu": 8}, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1496,6 +1506,7 @@ "model_id_override": "amdenterpriseai/aim-mistralai-mixtral-8x22b-instruct-v0-1:0.11.1", "image_digest": "sha256:bcbe47ea00037f1d3b203a47c95bcfaa2dc76b5739a94bfdf0d681be1385c8a7", "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 4, "gfx950-dcgpu": 4}, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1582,6 +1593,7 @@ "model_id_override": "amdenterpriseai/aim-openai-gpt-oss-120b:0.11.1", "image_digest": "sha256:a3052d6230d7a414837903c31be0122f6615819448598e9caea973c084dc3ea7", "supported_gfx_families": ["gfx90a", "gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx90a": 2}, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1668,6 +1680,7 @@ "model_id_override": "amdenterpriseai/aim-qwen-qwen3-235b-a22b:0.11.1", "image_digest": "sha256:427d25b93cce97b499f9c1e9268b33eea25a8da8a0f7ef7904f4cdeef5f52f5a", "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 2}, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1797,6 +1810,7 @@ "model_id_override": "amdenterpriseai/aim-qwen-qwen3-vl-235b-a22b-instruct:0.11.1", "image_digest": "sha256:4353ab9fb67d30b54d4c724429c0401490b42aa96045a050faea7b46a9e2d3d6", "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 2}, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1840,6 +1854,7 @@ "model_id_override": "amdenterpriseai/aim-qwen-qwen3-vl-235b-a22b-thinking:0.11.1", "image_digest": "sha256:c10ee1a4189708e101802b4b0b3d292dfc4802ad08c6da70a0f56ae2094eeccb", "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 2}, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -1931,6 +1946,7 @@ "model_id_override": "amdenterpriseai/aim-radeon-qwen-qwen3-6-35b-a3b:2026.11.0", "image_digest": "sha256:4df926af5c9e792675577cf881fb12b4964b6dac18f6d3f7c1e2cf460834eee2", "supported_gfx_families": ["gfx110X-all", "gfx120X-all"], + "min_gpu_counts": {"gfx110X-all": 4, "gfx120X-all": 4}, "supported_boards": ["W7900", "R9700"], "profile_ids": {"gfx110X-all": "vllm-w7900-bf16-tp4-latency", "gfx120X-all": "vllm-r9700-bf16-tp4-latency"}, "generated_at_unix_ms": 1793272421113, @@ -2022,6 +2038,7 @@ "model_id_override": "amdenterpriseai/aim-radeon-zai-org-glm-4-7-flash:0.12.0-preview", "image_digest": "sha256:b94130be1f12eaab28bdce274a63b3c10a24b8e292dc91f4c563b7a42246c1ab", "supported_gfx_families": ["gfx110X-all", "gfx120X-all"], + "min_gpu_counts": {"gfx110X-all": 2, "gfx120X-all": 4}, "supported_boards": ["W7900", "R9700"], "profile_ids": {"gfx110X-all": "vllm-w7900-bf16-tp2-latency", "gfx120X-all": "vllm-r9700-bf16-tp4-latency"}, "generated_at_unix_ms": 1793272421113, @@ -2067,6 +2084,7 @@ "model_id_override": "amdenterpriseai/aim-zai-org-glm-4-7:0.12.0", "image_digest": "sha256:6555005fe59ad9334ce292ca3df4b619106656a731e7319114c2b3a43a1b1219", "supported_gfx_families": ["gfx94X-dcgpu", "gfx950-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 2}, "generated_at_unix_ms": 1793272421113, "extra_env": {} } @@ -2109,6 +2127,7 @@ "model_id_override": "amdenterpriseai/aim-zai-org-glm-5-2:2026.9.0", "image_digest": "sha256:1a0ad5bda56eb0784085583068d56e1cfc03f4b28875fe968d9477b30b9f5f5b", "supported_gfx_families": ["gfx94X-dcgpu"], + "min_gpu_counts": {"gfx94X-dcgpu": 8}, "generated_at_unix_ms": 1793272421113, "extra_env": {} } diff --git a/crates/rocm-core/src/model_catalog.rs b/crates/rocm-core/src/model_catalog.rs index 79d411cddd..1ed84aef50 100644 --- a/crates/rocm-core/src/model_catalog.rs +++ b/crates/rocm-core/src/model_catalog.rs @@ -120,6 +120,12 @@ pub struct ModelRecipeEngineRecord { /// on `aim`, and only checked when `profile_ids` pins a profile. #[serde(default)] pub supported_boards: Vec, + /// Minimum number of GPUs the image needs per gfx family (the smallest + /// tensor-parallel size across that family's profiles). A family left out + /// needs one. Keys must be listed in `supported_gfx_families`. `rocm serve` + /// refuses to start with fewer usable GPUs. Only valid on `aim`. + #[serde(default)] + pub min_gpu_counts: BTreeMap, } /// Outcome of matching an engine recipe against the detected GPU family. @@ -819,7 +825,16 @@ fn validate_model_recipe_engine_record( ); } } + for (family, count) in &engine_recipe.min_gpu_counts { + if !engine_recipe.supported_gfx_families.contains(family) || *count < 2 { + bail!( + "engine recipe min_gpu_counts for `{family}` on `{canonical_model_id}` must be \ + a listed supported_gfx_families entry with a count of at least 2" + ); + } + } if (!engine_recipe.supported_gfx_families.is_empty() + || !engine_recipe.min_gpu_counts.is_empty() || !engine_recipe.profile_ids.is_empty() || engine_recipe.requires_hf_token || !engine_recipe.supported_boards.is_empty()) @@ -1224,6 +1239,10 @@ mod tests { bad.profile_ids .insert("gfx120X-all".to_owned(), "q".to_owned()); assert!(validate_model_recipe_engine_record(&bad, "m").is_err()); + let mut bad_count = bad.clone(); + bad_count.profile_ids.clear(); + bad_count.min_gpu_counts.insert("gfx110X-all".to_owned(), 1); + assert!(validate_model_recipe_engine_record(&bad_count, "m").is_err()); let mut wrong_engine = aim; wrong_engine.engine = "vllm".to_owned(); assert!(validate_model_recipe_engine_record(&wrong_engine, "m").is_err()); @@ -1271,6 +1290,7 @@ mod tests { profile_ids: BTreeMap::new(), requires_hf_token: false, supported_boards: Vec::new(), + min_gpu_counts: BTreeMap::new(), }); ModelRecipeIndexDocument { schema_version: 1, @@ -1329,6 +1349,7 @@ mod tests { profile_ids: BTreeMap::new(), requires_hf_token: false, supported_boards: Vec::new(), + min_gpu_counts: BTreeMap::new(), }); let index = ModelRecipeIndexDocument { schema_version: 1, diff --git a/docs/aim-catalog.md b/docs/aim-catalog.md index 4d6eab43be..61f1608cd7 100644 --- a/docs/aim-catalog.md +++ b/docs/aim-catalog.md @@ -68,3 +68,9 @@ not need a rocm-cli maintainer. parameter count is not published (DeepSeek-V4-Flash, MiniMax-M3, GLM-5.2) are left unset rather than guessed. Replace an estimate with a measured figure when one is available. +10. Set `min_gpu_counts` for any family whose smallest profile uses more than one + GPU (the smallest tensor-parallel size in that family's profiles; leave the + family out when a single-GPU profile exists). `rocm serve` refuses such an + image up front when `--gpu` pins one device or too few GPUs are usable. + `rocm serve` does not yet select several GPUs, so these images cannot be + started on a host where it would pick one; that is tracked as ROCMAI-751. From 29946513dec36fd56da160ca334b2d18209b0784 Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Fri, 30 Oct 2026 13:53:38 +0200 Subject: [PATCH 09/21] ROCMAI-556: address review feedback on AIM catalog routing, docs link and GPU checks - keep vllm as a second preferred engine on AIM-catalogued HF ids so Instinct hosts still select vLLM - refuse multi-GPU AIM images at the first check and drop the late second check - skip the AIM gfx-family check for --device cpu_only - cap/unset min_gpu_mem_gb estimates that contradicted min_gpu_counts - extend the engine-recipe empty check and the aim-only error to the new fields - point end users at updating rocm-cli in the staleness warning - scope README/docs to rocm model --verbose; use a bare path for the Sphinx include - test the staleness rendering helper and select_serve_engine routing Signed-off-by: Juho Vainio --- README.md | 4 +- apps/rocm/src/examine_report_text.rs | 39 +++++++-- apps/rocm/src/serve_cmd.rs | 110 ++++++++++------------- crates/rocm-core/src/model_catalog.json | 112 +++++++++++++++--------- crates/rocm-core/src/model_catalog.rs | 10 ++- docs/aim-catalog.md | 18 ++-- docs/manual-testing.md | 2 +- 7 files changed, 173 insertions(+), 122 deletions(-) diff --git a/README.md b/README.md index 0b9f33ea52..f993791f54 100644 --- a/README.md +++ b/README.md @@ -822,8 +822,8 @@ rocm model [--verbose] the hardware they target. Pass `--verbose` to list every recipe, including ones hidden from the default list, with details such as aliases, data type, minimum GPU memory, and preferred engines. AIM entries are hand-maintained and record -when they were last verified; `rocm model` warns when one is over 180 days old -(see [docs/aim-catalog.md](docs/aim-catalog.md)). +when they were last verified; `rocm model --verbose` warns when one is over 180 +days old (see `docs/aim-catalog.md`). ### Model serving diff --git a/apps/rocm/src/examine_report_text.rs b/apps/rocm/src/examine_report_text.rs index a7ad72e44b..018a52cc33 100644 --- a/apps/rocm/src/examine_report_text.rs +++ b/apps/rocm/src/examine_report_text.rs @@ -693,19 +693,24 @@ pub(crate) fn render_model_registry_verbose_text_with_context_and_host( for warning in &recipe.warnings { let _ = writeln!(output, " warning: {warning}"); } - for engine_recipe in &recipe.engine_recipes { - if let Some(warning) = rocm_core::model_recipe_staleness_warning( - engine_recipe, - rocm_core::unix_time_millis(), - ) { - let _ = writeln!(output, " warning: {warning}"); - } - } + append_model_recipe_staleness_lines(&mut output, recipe, rocm_core::unix_time_millis()); } append_model_recipe_registry_source(&mut output, ®istry); output } +fn append_model_recipe_staleness_lines( + output: &mut String, + recipe: &ModelRecipeRecord, + now_ms: u128, +) { + for engine_recipe in &recipe.engine_recipes { + if let Some(warning) = rocm_core::model_recipe_staleness_warning(engine_recipe, now_ms) { + let _ = writeln!(output, " warning: {warning}"); + } + } +} + /// The `rocm model` header. For the default built-in list it just names the /// action; a configured recipe index instead advertises its provenance (the /// only case where the source differs from the default and is worth surfacing). @@ -2113,6 +2118,24 @@ mod tests { assert!(rendered.contains("tiny-gpt2 (2 GiB min GPU)")); } + #[test] + fn model_recipe_staleness_lines_warn_only_for_stale_engine_recipes() { + let recipe = rocm_core::resolve_builtin_model_recipe("Qwen/Qwen3-32B").expect("aim entry"); + let verified = recipe.engine_recipes[0] + .generated_at_unix_ms + .expect("aim entries record when they were verified"); + let stale_ms = rocm_core::MODEL_RECIPE_STALE_AFTER_MS; + let mut fresh = String::new(); + append_model_recipe_staleness_lines(&mut fresh, &recipe, verified + stale_ms); + assert!(fresh.is_empty(), "{fresh}"); + let mut stale = String::new(); + append_model_recipe_staleness_lines(&mut stale, &recipe, verified + stale_ms + 1); + assert!( + stale.contains("warning: aim entry was last verified 180 days ago"), + "{stale}" + ); + } + #[test] fn render_model_registry_text_marks_tiny_recipe_as_gpu_smoke_support() { let rendered = render_model_registry_verbose_text_with_context_and_host(None, None, None); diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 0364189765..4c40e485c8 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -290,40 +290,14 @@ fn aim_min_gpu_count( .map(|(_, count)| *count) } -/// Refuses an AIM image that needs `needed` GPUs when `--gpu` pins one device, -/// fewer usable GPUs exist, or serve selected fewer devices than it needs, -/// before anything is pulled: the container would start with too few devices and -/// fail inside the image. `usable` is `None` when the count could not be probed, -/// and `selected` is `None` before GPU selection or when none was made (the -/// container then sees every GPU); neither is enforced then. -fn ensure_aim_gpu_count( - model: &str, - needed: u32, - pinned_single_gpu: bool, - usable: Option, - selected: Option, -) -> Result<()> { - let needed_usize = needed as usize; - if pinned_single_gpu { - bail!( - "`{model}` needs {needed} GPUs on this hardware, but `--gpu ` pins one device; \ - drop `--gpu` (`rocm serve` does not support multi-GPU serving yet, so this model \ - cannot be started here)" - ); - } - if let Some(usable) = usable.filter(|usable| *usable < needed_usize) { - bail!( - "`{model}` needs {needed} GPUs on this hardware, but only {usable} usable {} found", - if usable == 1 { "GPU was" } else { "GPUs were" } - ); - } - if selected.is_some_and(|selected| selected > 0 && selected < needed_usize) { - bail!( - "`{model}` needs {needed} GPUs on this hardware, but `rocm serve` selects a single \ - GPU and does not support multi-GPU serving yet, so this model cannot be started here" - ); - } - Ok(()) +/// Refuses an AIM image that needs `needed` GPUs before anything is pulled: +/// `rocm serve` selects a single GPU, so the container would start with too few +/// devices and fail inside the image. +fn ensure_aim_multi_gpu_supported(model: &str, needed: u32) -> Result<()> { + bail!( + "`{model}` needs {needed} GPUs on this hardware, but `rocm serve` does not support \ + multi-GPU serving yet, so this model cannot be started here" + ) } /// Note shown when the pinned AIM profile targets boards other than the detected @@ -897,8 +871,12 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { .as_ref() .filter(|recipe| model_recipe_supports_engine(recipe, &selected_engine)) .and_then(|recipe| protocol_engine_recipe_hint(recipe, &selected_engine)); + let device_policy = parse_device_policy(device.as_ref().map(|policy| policy.as_policy_str()))?; + let cpu_only = matches!(device_policy, DevicePolicy::CpuOnly); + // The AIM GPU checks are about the GPU the container would use, so they do + // not apply to `--device cpu_only`. let aim_host_family = aim_engine_record(&selected_engine, shared_recipe.as_ref()) - .filter(|record| !record.supported_gfx_families.is_empty()) + .filter(|record| !record.supported_gfx_families.is_empty() && !cpu_only) .and_then(|_| { detect_host_gfx_target() .as_deref() @@ -915,6 +893,9 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { shared_recipe.as_ref(), aim_host_family.as_deref(), ); + if let Some(needed) = aim_min_gpus { + ensure_aim_multi_gpu_supported(&model, needed)?; + } // vLLM rejects the TUI chat tab's tool-bearing requests with HTTP 400 unless it // is launched with `--enable-auto-tool-choice`/`--tool-call-parser`. The parser // is model-specific and vLLM does not auto-detect it, so it is never guessed: it @@ -991,7 +972,6 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { } else { None }; - let device_policy = parse_device_policy(device.as_ref().map(|policy| policy.as_policy_str()))?; let gpu_selection = parse_gpu_selection(gpu.as_deref())?; // CPU-only serving never pins a GPU, so skip GPU resolution entirely and // surface the explicit `--gpu` as ignored rather than printing a device the @@ -1005,7 +985,6 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // on a constant, so admitting the policy later cannot silently turn CPU-only // serving into GPU pre-flight failures. It is not evidence that the CPU path // is exercised; nothing in the suite can reach it. - let cpu_only = matches!(device_policy, DevicePolicy::CpuOnly); // AMD GPU ordinals still usable after the active visibility mask // (`HIP_VISIBLE_DEVICES`, then `ROCR_VISIBLE_DEVICES`) is applied, in HIP // ordinal space — the space `--gpu` is validated and exported through. A @@ -1231,15 +1210,6 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { } else { None }; - if let (Some(needed), false) = (aim_min_gpus, cpu_only) { - ensure_aim_gpu_count( - &model, - needed, - matches!(gpu_selection, GpuSelection::Index(_)), - visible_gpu_indices.as_ref().map(Vec::len), - None, - )?; - } // Resolve which runtime/env this serve will use, now that the host GPU // pre-condition and `--gpu` have both been checked (or, on the reuse path, // the pre-gate already resolved it under the exemption). This must not move back @@ -1319,9 +1289,6 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { visible_gpu_indices.as_deref(), gpu_vram.as_deref(), )?; - if let (Some(needed), false) = (aim_min_gpus, cpu_only) { - ensure_aim_gpu_count(&model, needed, false, None, Some(gpu_indices.len()))?; - } let service_id = generate_service_id(&selected_engine, &resolve.canonical_model_id); // Attached foreground streaming is the debugging path, selected by `--verbose` @@ -1669,6 +1636,34 @@ mod tests { ); } + #[test] + fn aim_catalogued_hf_ids_still_select_vllm_on_instinct_and_aim_elsewhere() { + // These ids had no recipe before the AIM catalog; cataloguing them must not + // move an Instinct host off vLLM, or refuse ids AIM has no Instinct profile for. + let instinct = rocm_core::HostGpuSummary { + therock_family: Some("gfx94X-dcgpu".to_owned()), + ..rocm_core::HostGpuSummary::default() + }; + let radeon = rocm_core::HostGpuSummary { + therock_family: Some("gfx1151".to_owned()), + ..rocm_core::HostGpuSummary::default() + }; + for id in [ + "meta-llama/Llama-3.1-8B-Instruct", + "openai/gpt-oss-20b", + "Qwen/Qwen3-32B", + ] { + let recipe = resolve_builtin_model_recipe(id).expect("aim-catalogued recipe"); + assert!(model_recipe_supports_engine(&recipe, "aim")); + let on_instinct = select_serve_engine(None, None, Some(&recipe), Some(&instinct)); + let on_radeon = select_serve_engine(None, None, Some(&recipe), Some(&radeon)); + if !cfg!(windows) { + assert_eq!(on_instinct.engine, "vllm", "{id}"); + } + assert_eq!(on_radeon.engine, "aim", "{id}"); + } + } + #[test] fn serve_qwen_uses_vllm_with_hf_repo_on_vllm_preferred_gpu() { // The qwen alias serves the GGUF via Lemonade by default, but on a vLLM-preferred @@ -1896,19 +1891,10 @@ mod tests { Some(4) ); - let err = |pinned, usable, selected| { - ensure_aim_gpu_count(id, 8, pinned, usable, selected) - .unwrap_err() - .to_string() - }; - assert!(err(true, Some(8), None).contains("`--gpu ` pins one device")); - assert!(err(false, Some(2), None).contains("only 2 usable GPUs were found")); - assert!(err(false, Some(1), None).contains("only 1 usable GPU was found")); - assert!(err(false, Some(8), Some(1)).contains("does not support multi-GPU serving yet")); - // Unknown counts, or no selection made (container sees every GPU), pass. - assert!(ensure_aim_gpu_count(id, 8, false, None, None).is_ok()); - assert!(ensure_aim_gpu_count(id, 8, false, Some(8), Some(0)).is_ok()); - assert!(ensure_aim_gpu_count(id, 8, false, Some(8), Some(8)).is_ok()); + let err = ensure_aim_multi_gpu_supported(id, 8) + .unwrap_err() + .to_string(); + assert!(err.contains("needs 8 GPUs") && err.contains("does not support multi-GPU")); } #[test] diff --git a/crates/rocm-core/src/model_catalog.json b/crates/rocm-core/src/model_catalog.json index 1b167a7022..0e82286f4b 100644 --- a/crates/rocm-core/src/model_catalog.json +++ b/crates/rocm-core/src/model_catalog.json @@ -584,7 +584,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "Radeon-targeted AIM image (W7900 / R9700 profiles only); requires Docker" @@ -630,7 +631,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -674,7 +676,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -718,7 +721,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -762,7 +766,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -806,7 +811,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -849,7 +855,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -894,7 +901,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -939,7 +947,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -984,7 +993,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -1030,7 +1040,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -1077,7 +1088,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -1122,7 +1134,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -1167,7 +1180,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -1211,7 +1225,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -1254,7 +1269,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -1297,7 +1313,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -1340,7 +1357,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -1384,7 +1402,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -1427,7 +1446,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -1470,7 +1490,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -1514,7 +1535,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -1557,7 +1579,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -1575,7 +1598,6 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", - "min_gpu_mem_gb": 281, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1601,7 +1623,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -1619,7 +1642,6 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", - "min_gpu_mem_gb": 51, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -1644,7 +1666,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -1688,7 +1711,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -1731,7 +1755,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -1774,7 +1799,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -1818,7 +1844,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -1862,7 +1889,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -1909,7 +1937,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -1956,7 +1985,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -2001,7 +2031,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -2048,7 +2079,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -2066,7 +2098,7 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", - "min_gpu_mem_gb": 426, + "min_gpu_mem_gb": 384, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -2092,7 +2124,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" @@ -2135,7 +2168,8 @@ "manual_alternatives": [], "chat_template_mode": "auto", "preferred_engines": [ - "aim" + "aim", + "vllm" ], "warnings": [ "AIM image from the AMD Enterprise AI catalog; requires Docker" diff --git a/crates/rocm-core/src/model_catalog.rs b/crates/rocm-core/src/model_catalog.rs index 1ed84aef50..f6051de284 100644 --- a/crates/rocm-core/src/model_catalog.rs +++ b/crates/rocm-core/src/model_catalog.rs @@ -179,7 +179,7 @@ pub fn model_recipe_staleness_warning( let age_ms = now_ms.checked_sub(engine_recipe.generated_at_unix_ms?)?; (age_ms > MODEL_RECIPE_STALE_AFTER_MS).then(|| { format!( - "{} entry was last verified {} days ago; the pinned image may no longer exist (see docs/aim-catalog.md to refresh)", + "{} entry was last verified {} days ago; the pinned image may no longer exist; update rocm-cli for a refreshed catalog", engine_recipe.engine, age_ms / (24 * 60 * 60 * 1000) ) @@ -842,7 +842,8 @@ fn validate_model_recipe_engine_record( { bail!( "engine recipe for `{}` on `{canonical_model_id}` sets supported_gfx_families, \ - profile_ids, requires_hf_token or supported_boards, which only apply to aim", + profile_ids, min_gpu_counts, requires_hf_token or supported_boards, which only \ + apply to aim", engine_recipe.engine ); } @@ -855,6 +856,11 @@ fn validate_model_recipe_engine_record( && engine_recipe.image_digest.is_none() && engine_recipe.extra_env.is_empty() && engine_recipe.supported_gfx_families.is_empty() + && engine_recipe.generated_at_unix_ms.is_none() + && engine_recipe.profile_ids.is_empty() + && !engine_recipe.requires_hf_token + && engine_recipe.supported_boards.is_empty() + && engine_recipe.min_gpu_counts.is_empty() { bail!( "engine recipe for `{}` on `{canonical_model_id}` must not be empty", diff --git a/docs/aim-catalog.md b/docs/aim-catalog.md index 61f1608cd7..2cc45360c9 100644 --- a/docs/aim-catalog.md +++ b/docs/aim-catalog.md @@ -17,7 +17,7 @@ recipe index is blocked on production signing keys (see Each AIM `engine_recipes` entry pins an image by `image_digest` and records `generated_at_unix_ms`, the time a maintainer last confirmed the image and -digest still exist in the registry. `rocm model` prints a warning for any +digest still exist in the registry. `rocm model --verbose` prints a warning for any engine recipe whose timestamp is older than 180 days (`MODEL_RECIPE_STALE_AFTER_MS` in [model_catalog.rs](../crates/rocm-core/src/model_catalog.rs)), because the @@ -34,7 +34,7 @@ not need a rocm-cli maintainer. 2. For each AIM entry, find the newest stable tag (`oras repo tags docker.io/amdenterpriseai/`) and copy its digest (`oras resolve docker.io/amdenterpriseai/:`) into `image_digest` (and `model_id_override` if the tag moved). - Check `AIM_PROFILE_ID` in `extra_env` is still a valid profile. + Check each `profile_ids` entry is still a valid profile. 3. Set `generated_at_unix_ms` to now (`date +%s%3N`). 4. Add entries for new AIM images the same way as the existing ones. The list of images is the AMD Enterprise AI AIM catalog @@ -42,7 +42,6 @@ not need a rocm-cli maintainer. CPU-only (`aim-epyc-*`) images are left out, since rocm-cli never falls back to CPU, as are non-chat images (speech, video, bio) until the serve path supports their tasks. -5. Open a PR; `cargo test -p rocm-core model_recipe` validates the catalog. 6. Check each image's model page in the AIM catalog for the line "requires a Hugging Face authentication". For those, set `requires_hf_token: true` and add the `HF_TOKEN` note to the entry's `notes` (the pattern used by the @@ -53,7 +52,7 @@ not need a rocm-cli maintainer. MI325X `gfx94X-dcgpu`; MI350X and MI355X `gfx950-dcgpu`; W7900 `gfx110X-all`; R9700 `gfx120X-all`). `rocm serve` refuses a GPU outside that list, and `rocm model` groups the recipe under the platform covering those families. - Instinct images auto-select a profile, so leave `AIM_PROFILE_ID` alone. Radeon + Instinct images auto-select a profile, so leave `profile_ids` unset. Radeon images do not select reliably: add `profile_ids` mapping each family to a profile (the lowest-GPU-count optimized latency profile, noting in `notes` if it needs more than one GPU). An explicit `--engine-env AIM_PROFILE_ID=...` @@ -64,13 +63,16 @@ not need a rocm-cli maintainer. cover boards the pinned profile was not built for. 9. Set `min_gpu_mem_gb`. The AIM docs publish GPU counts but no VRAM, so the values in the catalog are estimates: total parameters times 1 byte (2 when - the image has no fp8 profile) plus 20% headroom, rounded up. Entries whose + the image has no fp8 profile) plus 20% headroom, rounded up. Leave it unset + for other dtypes (for example MXFP4), and cap it at the entry's + `min_gpu_counts` times the smallest supported GPU's memory so it never + contradicts the GPU count. Entries whose parameter count is not published (DeepSeek-V4-Flash, MiniMax-M3, GLM-5.2) are left unset rather than guessed. Replace an estimate with a measured figure when one is available. 10. Set `min_gpu_counts` for any family whose smallest profile uses more than one GPU (the smallest tensor-parallel size in that family's profiles; leave the family out when a single-GPU profile exists). `rocm serve` refuses such an - image up front when `--gpu` pins one device or too few GPUs are usable. - `rocm serve` does not yet select several GPUs, so these images cannot be - started on a host where it would pick one; that is tracked as ROCMAI-751. + image up front, because `rocm serve` does not yet select several GPUs; that + is tracked as ROCMAI-751. +11. Open a PR; `cargo test -p rocm-core model_recipe` validates the catalog. diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 8ea43bfaaf..098853a25f 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -969,7 +969,7 @@ rocm serve amdenterpriseai/aim-radeon-qwen-qwen3-5-9b:0.12.0-preview --engine ai non-secret values only. For a secret such as a gated model's `HF_TOKEN`, export it and pass the bare name (`export HF_TOKEN=...`, then `--engine-env HF_TOKEN`): it is forwarded from your environment and never stored. A gated model -(`rocm model --verbose` shows `requires_hf_token` in its notes) is refused before +(`rocm model --verbose` shows the `HF_TOKEN` note) is refused before anything is pulled if no token was passed. `--engine-env` is ignored by the other engines (the command prints a note saying so). From fe3bb42a4076e1e51f8a1089823c0b32ee083333 Mon Sep 17 00:00:00 2001 From: Juho Vainio Date: Fri, 30 Oct 2026 15:13:17 +0200 Subject: [PATCH 10/21] ROCMAI-556: cover the gated AIM HF-token refusal with e2e scenarios Signed-off-by: Juho Vainio --- .../features/model_serving.feature | 24 +++++++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 65 +++++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 9a59d533a9..5bfa1ca7dd 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -519,3 +519,27 @@ Feature: Model serving Then serving is refused before any engine starts And the user is told no AMD GPU was detected And the user is not told to relaunch the server with different options + + # ROCMAI-556: a gated AIM image is refused before anything is pulled when no + # Hugging Face token was passed. The check runs ahead of GPU detection and + # Docker, so it needs neither and gates every PR (ungated). The refusal names + # the remedy and leaves no service record behind; serve-34 proves the named + # remedy clears the condition. The gfx-family, board-mismatch and multi-GPU + # refusals depend on the detected GPU family, so they are unit-tested in + # `apps/rocm/src/serve_cmd.rs` rather than covered here. + @id:serve-gated-aim-refused-without-hf-token + Scenario: serve-33 - A gated AIM image is refused without a Hugging Face token + When the user serves a gated AIM model without passing a Hugging Face token + Then serving is refused before any engine starts + And the CLI explains that the model needs a Hugging Face token + And no service was started + + # The remedy printed by serve-33 (`export HF_TOKEN`, then `--engine-env + # HF_TOKEN`) must actually clear that refusal. Every GPU is masked, so the + # serve then stops at the GPU pre-flight, still before anything is pulled; the + # refusal is no longer about the token. + @id:serve-gated-aim-token-remedy-clears-refusal + Scenario: serve-34 - Passing the exported Hugging Face token clears the gated AIM refusal + When the user serves a gated AIM model with an exported Hugging Face token and no visible GPU + Then serving is refused before any engine starts + And the refusal is not about the Hugging Face token diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index d3a7482821..f15be78dc9 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1096,6 +1096,41 @@ async fn user_serves_negative_temperature(world: &mut E2eWorld) { /// Serve with every GPU hidden from the CLI and engine via an empty /// `HIP_VISIBLE_DEVICES`, so a GPU host presents as having no usable device. +/// A gated AIM catalog entry (`requires_hf_token`). The token check runs before +/// GPU detection and Docker, so no engine or GPU is needed to reach it. +const GATED_AIM_MODEL: &str = "meta-llama/Llama-3.1-8B-Instruct"; + +#[when("the user serves a gated AIM model without passing a Hugging Face token")] +async fn user_serves_gated_aim_without_token(world: &mut E2eWorld) { + let (stdout, stderr, rc) = + crate::run_rocm(world, &["serve", GATED_AIM_MODEL, "--engine", "aim"]); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +#[when("the user serves a gated AIM model with an exported Hugging Face token and no visible GPU")] +async fn user_serves_gated_aim_with_token(world: &mut E2eWorld) { + let (stdout, stderr, rc) = crate::run_rocm_with_env( + world, + &[ + "serve", + GATED_AIM_MODEL, + "--engine", + "aim", + "--engine-env", + "HF_TOKEN", + ], + &[ + ("HF_TOKEN", "hf_e2e_placeholder"), + ("HIP_VISIBLE_DEVICES", ""), + ], + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + #[when("the user serves a model with every GPU masked from view")] async fn user_serves_with_masked_gpus(world: &mut E2eWorld) { let (model, engine, _) = host_serve_target(); @@ -1693,6 +1728,36 @@ async fn assert_no_engine_preparation(world: &mut E2eWorld) { !output.contains("Preparing "), "a live managed service for an unrelated model must not drag engine \ preparation ahead of the no-usable-GPU refusal:\n{output}" +#[then("the CLI explains that the model needs a Hugging Face token")] +async fn assert_hf_token_message(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + output.contains("needs a Hugging Face token") && output.contains("--engine-env HF_TOKEN"), + "expected the gated-model refusal naming the remedy, got:\n{output}" + ); +} + +#[then("no service was started")] +async fn assert_no_service_started(world: &mut E2eWorld) { + let services_dir = world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .join("data") + .join("services"); + let records = std::fs::read_dir(&services_dir) + .map(|entries| entries.count()) + .unwrap_or(0); + assert_eq!(records, 0, "a refused serve left service records behind"); +} + +#[then("the refusal is not about the Hugging Face token")] +async fn assert_refusal_is_not_hf_token(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + !output.contains("needs a Hugging Face token"), + "the exported token was not accepted:\n{output}" ); } From 840fe283fd864eb640de8fe934d8404482a95855 Mon Sep 17 00:00:00 2001 From: juhovainio Date: Mon, 2 Nov 2026 12:20:29 +0200 Subject: [PATCH 11/21] ROCMAI-556: fix clippy lint and model-catalog-01 for the renamed AIM recipe Use Iterator::count directly in assert_no_service_started, and match the AIM recipe by its model name with engines=[aim, vllm] now that aim-qwen3.5-9b is an alias. Signed-off-by: juhovainio --- tests/e2e-cucumber/features/model_catalog.feature | 2 +- tests/e2e-cucumber/tests/e2e/model_catalog_steps.rs | 6 +++++- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/e2e-cucumber/features/model_catalog.feature b/tests/e2e-cucumber/features/model_catalog.feature index a7acdee31e..22def3429e 100644 --- a/tests/e2e-cucumber/features/model_catalog.feature +++ b/tests/e2e-cucumber/features/model_catalog.feature @@ -1,6 +1,6 @@ Feature: Model catalog listing - # ROCMAI-555: the AIM engine recipe (aim-qwen3.5-9b) is curated out of the + # ROCMAI-555: the AIM engine recipe (Qwen/Qwen3.5-9B) is curated out of the # default `rocm model` view (it is not `featured`), so only `--verbose` # actually lists it. `model` and its `models` alias share one parser branch, # so exercising `model --verbose` covers the epic's "rocm model or rocm diff --git a/tests/e2e-cucumber/tests/e2e/model_catalog_steps.rs b/tests/e2e-cucumber/tests/e2e/model_catalog_steps.rs index 3e0e077d5b..478084206f 100644 --- a/tests/e2e-cucumber/tests/e2e/model_catalog_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/model_catalog_steps.rs @@ -24,8 +24,12 @@ async fn catalog_lists_aim_recipe(world: &mut E2eWorld) { Some(0), "listing the catalog is not a failure:\n{output}" ); + // The recipe is listed by its model name (`aim-qwen3.5-9b` is an alias) and + // shares the entry with vLLM, so match the AIM engine within that one line. assert!( - output.contains("aim-qwen3.5-9b") && output.contains("engines=[aim]"), + output + .lines() + .any(|line| line.contains("Qwen/Qwen3.5-9B") && line.contains("engines=[aim")), "expected the AIM recipe listed by canonical id and engine:\n{output}" ); } diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index f15be78dc9..989c6b58e3 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1747,7 +1747,7 @@ async fn assert_no_service_started(world: &mut E2eWorld) { .join("data") .join("services"); let records = std::fs::read_dir(&services_dir) - .map(|entries| entries.count()) + .map(Iterator::count) .unwrap_or(0); assert_eq!(records, 0, "a refused serve left service records behind"); } From 629ee8ad69aabf89b2ed19c2618b270f5b999e94 Mon Sep 17 00:00:00 2001 From: juhovainio Date: Mon, 2 Nov 2026 12:24:43 +0200 Subject: [PATCH 12/21] ROCMAI-556: use map_or in assert_no_service_started Signed-off-by: juhovainio --- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 989c6b58e3..d39ca63d85 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1746,9 +1746,7 @@ async fn assert_no_service_started(world: &mut E2eWorld) { .path() .join("data") .join("services"); - let records = std::fs::read_dir(&services_dir) - .map(Iterator::count) - .unwrap_or(0); + let records = std::fs::read_dir(&services_dir).map_or(0, Iterator::count); assert_eq!(records, 0, "a refused serve left service records behind"); } From ed0701c6192d5df6cda2e8495b66f3690695b026 Mon Sep 17 00:00:00 2001 From: juhovainio Date: Mon, 2 Nov 2026 14:04:17 +0200 Subject: [PATCH 13/21] ROCMAI-556: describe AIM in `rocm model --help` and spell out its name `rocm model --help` and the README now say that the AIM images are not in the default list, that `--verbose` shows them with their GPU families, GPU count and Hugging Face token requirement, and that `rocm serve --engine aim` runs them (needs a Docker daemon). AIM stands for AMD Inference Microservices. Replace the "AMD Enterprise AI" expansion in the catalog hints and warnings, the engine descriptions in `rocm` and `rocmd`, and docs/aim-catalog.md. The docs URL is unchanged. Signed-off-by: juhovainio --- README.md | 10 +- apps/rocm/src/main.rs | 10 +- crates/rocm-core/src/model_catalog.json | 142 ++++++++++++------------ docs/aim-catalog.md | 6 +- 4 files changed, 89 insertions(+), 79 deletions(-) diff --git a/README.md b/README.md index f993791f54..e3676f7dae 100644 --- a/README.md +++ b/README.md @@ -821,9 +821,13 @@ rocm model [--verbose] `rocm model` (alias `rocm models`) lists the recommended local models, grouped by the hardware they target. Pass `--verbose` to list every recipe, including ones hidden from the default list, with details such as aliases, data type, minimum -GPU memory, and preferred engines. AIM entries are hand-maintained and record -when they were last verified; `rocm model --verbose` warns when one is over 180 -days old (see `docs/aim-catalog.md`). +GPU memory, and preferred engines. The AMD Inference Microservices (AIM) Docker +images are among the hidden recipes: `--verbose` shows each one's supported GPU +families, GPU count, and whether it needs a Hugging Face token, and +`rocm serve --engine aim` runs it (needs a reachable Docker daemon). +AIM entries are hand-maintained and record when they were last verified; +`rocm model --verbose` warns when one is over 180 days old (see +`docs/aim-catalog.md`). ### Model serving diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 19313b7bb6..270ab29d8a 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -491,6 +491,12 @@ rocm update --json")] command: EnginesCommand, }, /// Show recommended local models and what this machine can run. + /// + /// The default list is the curated recommendations. The AMD Inference Microservices + /// (AIM) catalog of pre-built Docker images is not part of it: use --verbose to see + /// every AIM image with the GPU families it supports, how many GPUs it needs, and + /// whether it requires a Hugging Face token. Serve one with + /// `rocm serve --engine aim` (needs a reachable Docker daemon). #[command( alias = "models", after_help = "EXAMPLES:\n \ @@ -2717,7 +2723,7 @@ const fn builtin_engine_inventory() -> &'static [(&'static str, &'static str)] { ), ( "aim", - "AMD Enterprise AI pre-built Docker serving containers", + "AMD Inference Microservices (AIM) pre-built Docker serving containers", ), ] } @@ -5070,7 +5076,7 @@ pub(crate) const fn engine_inventory() -> &'static [(&'static str, &'static str) ), ( "aim", - "AMD Enterprise AI pre-built Docker serving containers", + "AMD Inference Microservices (AIM) pre-built Docker serving containers", ), ] } diff --git a/crates/rocm-core/src/model_catalog.json b/crates/rocm-core/src/model_catalog.json index 0e82286f4b..613d1546ea 100644 --- a/crates/rocm-core/src/model_catalog.json +++ b/crates/rocm-core/src/model_catalog.json @@ -560,7 +560,7 @@ "min_gpu_mem_gb": 24, "recommended_system_ram_gb": 32, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -605,7 +605,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 267, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -635,7 +635,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -652,7 +652,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 806, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -680,7 +680,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -697,7 +697,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 806, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -725,7 +725,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -742,7 +742,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 806, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -770,7 +770,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -787,7 +787,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 806, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -815,7 +815,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -831,7 +831,7 @@ "dtype": "auto", "device_policy": "gpu_required", "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -859,7 +859,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -876,7 +876,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 3, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -905,7 +905,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -922,7 +922,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 65, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -951,7 +951,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -968,7 +968,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 65, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -997,7 +997,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -1014,7 +1014,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 486, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -1044,7 +1044,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -1061,7 +1061,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 20, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -1092,7 +1092,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -1109,7 +1109,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 3, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -1138,7 +1138,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -1155,7 +1155,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 85, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -1184,7 +1184,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -1201,7 +1201,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 276, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -1229,7 +1229,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -1245,7 +1245,7 @@ "dtype": "auto", "device_policy": "gpu_required", "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -1273,7 +1273,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -1290,7 +1290,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 17, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -1317,7 +1317,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -1334,7 +1334,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 17, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -1361,7 +1361,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -1378,7 +1378,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 810, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -1406,7 +1406,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -1423,7 +1423,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 58, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -1450,7 +1450,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -1467,7 +1467,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 58, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -1494,7 +1494,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -1511,7 +1511,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 339, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -1539,7 +1539,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -1556,7 +1556,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 113, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -1583,7 +1583,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -1599,7 +1599,7 @@ "dtype": "auto", "device_policy": "gpu_required", "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -1627,7 +1627,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -1643,7 +1643,7 @@ "dtype": "auto", "device_policy": "gpu_required", "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -1670,7 +1670,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -1687,7 +1687,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 282, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -1715,7 +1715,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -1732,7 +1732,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 40, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -1759,7 +1759,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -1776,7 +1776,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 96, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -1803,7 +1803,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -1820,7 +1820,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 282, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -1848,7 +1848,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -1865,7 +1865,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 282, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -1893,7 +1893,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -1910,7 +1910,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 20, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -1941,7 +1941,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -1958,7 +1958,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 84, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -1989,7 +1989,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -2006,7 +2006,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 22, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -2035,7 +2035,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -2052,7 +2052,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 72, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -2083,7 +2083,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -2100,7 +2100,7 @@ "device_policy": "gpu_required", "min_gpu_mem_gb": 384, "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -2128,7 +2128,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] }, { @@ -2144,7 +2144,7 @@ "dtype": "auto", "device_policy": "gpu_required", "quantization": "none; served as-is by the AIM image's bundled vLLM", - "artifact_hint": "AMD Enterprise AI (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", + "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], "engine_recipes": [ { @@ -2172,7 +2172,7 @@ "vllm" ], "warnings": [ - "AIM image from the AMD Enterprise AI catalog; requires Docker" + "AIM (AMD Inference Microservices) image; requires Docker" ] } ] diff --git a/docs/aim-catalog.md b/docs/aim-catalog.md index 2cc45360c9..9b1ce69526 100644 --- a/docs/aim-catalog.md +++ b/docs/aim-catalog.md @@ -29,15 +29,15 @@ flagged. The owner of the AIM relationship (likely the AMD EAI team) does this; it does not need a rocm-cli maintainer. -1. Watch for AMD Enterprise AI Suite releases. A version bump is the practical - signal that AIM images were added or changed. +1. Watch for new AIM releases. A version bump is the practical signal that AIM + images were added or changed. 2. For each AIM entry, find the newest stable tag (`oras repo tags docker.io/amdenterpriseai/`) and copy its digest (`oras resolve docker.io/amdenterpriseai/:`) into `image_digest` (and `model_id_override` if the tag moved). Check each `profile_ids` entry is still a valid profile. 3. Set `generated_at_unix_ms` to now (`date +%s%3N`). 4. Add entries for new AIM images the same way as the existing ones. The list - of images is the AMD Enterprise AI AIM catalog + of images is the AMD Inference Microservices (AIM) catalog (). CPU-only (`aim-epyc-*`) images are left out, since rocm-cli never falls back to CPU, as are non-chat images (speech, video, bio) until the serve path From d951886bbd6f5a3f880405993b1467ae8b86a30e Mon Sep 17 00:00:00 2001 From: juhovainio Date: Mon, 2 Nov 2026 14:27:14 +0200 Subject: [PATCH 14/21] ROCMAI-556: refuse a gated AIM image for its token before the GPU check The Hugging Face token check ran after the GPU-family check, so on a host whose GPU the image has no profile for, a gated model was refused for its GPU and never for its missing token. That contradicted the refusal text and left serve-gated-aim-refused-without-hf-token failing on such hosts (for example gfx1151), while passing on CI runners with no detectable GPU. Check the token first, from a hint built from `--engine-env` alone. The real hint is unchanged, so `--engine-env AIM_PROFILE_ID` still overrides the catalog's profile pin. Signed-off-by: juhovainio --- apps/rocm/src/serve_cmd.rs | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 4c40e485c8..a59fb19331 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -249,7 +249,7 @@ fn aim_engine_record<'a>( /// Refuses to launch a gated AIM image before the container is pulled when no /// Hugging Face token will reach it: `--engine-env HF_TOKEN=`, or the bare /// `--engine-env HF_TOKEN` with `HF_TOKEN` exported (the form that is never -/// stored). Runs after `--engine-env` is applied to the hint. +/// stored). `hint` carries the `--engine-env` entries. fn ensure_aim_hf_token( engine: &str, recipe: Option<&ModelRecipeRecord>, @@ -873,6 +873,17 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { .and_then(|recipe| protocol_engine_recipe_hint(recipe, &selected_engine)); let device_policy = parse_device_policy(device.as_ref().map(|policy| policy.as_policy_str()))?; let cpu_only = matches!(device_policy, DevicePolicy::CpuOnly); + // The token needs nothing from the GPU, so it is checked first: a gated model + // is refused for its missing token on any host, not only on one whose GPU the + // image supports. Only `--engine-env` can supply it, so a hint built from + // that alone is enough; the real hint below still gets the same env applied + // after the catalog's profile pin, which is what lets the flag override it. + ensure_aim_hf_token( + &selected_engine, + shared_recipe.as_ref(), + engine_recipe_with_extra_env(&selected_engine, None, &engine_env)?.as_ref(), + |key| std::env::var(key).ok(), + )?; // The AIM GPU checks are about the GPU the container would use, so they do // not apply to `--device cpu_only`. let aim_host_family = aim_engine_record(&selected_engine, shared_recipe.as_ref()) @@ -936,12 +947,6 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { generation_defaults, )?; let engine_recipe = engine_recipe_with_extra_env(&selected_engine, engine_recipe, &engine_env)?; - ensure_aim_hf_token( - &selected_engine, - shared_recipe.as_ref(), - engine_recipe.as_ref(), - |key| std::env::var(key).ok(), - )?; let aim_gpu_name = aim_engine_record(&selected_engine, shared_recipe.as_ref()) .filter(|record| !record.profile_ids.is_empty()) .and_then(|_| { From 6718ca20f941eaf4b0dcacf958171ad136182e13 Mon Sep 17 00:00:00 2001 From: juhovainio Date: Mon, 2 Nov 2026 17:29:24 +0200 Subject: [PATCH 15/21] ROCMAI-556: address review on AIM engine selection, warnings and coverage Select aim by default only on Linux/WSL when the detected GPU family is in the image's supported families; pin the board warning to the profile actually applied; print requires_hf_token in --verbose; add scenarios for the stale entry warning and the unsupported-GPU and multi-GPU refusals (Docker call log asserts nothing was pulled); table-driven catalog tests; fix doc drift. Signed-off-by: juhovainio --- README.md | 8 +- apps/rocm/src/examine_report_text.rs | 37 +++- apps/rocm/src/serve_cmd.rs | 181 ++++++++++++++---- crates/rocm-core/src/model_catalog.rs | 131 +++++++++++-- docs/aim-catalog.md | 22 ++- docs/manual-testing.md | 11 +- .../features/model_catalog.feature | 11 ++ .../features/model_serving.feature | 54 +++++- tests/e2e-cucumber/src/bin/fake-docker.rs | 15 ++ .../e2e-cucumber/tests/e2e/diagnose_steps.rs | 4 +- .../tests/e2e/model_catalog_steps.rs | 87 +++++++++ tests/e2e-cucumber/tests/e2e/serving_steps.rs | 178 ++++++++++++++--- 12 files changed, 633 insertions(+), 106 deletions(-) diff --git a/README.md b/README.md index e3676f7dae..0bcb92e10f 100644 --- a/README.md +++ b/README.md @@ -823,7 +823,8 @@ the hardware they target. Pass `--verbose` to list every recipe, including ones hidden from the default list, with details such as aliases, data type, minimum GPU memory, and preferred engines. The AMD Inference Microservices (AIM) Docker images are among the hidden recipes: `--verbose` shows each one's supported GPU -families, GPU count, and whether it needs a Hugging Face token, and +families, the GPU count it needs per family, and whether it needs a Hugging Face +token (`requires_hf_token=yes`), and `rocm serve --engine aim` runs it (needs a reachable Docker daemon). AIM entries are hand-maintained and record when they were last verified; `rocm model --verbose` warns when one is over 180 days old (see @@ -918,7 +919,10 @@ instead of silently substituting a different model. Some models (such as Llama) are gated and require Hugging Face authentication. Log in with `huggingface-cli login` or set `HF_TOKEN` in your environment -before serving gated models. +before serving gated models. The exception is `--engine aim`, whose container +does not read your environment on its own: export `HF_TOKEN` and pass +`--engine-env HF_TOKEN` (see above), or `rocm serve` refuses a gated AIM image +before pulling anything. `--gpu` selects which AMD GPU the server runs on. `auto` (the default) probes per-GPU VRAM (through `amd-smi`, or the amdgpu DRM sysfs counters when `amd-smi` diff --git a/apps/rocm/src/examine_report_text.rs b/apps/rocm/src/examine_report_text.rs index 018a52cc33..5b81c32fe7 100644 --- a/apps/rocm/src/examine_report_text.rs +++ b/apps/rocm/src/examine_report_text.rs @@ -809,9 +809,14 @@ fn append_model_engine_recipe_settings_lines(output: &mut String, recipe: &Model if !engine_recipe.supported_gfx_families.is_empty() { let _ = writeln!( output, - " engine_recipe {} supported_gfx_families=[{}] profile_ids=[{}] min_gpu_counts=[{}]", + " engine_recipe {} supported_gfx_families=[{}] requires_hf_token={} profile_ids=[{}] min_gpu_counts=[{}]", engine_recipe.engine, engine_recipe.supported_gfx_families.join(","), + if engine_recipe.requires_hf_token { + "yes" + } else { + "no" + }, format_string_map_or_none(&engine_recipe.profile_ids), format_string_map_or_none( &engine_recipe @@ -2118,6 +2123,36 @@ mod tests { assert!(rendered.contains("tiny-gpt2 (2 GiB min GPU)")); } + #[test] + fn verbose_model_listing_shows_aim_gpu_families_and_token_requirement() { + let engine_lines = |id: &str| { + let recipe = rocm_core::resolve_builtin_model_recipe(id).expect("aim recipe"); + let mut output = String::new(); + append_model_engine_recipe_settings_lines(&mut output, &recipe); + output + }; + let gated = engine_lines("meta-llama/Llama-3.2-3B-Instruct"); + assert!( + gated.contains( + "engine_recipe aim supported_gfx_families=[gfx94X-dcgpu,gfx950-dcgpu] \ + requires_hf_token=yes profile_ids=[] min_gpu_counts=[]" + ), + "{gated}" + ); + // Radeon images list the profile pinned per family and the GPU counts. + let radeon = engine_lines("zai-org/GLM-4.7-Flash"); + assert!( + radeon.contains( + "supported_gfx_families=[gfx110X-all,gfx120X-all] requires_hf_token=no \ + profile_ids=[gfx110X-all=" + ) && radeon.contains("min_gpu_counts=[gfx110X-all=2,gfx120X-all=4]"), + "{radeon}" + ); + // A recipe with no AIM entry prints no such line. + let lemonade = engine_lines("qwen"); + assert!(!lemonade.contains("supported_gfx_families"), "{lemonade}"); + } + #[test] fn model_recipe_staleness_lines_warn_only_for_stale_engine_recipes() { let recipe = rocm_core::resolve_builtin_model_recipe("Qwen/Qwen3-32B").expect("aim entry"); diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index a59fb19331..44cde22525 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -112,7 +112,7 @@ pub(crate) fn select_serve_engine( } if let Some(engine) = recipe - .and_then(|recipe| recipe.preferred_engines.first()) + .and_then(|recipe| recipe_preferred_engine(recipe, host_gpu_summary, cfg!(windows))) .filter(|value| !value.trim().is_empty()) { return ServeEngineSelection { @@ -127,6 +127,36 @@ pub(crate) fn select_serve_engine( } } +/// The recipe's first preferred engine that can run on this host. AIM is only +/// chosen on its own when the detected GPU family is one the image has a profile +/// for and the host is Linux or WSL, so an undetected GPU or native Windows is not; +/// otherwise the platform default applies, as it did before the recipe was +/// catalogued. An explicit `--engine aim` never comes through here, and still +/// gets the refusal that names the supported GPUs. +fn recipe_preferred_engine<'a>( + recipe: &'a ModelRecipeRecord, + host_gpu_summary: Option<&rocm_core::HostGpuSummary>, + windows: bool, +) -> Option<&'a str> { + let is_aim = |engine: &str| engine.eq_ignore_ascii_case("aim"); + let aim_usable = !windows + && recipe + .engine_recipes + .iter() + .find(|record| is_aim(&record.engine)) + .is_none_or(|record| { + matches!( + engine_recipe_gfx_support( + record, + host_gpu_summary.and_then(|summary| summary.therock_family.as_deref()), + ), + GfxSupport::Supported(_) + ) || record.supported_gfx_families.is_empty() + }); + let first = recipe.preferred_engines.first()?; + (aim_usable || !is_aim(first)).then_some(first.as_str()) +} + fn model_recipe_supports_engine(recipe: &ModelRecipeRecord, engine: &str) -> bool { recipe .preferred_engines @@ -219,7 +249,8 @@ fn engine_recipe_with_aim_gfx_profile( match engine_recipe_gfx_support(record, family.as_deref()) { GfxSupport::Unrestricted => Ok(hint), GfxSupport::Unsupported => bail!( - "the AIM image for `{}` has no profile for this GPU ({}); it supports: {}", + "the AIM image for `{}` has no profile for this GPU ({}); it supports: {}. Choose a model \ + or engine that supports this GPU (`rocm model --verbose`)", recipe.canonical_model_id, family.as_deref().unwrap_or("unknown"), record.supported_gfx_families.join(", ") @@ -295,21 +326,32 @@ fn aim_min_gpu_count( /// devices and fail inside the image. fn ensure_aim_multi_gpu_supported(model: &str, needed: u32) -> Result<()> { bail!( - "`{model}` needs {needed} GPUs on this hardware, but `rocm serve` does not support \ - multi-GPU serving yet, so this model cannot be started here" + "`{model}` needs {needed} GPUs for this GPU family, but `rocm serve` runs an engine on \ + one GPU and does not support multi-GPU serving yet, so this model cannot be started \ + here however many GPUs the host has. Choose a model with a single-GPU profile \ + (`rocm model --verbose`)" ) } -/// Note shown when the pinned AIM profile targets boards other than the detected -/// GPU (a gfx family can cover cards the profile was not built for). +/// Note shown when the AIM profile that was pinned for this serve targets boards +/// other than the detected GPU (a gfx family can cover cards the profile was not +/// built for). `hint` is the final engine hint: the note needs the catalog's +/// profile to be what will actually run, so it stays quiet when no profile was +/// pinned (GPU family undetected, `--device cpu_only`) or `--engine-env +/// AIM_PROFILE_ID` replaced it. fn aim_board_note( engine: &str, recipe: Option<&ModelRecipeRecord>, gpu_name: Option<&str>, + hint: Option<&EngineRecipeHint>, ) -> Option { let record = aim_engine_record(engine, recipe)?; let name = gpu_name?; - if record.profile_ids.is_empty() || record.supported_boards.is_empty() { + if record.supported_boards.is_empty() { + return None; + } + let pinned = hint?.extra_env.get("AIM_PROFILE_ID")?; + if !record.profile_ids.values().any(|id| id == pinned) { return None; } let squash = |value: &str| { @@ -328,7 +370,7 @@ fn aim_board_note( return None; } Some(format!( - "the pinned AIM profile targets {}; `{name}` is not one of them, so it may not run well. Override with `--engine-env AIM_PROFILE_ID=`", + "the pinned AIM profile `{pinned}` targets {}; `{name}` is not one of them, so it may not run well. Override with `--engine-env AIM_PROFILE_ID=`", record.supported_boards.join(" / ") )) } @@ -961,6 +1003,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { &selected_engine, shared_recipe.as_ref(), aim_gpu_name.as_deref(), + engine_recipe.as_ref(), ), ] .into_iter() @@ -1642,31 +1685,66 @@ mod tests { } #[test] - fn aim_catalogued_hf_ids_still_select_vllm_on_instinct_and_aim_elsewhere() { + fn aim_catalogued_hf_ids_pick_aim_only_on_a_gpu_it_supports() { // These ids had no recipe before the AIM catalog; cataloguing them must not - // move an Instinct host off vLLM, or refuse ids AIM has no Instinct profile for. - let instinct = rocm_core::HostGpuSummary { - therock_family: Some("gfx94X-dcgpu".to_owned()), - ..rocm_core::HostGpuSummary::default() - }; - let radeon = rocm_core::HostGpuSummary { - therock_family: Some("gfx1151".to_owned()), + // move an Instinct host off vLLM, nor send a host the image has no profile + // for (or an undetected GPU, or native Windows) to an engine that refuses it. + let host = |family: Option<&str>| rocm_core::HostGpuSummary { + therock_family: family.map(str::to_owned), ..rocm_core::HostGpuSummary::default() }; - for id in [ - "meta-llama/Llama-3.1-8B-Instruct", - "openai/gpt-oss-20b", - "Qwen/Qwen3-32B", + let instinct = host(Some("gfx94X-dcgpu")); + let radeon = host(Some("gfx110X-all")); + let strix = host(Some("gfx1151")); + let undetected = host(None); + + // An Instinct-only image (`Qwen/Qwen3-32B`) and a Radeon one + // (`Qwen/Qwen3.5-9B`) each get AIM on their own family and the platform + // default elsewhere. + let instinct_image = resolve_builtin_model_recipe("Qwen/Qwen3-32B").unwrap(); + let radeon_image = resolve_builtin_model_recipe("Qwen/Qwen3.5-9B").unwrap(); + for (recipe, family, windows, expected) in [ + (&instinct_image, &instinct, false, Some("aim")), + (&instinct_image, &instinct, true, None), + (&instinct_image, &radeon, false, None), + (&instinct_image, &strix, false, None), + (&instinct_image, &undetected, false, None), + (&radeon_image, &radeon, false, Some("aim")), + (&radeon_image, &radeon, true, None), + (&radeon_image, &instinct, false, None), + (&radeon_image, &strix, false, None), ] { - let recipe = resolve_builtin_model_recipe(id).expect("aim-catalogued recipe"); - assert!(model_recipe_supports_engine(&recipe, "aim")); - let on_instinct = select_serve_engine(None, None, Some(&recipe), Some(&instinct)); - let on_radeon = select_serve_engine(None, None, Some(&recipe), Some(&radeon)); - if !cfg!(windows) { - assert_eq!(on_instinct.engine, "vllm", "{id}"); - } - assert_eq!(on_radeon.engine, "aim", "{id}"); + assert_eq!( + recipe_preferred_engine(recipe, Some(family), windows), + expected, + "{} on {:?} (windows: {windows})", + recipe.canonical_model_id, + family.therock_family + ); } + // No GPU summary at all is treated like an undetected GPU. + assert_eq!(recipe_preferred_engine(&radeon_image, None, false), None); + // Recipes that do not prefer AIM are untouched. + let vllm_only = resolve_builtin_model_recipe("meta-llama/Llama-3.2-3B-Instruct").unwrap(); + assert_eq!( + recipe_preferred_engine(&vllm_only, Some(&strix), false), + Some("vllm") + ); + + // End to end through the selector. A vLLM-preferred host keeps vLLM; the + // rest follow the table above (the platform default is OS-specific, so + // compare against it rather than a literal). + let on_instinct = select_serve_engine(None, None, Some(&instinct_image), Some(&instinct)); + if !cfg!(windows) { + assert_eq!(on_instinct.engine, "vllm"); + let on_radeon = select_serve_engine(None, None, Some(&radeon_image), Some(&radeon)); + assert_eq!(on_radeon.engine, "aim"); + } + let on_strix = select_serve_engine(None, None, Some(&radeon_image), Some(&strix)); + assert_eq!(on_strix.engine, default_engine_for_platform()); + // An explicit engine is never overridden. + let explicit = select_serve_engine(Some("aim"), None, Some(&radeon_image), Some(&strix)); + assert_eq!(explicit.engine, "aim"); } #[test] @@ -1903,17 +1981,48 @@ mod tests { } #[test] - fn aim_board_note_only_fires_for_a_pinned_profile_on_another_board() { + fn aim_board_note_only_fires_when_the_catalog_profile_is_what_runs() { let radeon = resolve_builtin_model_recipe("aim-qwen3.5-9b").expect("radeon aim entry"); - assert!(aim_board_note("aim", Some(&radeon), Some("AMD Radeon PRO W7900")).is_none()); - assert!(aim_board_note("aim", Some(&radeon), Some("AMD Radeon AI PRO R9700")).is_none()); - let note = aim_board_note("aim", Some(&radeon), Some("AMD Radeon RX 7900 XTX")) - .expect("other board warns"); - assert!(note.contains("W7900 / R9700") && note.contains("--engine-env AIM_PROFILE_ID")); - assert!(aim_board_note("aim", Some(&radeon), None).is_none()); + // The hint the serve path ends up with once a GPU family was detected. + let pinned = engine_recipe_with_aim_gfx_profile( + "aim", + Some(&radeon), + protocol_engine_recipe_hint(&radeon, "aim"), + || Some("gfx110X-all".to_owned()), + ) + .unwrap(); + let note = |gpu: Option<&str>, hint: Option<&EngineRecipeHint>| { + aim_board_note("aim", Some(&radeon), gpu, hint) + }; + assert!(note(Some("AMD Radeon PRO W7900"), pinned.as_ref()).is_none()); + assert!(note(Some("AMD Radeon AI PRO R9700"), pinned.as_ref()).is_none()); + let other = note(Some("AMD Radeon RX 7900 XTX"), pinned.as_ref()).expect("warns"); + assert!( + other.contains("vllm-w7900-bf16-tp1-latency") + && other.contains("W7900 / R9700") + && other.contains("--engine-env AIM_PROFILE_ID"), + "{other}" + ); + assert!(note(None, pinned.as_ref()).is_none()); + + // No pin applied (family undetected, or `--device cpu_only`): no claim about it. + let unpinned = engine_recipe_with_aim_gfx_profile( + "aim", + Some(&radeon), + protocol_engine_recipe_hint(&radeon, "aim"), + || None, + ) + .unwrap(); + assert!(note(Some("AMD Radeon RX 7900 XTX"), unpinned.as_ref()).is_none()); + assert!(note(Some("AMD Radeon RX 7900 XTX"), None).is_none()); + // An explicit AIM_PROFILE_ID replaced the catalog's: its board is the user's call. + let overridden = + engine_recipe_with_extra_env("aim", pinned, &["AIM_PROFILE_ID=vllm-other".to_owned()]) + .unwrap(); + assert!(note(Some("AMD Radeon RX 7900 XTX"), overridden.as_ref()).is_none()); // Instinct images pin no profile, so there is nothing to mismatch. let instinct = resolve_builtin_model_recipe("Qwen/Qwen3-32B").expect("instinct entry"); - assert!(aim_board_note("aim", Some(&instinct), Some("anything")).is_none()); + assert!(aim_board_note("aim", Some(&instinct), Some("anything"), None).is_none()); } #[test] diff --git a/crates/rocm-core/src/model_catalog.rs b/crates/rocm-core/src/model_catalog.rs index f6051de284..cc6e3de667 100644 --- a/crates/rocm-core/src/model_catalog.rs +++ b/crates/rocm-core/src/model_catalog.rs @@ -93,7 +93,7 @@ pub struct ModelRecipeEngineRecord { #[serde(default)] pub image_digest: Option, /// When a maintainer last checked this entry against the upstream image - /// registry (unix ms). `rocm model` warns once it is older than + /// registry (unix ms). `rocm model --verbose` warns once it is older than /// [`MODEL_RECIPE_STALE_AFTER_MS`], since the pinned image may be gone. #[serde(default)] pub generated_at_unix_ms: Option, @@ -123,7 +123,8 @@ pub struct ModelRecipeEngineRecord { /// Minimum number of GPUs the image needs per gfx family (the smallest /// tensor-parallel size across that family's profiles). A family left out /// needs one. Keys must be listed in `supported_gfx_families`. `rocm serve` - /// refuses to start with fewer usable GPUs. Only valid on `aim`. + /// gives an engine one GPU, so it refuses to start an image listed here for + /// the detected family, however many GPUs the host has. Only valid on `aim`. #[serde(default)] pub min_gpu_counts: BTreeMap, } @@ -179,7 +180,7 @@ pub fn model_recipe_staleness_warning( let age_ms = now_ms.checked_sub(engine_recipe.generated_at_unix_ms?)?; (age_ms > MODEL_RECIPE_STALE_AFTER_MS).then(|| { format!( - "{} entry was last verified {} days ago; the pinned image may no longer exist; update rocm-cli for a refreshed catalog", + "{} entry was last verified {} days ago; the pinned image may no longer exist; update rocm-cli (rerun the installer) for a refreshed catalog", engine_recipe.engine, age_ms / (24 * 60 * 60 * 1000) ) @@ -1255,14 +1256,122 @@ mod tests { } #[test] - fn builtin_aim_recipe_carries_a_verification_timestamp() { - let recipe = resolve_builtin_model_recipe("Qwen/Qwen3.5-9B").expect("aim recipe"); - let aim = recipe - .engine_recipes - .iter() - .find(|e| e.engine == "aim") - .expect("aim engine"); - assert!(aim.generated_at_unix_ms.is_some()); + fn every_builtin_aim_recipe_carries_a_verification_timestamp() { + let mut checked = 0; + for recipe in builtin_model_recipes() { + for aim in recipe.engine_recipes.iter().filter(|e| e.engine == "aim") { + checked += 1; + assert!( + aim.generated_at_unix_ms.is_some(), + "{} aim entry has no generated_at_unix_ms, so it can never be flagged stale", + recipe.canonical_model_id + ); + } + } + assert!(checked > 0, "the built-in catalog lists no aim entries"); + } + + #[test] + fn engine_recipe_gfx_support_matches_families_without_regard_to_case() { + let unrestricted = ModelRecipeEngineRecord { + engine: "aim".to_owned(), + ..Default::default() + }; + // No families listed: any host, detected or not, is unrestricted. + for family in [None, Some("gfx1151"), Some("gfx94X-dcgpu")] { + assert_eq!( + engine_recipe_gfx_support(&unrestricted, family), + GfxSupport::Unrestricted, + "{family:?}" + ); + } + let aim = ModelRecipeEngineRecord { + engine: "aim".to_owned(), + supported_gfx_families: vec!["gfx94X-dcgpu".to_owned(), "gfx110X-all".to_owned()], + profile_ids: BTreeMap::from([("gfx110X-all".to_owned(), "p".to_owned())]), + ..Default::default() + }; + // Matching ignores case on both the family list and the pinned profile key. + assert_eq!( + engine_recipe_gfx_support(&aim, Some("GFX110X-ALL")), + GfxSupport::Supported(Some("p")) + ); + assert_eq!( + engine_recipe_gfx_support(&aim, Some("gfx94x-DCGPU")), + GfxSupport::Supported(None) + ); + assert_eq!( + engine_recipe_gfx_support(&aim, Some("gfx120X-all")), + GfxSupport::Unsupported + ); + assert_eq!( + engine_recipe_gfx_support(&aim, None), + GfxSupport::Unrestricted + ); + } + + #[test] + fn model_recipe_targets_platform_needs_the_engine_and_a_covered_family() { + let platform = |engines: &[&str], families: &[&str]| ModelCatalogPlatform { + label: "p".to_owned(), + engines: engines.iter().map(|e| (*e).to_owned()).collect(), + gfx_families: families.iter().map(|f| (*f).to_owned()).collect(), + }; + let recipe = |preferred: &[&str], families: &[&str]| { + let mut recipe = resolve_builtin_model_recipe("qwen").expect("recipe"); + recipe.preferred_engines = preferred.iter().map(|e| (*e).to_owned()).collect(); + recipe.engine_recipes = vec![ModelRecipeEngineRecord { + engine: "aim".to_owned(), + supported_gfx_families: families.iter().map(|f| (*f).to_owned()).collect(), + ..Default::default() + }]; + recipe + }; + let two_families = recipe(&["aim"], &["gfx94X-dcgpu", "gfx110X-all"]); + for (recipe, platform, expected, why) in [ + ( + &two_families, + platform(&["aim"], &["gfx110X-all"]), + true, + "covering just one of the recipe's families is enough", + ), + ( + &two_families, + platform(&["aim"], &["gfx1151"]), + false, + "no family in common", + ), + ( + &two_families, + platform(&["vllm"], &["gfx94X-dcgpu"]), + false, + "the engine does not run on the platform", + ), + ( + &two_families, + platform(&["AIM"], &["GFX94X-DCGPU"]), + true, + "engine and family names compare without regard to case", + ), + ( + &recipe(&["aim"], &[]), + platform(&["aim"], &["gfx1151"]), + true, + "an unrestricted recipe fits any platform running the engine", + ), + ( + &recipe(&[], &["gfx94X-dcgpu"]), + platform(&["aim"], &["gfx94X-dcgpu"]), + false, + "no preferred engine, no platform", + ), + ] { + assert_eq!( + model_recipe_targets_platform(recipe, &platform), + expected, + "{why}" + ); + } } #[test] diff --git a/docs/aim-catalog.md b/docs/aim-catalog.md index 9b1ce69526..d12d85c841 100644 --- a/docs/aim-catalog.md +++ b/docs/aim-catalog.md @@ -21,13 +21,14 @@ digest still exist in the registry. `rocm model --verbose` prints a warning for engine recipe whose timestamp is older than 180 days (`MODEL_RECIPE_STALE_AFTER_MS` in [model_catalog.rs](../crates/rocm-core/src/model_catalog.rs)), because the -pinned image may have been withdrawn. Entries with no timestamp are never -flagged. +pinned image may have been withdrawn. Only `rocm model --verbose` shows the +warning; `rocm serve` does not check it. Every built-in AIM entry must carry a +timestamp (a unit test enforces it); an entry without one, such as one from a +configured recipe index, is never flagged. ## Refreshing an entry -The owner of the AIM relationship (likely the AMD EAI team) does this; it does -not need a rocm-cli maintainer. +Whoever owns the AIM images does this; it does not need a rocm-cli maintainer. 1. Watch for new AIM releases. A version bump is the practical signal that AIM images were added or changed. @@ -42,12 +43,12 @@ not need a rocm-cli maintainer. CPU-only (`aim-epyc-*`) images are left out, since rocm-cli never falls back to CPU, as are non-chat images (speech, video, bio) until the serve path supports their tasks. -6. Check each image's model page in the AIM catalog for the line "requires a +5. Check each image's model page in the AIM catalog for the line "requires a Hugging Face authentication". For those, set `requires_hf_token: true` and add the `HF_TOKEN` note to the entry's `notes` (the pattern used by the existing gated entries). `rocm serve` then refuses to start the image until `HF_TOKEN` is passed with `--engine-env`. -7. Record which GPUs the image has profiles for. From the model page's profile +6. Record which GPUs the image has profiles for. From the model page's profile table, set `supported_gfx_families` on the entry (MI250X `gfx90a`; MI300X and MI325X `gfx94X-dcgpu`; MI350X and MI355X `gfx950-dcgpu`; W7900 `gfx110X-all`; R9700 `gfx120X-all`). `rocm serve` refuses a GPU outside that list, and @@ -57,11 +58,11 @@ not need a rocm-cli maintainer. profile (the lowest-GPU-count optimized latency profile, noting in `notes` if it needs more than one GPU). An explicit `--engine-env AIM_PROFILE_ID=...` still wins. -8. On Radeon entries, set `supported_boards` to the board names in the profile +7. On Radeon entries, set `supported_boards` to the board names in the profile table (for example `W7900`, `R9700`). `rocm serve` warns, without refusing, when the detected GPU name matches none of them, because a gfx family can cover boards the pinned profile was not built for. -9. Set `min_gpu_mem_gb`. The AIM docs publish GPU counts but no VRAM, so the +8. Set `min_gpu_mem_gb`. The AIM docs publish GPU counts but no VRAM, so the values in the catalog are estimates: total parameters times 1 byte (2 when the image has no fp8 profile) plus 20% headroom, rounded up. Leave it unset for other dtypes (for example MXFP4), and cap it at the entry's @@ -70,9 +71,10 @@ not need a rocm-cli maintainer. parameter count is not published (DeepSeek-V4-Flash, MiniMax-M3, GLM-5.2) are left unset rather than guessed. Replace an estimate with a measured figure when one is available. -10. Set `min_gpu_counts` for any family whose smallest profile uses more than one +9. Set `min_gpu_counts` for any family whose smallest profile uses more than one GPU (the smallest tensor-parallel size in that family's profiles; leave the family out when a single-GPU profile exists). `rocm serve` refuses such an image up front, because `rocm serve` does not yet select several GPUs; that is tracked as ROCMAI-751. -11. Open a PR; `cargo test -p rocm-core model_recipe` validates the catalog. +10. Open a PR; `cargo test -p rocm-core model_catalog` validates the catalog, + including `builtin_model_catalog_json_parses_and_validates`. diff --git a/docs/manual-testing.md b/docs/manual-testing.md index 098853a25f..adc75aede6 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -938,7 +938,10 @@ Expected result: ## 10. AIM (Docker) Engine Verification Linux/WSL2 only, and requires a reachable Docker daemon (`docker info` must -succeed). +succeed). The catalog entry used below has profiles for Radeon PRO W7900 and +Radeon AI PRO R9700 (`gfx110X-all`, `gfx120X-all`) only; on any other GPU, +such as Strix Halo or an Instinct card, `rocm serve` refuses it, so use the +raw image form further down. ```bash rocm serve aim-qwen3.5-9b --engine aim --managed @@ -957,8 +960,10 @@ and its downloaded weights, so a retry downloads them again. `aim-qwen3.5-9b` (canonical id `Qwen/Qwen3.5-9B`) is the catalog entry (`rocm model --verbose`). It supplies the pinned image and, for the detected GPU family, `AIM_PROFILE_ID`; on a GPU the image has no profile for (for example -Strix Halo or an Instinct card) `rocm serve` refuses with the supported families. To serve a raw image reference instead, pass -the profile yourself: +Strix Halo or an Instinct card) `rocm serve` refuses with the supported +families. An image whose smallest profile needs several GPUs is refused on every +host, because `rocm serve` gives an engine one GPU. To serve a raw image +reference instead, which skips these catalog checks, pass the profile yourself: ```bash rocm serve amdenterpriseai/aim-radeon-qwen-qwen3-5-9b:0.12.0-preview --engine aim \ diff --git a/tests/e2e-cucumber/features/model_catalog.feature b/tests/e2e-cucumber/features/model_catalog.feature index 22def3429e..24df934d3f 100644 --- a/tests/e2e-cucumber/features/model_catalog.feature +++ b/tests/e2e-cucumber/features/model_catalog.feature @@ -25,3 +25,14 @@ Feature: Model catalog listing Given no Docker daemon is reachable When the user asks for the model catalog in verbose detail Then the catalog's AIM entry is flagged unavailable for lack of Docker + + # ROCMAI-556: an AIM entry pins an image that may since have been withdrawn, so + # `rocm model --verbose` warns once the entry was last verified more than 180 + # days ago. The entry is shipped with a verification timestamp; this plants a + # signed catalog whose timestamp is decades old, so the warning is exercised + # without depending on the date the suite runs. + @id:model-catalog-warns-on-stale-aim-entry + Scenario: model-catalog-03 - A stale AIM catalog entry is flagged in rocm model --verbose + Given a recipe catalog whose AIM entry was last verified long ago + When the user asks for the model catalog in verbose detail + Then the catalog warns that the AIM entry may no longer be current diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index 5bfa1ca7dd..1c6f881f7e 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -345,7 +345,9 @@ Feature: Model serving # AIM serve + inference canary (ROCMAI-554). This scenario deliberately still # serves via the raw AIM Docker image reference rather than the catalog # recipe added in ROCMAI-555 — it's a hardware-verified canary pinned to one - # exact image/profile, not a catalog-discovery test. `@requires-engine:aim` + # exact image/profile, not a catalog-discovery test. Because it names the raw + # image, the catalog's GPU-family refusal (no profile for Strix Halo) does not + # apply to it. `@requires-engine:aim` # skips here when Docker is unreachable OR the gfx family isn't one aim is # verified on (see `HostCapability::engine_available`'s "aim" arm — Radeon # gfx110X/120X and Strix Halo gfx1151, not Instinct); a Strix Halo @@ -524,22 +526,54 @@ Feature: Model serving # Hugging Face token was passed. The check runs ahead of GPU detection and # Docker, so it needs neither and gates every PR (ungated). The refusal names # the remedy and leaves no service record behind; serve-34 proves the named - # remedy clears the condition. The gfx-family, board-mismatch and multi-GPU - # refusals depend on the detected GPU family, so they are unit-tested in - # `apps/rocm/src/serve_cmd.rs` rather than covered here. + # remedy clears the condition. A Docker that logs its calls shows that nothing + # was pulled. serve-35 and serve-36 cover the GPU-dependent refusals. @id:serve-gated-aim-refused-without-hf-token Scenario: serve-33 - A gated AIM image is refused without a Hugging Face token + Given Docker is watched for any call When the user serves a gated AIM model without passing a Hugging Face token Then serving is refused before any engine starts And the CLI explains that the model needs a Hugging Face token And no service was started + And Docker was never called # The remedy printed by serve-33 (`export HF_TOKEN`, then `--engine-env - # HF_TOKEN`) must actually clear that refusal. Every GPU is masked, so the - # serve then stops at the GPU pre-flight, still before anything is pulled; the - # refusal is no longer about the token. - @id:serve-gated-aim-token-remedy-clears-refusal + # HF_TOKEN`) must actually clear that refusal. The host's one GPU is one the + # image supports and every GPU is masked, so the serve passes the token and + # GPU-family gates and stops at the GPU pre-flight, still before anything is + # pulled. The GPU is planted, so the outcome does not depend on the runner. + # Linux only: the empty `HIP_VISIBLE_DEVICES` mask and the planted kernel + # topology are Linux mechanisms. + @id:serve-gated-aim-token-remedy-clears-refusal @requires-os:linux Scenario: serve-34 - Passing the exported Hugging Face token clears the gated AIM refusal - When the user serves a gated AIM model with an exported Hugging Face token and no visible GPU + Given Docker is watched for any call + And a host whose only GPU is an AMD Radeon PRO W7900 + When the user serves a gated AIM model with an exported Hugging Face token while every GPU is masked Then serving is refused before any engine starts - And the refusal is not about the Hugging Face token + And the refusal is that no usable GPU is visible + And Docker was never called + + # An AIM image only carries profiles for some GPUs. On a GPU it has none for, + # `rocm serve --engine aim` is refused up front and names what the image does + # support, rather than pulling a container that fails inside. The detected GPU + # is planted (an Instinct card against a Radeon-only image), so this needs no + # AMD hardware and runs on every Linux lane. + @id:serve-aim-refused-on-unsupported-gpu @requires-os:linux + Scenario: serve-35 - An AIM image is refused on a GPU it has no profile for + Given Docker is watched for any call + And a host whose only GPU is an AMD Instinct MI300X + When the user serves a Radeon-only AIM image + Then serving is refused before any engine starts + And the CLI says the image has no profile for this GPU and names the GPUs it supports + And Docker was never called + + # `rocm serve` gives an engine one GPU, so an AIM image whose smallest profile + # needs several is refused before launch instead of starting with too few. + @id:serve-aim-refused-when-it-needs-several-gpus @requires-os:linux + Scenario: serve-36 - An AIM image that needs several GPUs is refused + Given Docker is watched for any call + And a host whose only GPU is an AMD Instinct MI300X + When the user serves an AIM image that needs eight GPUs + Then serving is refused before any engine starts + And the CLI says the image needs eight GPUs and multi-GPU serving is not supported + And Docker was never called diff --git a/tests/e2e-cucumber/src/bin/fake-docker.rs b/tests/e2e-cucumber/src/bin/fake-docker.rs index d6e55cf181..a43f2f9eac 100644 --- a/tests/e2e-cucumber/src/bin/fake-docker.rs +++ b/tests/e2e-cucumber/src/bin/fake-docker.rs @@ -11,9 +11,24 @@ //! simulates "no reachable daemon" regardless of whether the host running the //! suite actually has Docker installed. //! +//! When `FAKE_DOCKER_LOG` names a file, every invocation's arguments are +//! appended to it first (one line per call), so a scenario can prove a refused +//! `rocm serve` never reached Docker, not even for `docker info`. +//! //! A Rust binary rather than a shell script so the scenarios run on Windows //! too. fn main() { + if let Some(log) = std::env::var_os("FAKE_DOCKER_LOG") { + use std::io::Write as _; + let args: Vec = std::env::args().skip(1).collect(); + if let Ok(mut file) = std::fs::OpenOptions::new() + .create(true) + .append(true) + .open(log) + { + let _ = writeln!(file, "{}", args.join(" ")); + } + } std::process::exit(1); } diff --git a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs index 18b8819d06..e5cb167c54 100644 --- a/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/diagnose_steps.rs @@ -2238,7 +2238,7 @@ async fn assert_reason_is_not_curated(world: &mut E2eWorld) { ); } -fn workspace_root() -> std::path::PathBuf { +pub(crate) fn workspace_root() -> std::path::PathBuf { std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")) .parent() .and_then(std::path::Path::parent) @@ -2246,7 +2246,7 @@ fn workspace_root() -> std::path::PathBuf { .to_path_buf() } -fn xtask_command() -> std::process::Command { +pub(crate) fn xtask_command() -> std::process::Command { if let Some(binary) = std::env::var_os("ROCM_XTASK_BINARY") { std::process::Command::new(binary) } else { diff --git a/tests/e2e-cucumber/tests/e2e/model_catalog_steps.rs b/tests/e2e-cucumber/tests/e2e/model_catalog_steps.rs index 478084206f..5cf2a61c9b 100644 --- a/tests/e2e-cucumber/tests/e2e/model_catalog_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/model_catalog_steps.rs @@ -64,3 +64,90 @@ async fn catalog_aim_entry_flagged_unavailable(world: &mut E2eWorld) { "expected the AIM entry to name the Docker-unavailable reason:\n{output}" ); } + +/// Unix ms for 2001-09-09, far more than 180 days before any run of this suite. +const LONG_AGO_MS: u64 = 1_000_000_000_000; + +/// Point the CLI at a signed recipe index holding the built-in `Qwen/Qwen3.5-9B` +/// AIM entry with `generated_at_unix_ms` set `LONG_AGO_MS`, the only difference +/// from the shipped entry. Signed because that is the only way the CLI loads a +/// recipe index (same xtask keygen/sign path as `diagnose_steps`). +#[given("a recipe catalog whose AIM entry was last verified long ago")] +async fn catalog_with_stale_aim_entry(world: &mut E2eWorld) { + let mut recipe = rocm_core::resolve_builtin_model_recipe("Qwen/Qwen3.5-9B") + .expect("the built-in AIM recipe"); + for engine_recipe in &mut recipe.engine_recipes { + if engine_recipe.engine == "aim" { + engine_recipe.generated_at_unix_ms = Some(u128::from(LONG_AGO_MS)); + } + } + let root = world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .to_path_buf(); + let index = root.join("stale-aim-recipes.json"); + let signature = root.join("stale-aim-recipes.json.sig"); + let public_key = root.join("stale-aim-public.pem"); + let private_key = root.join("stale-aim-private.pem"); + let document = serde_json::json!({ + "schema_version": 1, + "source": "e2e-stale-aim", + "recipes": [recipe], + }); + std::fs::write( + &index, + serde_json::to_vec_pretty(&document).expect("failed to serialize the recipe fixture"), + ) + .expect("failed to write the recipe fixture"); + + let keygen = crate::e2e::diagnose_steps::xtask_command() + .args(["keygen", "--private-out"]) + .arg(&private_key) + .arg("--public-out") + .arg(&public_key) + .current_dir(crate::e2e::diagnose_steps::workspace_root()) + .status() + .expect("failed to run xtask keygen"); + assert!(keygen.success(), "xtask keygen failed"); + let sign = crate::e2e::diagnose_steps::xtask_command() + .args(["sign", "--private-key"]) + .arg(&private_key) + .arg("--in") + .arg(&index) + .arg("--out") + .arg(&signature) + .current_dir(crate::e2e::diagnose_steps::workspace_root()) + .status() + .expect("failed to run xtask sign"); + assert!(sign.success(), "xtask sign failed"); + + world + .command_env + .push(("ROCM_CLI_MODEL_RECIPE_INDEX_PATH", index.into_os_string())); + world.command_env.push(( + "ROCM_CLI_MODEL_RECIPE_INDEX_SIGNATURE_PATH", + signature.into_os_string(), + )); + world.command_env.push(( + "ROCM_CLI_MODEL_RECIPE_INDEX_PUBLIC_KEY_PATH", + public_key.into_os_string(), + )); +} + +#[then("the catalog warns that the AIM entry may no longer be current")] +async fn catalog_warns_aim_entry_is_stale(world: &mut E2eWorld) { + let output = world.cli_output.as_ref().expect("no model catalog output"); + assert_eq!( + world.cli_rc, + Some(0), + "listing the catalog failed:\n{output}" + ); + assert!( + output.contains("warning: aim entry was last verified") + && output.contains("days ago") + && output.contains("rerun the installer"), + "expected a staleness warning for the AIM entry, got:\n{output}" + ); +} diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index d39ca63d85..d5546da6fd 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1094,43 +1094,125 @@ async fn user_serves_negative_temperature(world: &mut E2eWorld) { world.cli_rc = Some(rc); } -/// Serve with every GPU hidden from the CLI and engine via an empty -/// `HIP_VISIBLE_DEVICES`, so a GPU host presents as having no usable device. -/// A gated AIM catalog entry (`requires_hf_token`). The token check runs before -/// GPU detection and Docker, so no engine or GPU is needed to reach it. +/// A gated AIM catalog entry (`requires_hf_token`), with a profile for a Radeon +/// PRO W7900 (`gfx110X-all`). The token check runs before GPU detection and +/// Docker, so no engine or GPU is needed to reach it. const GATED_AIM_MODEL: &str = "meta-llama/Llama-3.1-8B-Instruct"; -#[when("the user serves a gated AIM model without passing a Hugging Face token")] -async fn user_serves_gated_aim_without_token(world: &mut E2eWorld) { - let (stdout, stderr, rc) = - crate::run_rocm(world, &["serve", GATED_AIM_MODEL, "--engine", "aim"]); +/// An AIM image with profiles for Radeon (`gfx110X-all`, `gfx120X-all`) only. +const RADEON_AIM_MODEL: &str = "Qwen/Qwen3.5-9B"; + +/// An AIM image that needs 8 GPUs on an Instinct MI300X. +const EIGHT_GPU_AIM_MODEL: &str = "deepseek-ai/DeepSeek-R1"; + +/// Put a `docker` that always fails first on `PATH` and have it log every call to +/// a file in the scenario root, so a refused serve can be shown never to have +/// reached Docker (not even `docker info`). +#[given("Docker is watched for any call")] +async fn docker_is_watched(world: &mut E2eWorld) { + let root = world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .to_path_buf(); + let bin_dir = root.join("watched-docker-bin"); + crate::install_fixture_binary( + "fake-docker", + &bin_dir.join(e2e_cucumber::fixture_binary::fixture_target_name("docker")), + ); + let mut path = std::ffi::OsString::from(&bin_dir); + path.push(if cfg!(windows) { ";" } else { ":" }); + path.push(std::env::var_os("PATH").unwrap_or_default()); + world.command_env.push(("PATH", path)); + world + .command_env + .push(("FAKE_DOCKER_LOG", docker_call_log(world).into_os_string())); +} + +fn docker_call_log(world: &E2eWorld) -> std::path::PathBuf { + world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .join("docker-calls.log") +} + +/// Plant one GPU in the simulated kernel topology (`ROCM_CLI_TEST_HOST_ROOT`, honoured +/// by `e2e-test-hooks` builds) so the family the CLI detects does not depend on the +/// runner's hardware. `gfx_target_version` is the kernel's decimal encoding. +fn plant_single_gpu(world: &mut E2eWorld, gfx_target_version: u32) { + let host_root = world + .isolated_root + .as_ref() + .expect("scenario has no isolated root") + .path() + .join("host-root"); + let node = host_root.join("sys/class/kfd/kfd/topology/nodes/1"); + std::fs::create_dir_all(&node).expect("failed to create the planted KFD node"); + // `cpu_cores_count 0` with a nonzero `simd_count` makes the node a GPU. + std::fs::write( + node.join("properties"), + format!("cpu_cores_count 0\nsimd_count 56\ngfx_target_version {gfx_target_version}\n"), + ) + .expect("failed to write the planted KFD node properties"); + world + .command_env + .push(("ROCM_CLI_TEST_HOST_ROOT", host_root.into_os_string())); +} + +/// gfx942, TheRock family `gfx94X-dcgpu`. +#[given("a host whose only GPU is an AMD Instinct MI300X")] +async fn host_with_an_mi300x(world: &mut E2eWorld) { + plant_single_gpu(world, 90402); +} + +/// gfx1100, TheRock family `gfx110X-all`. +#[given("a host whose only GPU is an AMD Radeon PRO W7900")] +async fn host_with_a_w7900(world: &mut E2eWorld) { + plant_single_gpu(world, 110_000); +} + +fn run_serve_aim(world: &mut E2eWorld, args: &[&str]) { + let mut full = vec!["serve"]; + full.extend_from_slice(args); + full.extend_from_slice(&["--engine", "aim"]); + let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env(world, &full); world.cli_output = Some(stdout); world.cli_stderr = Some(stderr); world.cli_rc = Some(rc); } -#[when("the user serves a gated AIM model with an exported Hugging Face token and no visible GPU")] +#[when("the user serves a gated AIM model without passing a Hugging Face token")] +async fn user_serves_gated_aim_without_token(world: &mut E2eWorld) { + run_serve_aim(world, &[GATED_AIM_MODEL]); +} + +#[when( + "the user serves a gated AIM model with an exported Hugging Face token while every GPU is masked" +)] async fn user_serves_gated_aim_with_token(world: &mut E2eWorld) { - let (stdout, stderr, rc) = crate::run_rocm_with_env( - world, - &[ - "serve", - GATED_AIM_MODEL, - "--engine", - "aim", - "--engine-env", - "HF_TOKEN", - ], - &[ - ("HF_TOKEN", "hf_e2e_placeholder"), - ("HIP_VISIBLE_DEVICES", ""), - ], - ); - world.cli_output = Some(stdout); - world.cli_stderr = Some(stderr); - world.cli_rc = Some(rc); + world + .command_env + .push(("HF_TOKEN", "hf_e2e_placeholder".into())); + // An empty mask hides every device, so a GPU host presents as having none usable. + world.command_env.push(("HIP_VISIBLE_DEVICES", "".into())); + run_serve_aim(world, &[GATED_AIM_MODEL, "--engine-env", "HF_TOKEN"]); } +#[when("the user serves a Radeon-only AIM image")] +async fn user_serves_radeon_only_aim(world: &mut E2eWorld) { + run_serve_aim(world, &[RADEON_AIM_MODEL]); +} + +#[when("the user serves an AIM image that needs eight GPUs")] +async fn user_serves_eight_gpu_aim(world: &mut E2eWorld) { + run_serve_aim(world, &[EIGHT_GPU_AIM_MODEL]); +} + +/// Serve with every GPU hidden from the CLI and engine via an empty +/// `HIP_VISIBLE_DEVICES`, so a GPU host presents as having no usable device. #[when("the user serves a model with every GPU masked from view")] async fn user_serves_with_masked_gpus(world: &mut E2eWorld) { let (model, engine, _) = host_serve_target(); @@ -1750,12 +1832,46 @@ async fn assert_no_service_started(world: &mut E2eWorld) { assert_eq!(records, 0, "a refused serve left service records behind"); } -#[then("the refusal is not about the Hugging Face token")] -async fn assert_refusal_is_not_hf_token(world: &mut E2eWorld) { +#[then("the refusal is that no usable GPU is visible")] +async fn assert_refusal_is_no_usable_gpu(world: &mut E2eWorld) { + // The token gate and the GPU-family gate were both passed: the serve got as + // far as the GPU-required pre-flight, which the empty mask fails. + let output = serve_output(world); + assert!( + output.contains("no usable AMD GPU") && !output.contains("needs a Hugging Face token"), + "expected the exported token to clear its gate and the serve to stop at the \ + GPU pre-flight, got:\n{output}" + ); +} + +#[then("the CLI says the image has no profile for this GPU and names the GPUs it supports")] +async fn assert_no_profile_for_gpu(world: &mut E2eWorld) { let output = serve_output(world); assert!( - !output.contains("needs a Hugging Face token"), - "the exported token was not accepted:\n{output}" + output.contains("has no profile for this GPU (gfx94X-dcgpu)") + && output.contains("it supports: gfx110X-all, gfx120X-all"), + "expected the unsupported-GPU refusal naming the detected and supported families, \ + got:\n{output}" + ); +} + +#[then("the CLI says the image needs eight GPUs and multi-GPU serving is not supported")] +async fn assert_needs_eight_gpus(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + output.contains("needs 8 GPUs") && output.contains("does not support multi-GPU serving"), + "expected the multi-GPU refusal, got:\n{output}" + ); +} + +#[then("Docker was never called")] +async fn assert_docker_never_called(world: &mut E2eWorld) { + let log = docker_call_log(world); + let calls = std::fs::read_to_string(&log).unwrap_or_default(); + assert!( + calls.is_empty(), + "a refused serve reached Docker ({}):\n{calls}", + log.display() ); } From 1c728904158d047df6252547738b5d63f34937a4 Mon Sep 17 00:00:00 2001 From: juhovainio Date: Tue, 3 Nov 2026 11:35:09 +0200 Subject: [PATCH 16/21] ROCMAI-556: address review on planner engine choice, AIM pre-flight and verbose listing - chat planner and dash serve wizard no longer take a host-gated AIM first choice as an explicit engine when the recipe lists another one - extract aim_gpu_preflight (family check, profile pin, multi-GPU) from serve() so the pin, ordering and no-probe filter are unit-tested - `rocm model --verbose` prints the AIM host-fit line whenever any of its fields is set, not only when supported_gfx_families is - tests for each validator branch, the recipe_preferred_engine arms and the verbose line in model-catalog-01 - staleness remedy no longer promises a refresh; docs and --engine-env help state what is and is not refused - fix serving_steps.rs left unclosed by the rebase conflict resolution Signed-off-by: juhovainio --- apps/rocm/src/dash.rs | 4 +- apps/rocm/src/examine_report_text.rs | 31 +++- apps/rocm/src/freeform_planner.rs | 55 +++++- apps/rocm/src/main.rs | 3 +- apps/rocm/src/serve_cmd.rs | 165 +++++++++++++----- crates/rocm-core/src/model_catalog.rs | 89 +++++++++- docs/manual-testing.md | 15 +- .../features/model_catalog.feature | 5 +- .../tests/e2e/model_catalog_steps.rs | 11 +- tests/e2e-cucumber/tests/e2e/serving_steps.rs | 3 + 10 files changed, 321 insertions(+), 60 deletions(-) diff --git a/apps/rocm/src/dash.rs b/apps/rocm/src/dash.rs index 891255c1d7..2e88ef3d3a 100644 --- a/apps/rocm/src/dash.rs +++ b/apps/rocm/src/dash.rs @@ -190,10 +190,12 @@ fn model_recipe_summaries() -> Vec { builtin_model_recipes() .into_iter() .map(|r| ModelRecipeSummary { + // Same rule as the chat planner: a host-gated AIM first choice is + // not pre-selected when the recipe lists another engine. + preferred_engine: crate::freeform_planner::planner_recipe_engine(&r).map(str::to_owned), id: r.canonical_model_id, aliases: r.aliases, task: r.task, - preferred_engine: r.preferred_engines.into_iter().next(), }) .collect() } diff --git a/apps/rocm/src/examine_report_text.rs b/apps/rocm/src/examine_report_text.rs index 5b81c32fe7..ce39804192 100644 --- a/apps/rocm/src/examine_report_text.rs +++ b/apps/rocm/src/examine_report_text.rs @@ -29,7 +29,8 @@ //! `render_model_registry_verbose_text_with_context_and_host`, //! `model_catalog_header`, `append_model_recipe_registry_source`, //! `append_model_recipe_metadata_lines`, -//! `append_model_engine_recipe_settings_lines`, `format_list_or_none`, +//! `append_model_engine_recipe_settings_lines`, +//! `append_model_recipe_staleness_lines`, `format_list_or_none`, //! `format_string_map_or_none`, `append_model_artifact_lines`, //! `append_artifact_source_policy_lines`, `artifact_source_policy_label`, //! `format_bytes`, `append_model_host_ram_fit_lines`, @@ -806,12 +807,16 @@ fn append_model_engine_recipe_settings_lines(output: &mut String, recipe: &Model .join("; ") }; let notes = format_list_or_none(&engine_recipe.notes); - if !engine_recipe.supported_gfx_families.is_empty() { + if !engine_recipe.supported_gfx_families.is_empty() + || engine_recipe.requires_hf_token + || !engine_recipe.profile_ids.is_empty() + || !engine_recipe.min_gpu_counts.is_empty() + { let _ = writeln!( output, " engine_recipe {} supported_gfx_families=[{}] requires_hf_token={} profile_ids=[{}] min_gpu_counts=[{}]", engine_recipe.engine, - engine_recipe.supported_gfx_families.join(","), + format_list_or_none(&engine_recipe.supported_gfx_families), if engine_recipe.requires_hf_token { "yes" } else { @@ -2153,6 +2158,26 @@ mod tests { assert!(!lemonade.contains("supported_gfx_families"), "{lemonade}"); } + #[test] + fn verbose_model_listing_shows_a_token_requirement_without_gfx_families() { + // A schema-valid record can carry `requires_hf_token` alone; the listing + // must still say the image needs a token. + let mut recipe = + rocm_core::resolve_builtin_model_recipe("meta-llama/Llama-3.2-3B-Instruct") + .expect("aim recipe"); + for record in &mut recipe.engine_recipes { + record.supported_gfx_families.clear(); + } + let mut output = String::new(); + append_model_engine_recipe_settings_lines(&mut output, &recipe); + assert!( + output.contains( + "engine_recipe aim supported_gfx_families=[] requires_hf_token=yes" + ), + "{output}" + ); + } + #[test] fn model_recipe_staleness_lines_warn_only_for_stale_engine_recipes() { let recipe = rocm_core::resolve_builtin_model_recipe("Qwen/Qwen3-32B").expect("aim entry"); diff --git a/apps/rocm/src/freeform_planner.rs b/apps/rocm/src/freeform_planner.rs index 8268bbf580..9891d36b1e 100644 --- a/apps/rocm/src/freeform_planner.rs +++ b/apps/rocm/src/freeform_planner.rs @@ -368,11 +368,7 @@ fn build_freeform_plan_on( .or(requested_model) .unwrap_or(""); let engine = infer_engine_from_request(&lower) - .or_else(|| { - resolved_recipe - .as_ref() - .and_then(|recipe| recipe.preferred_engines.first().map(String::as_str)) - }) + .or_else(|| resolved_recipe.as_ref().and_then(planner_recipe_engine)) .unwrap_or(default_engine); let device_policy = infer_device_policy_from_request(&lower).or_else(|| { resolved_recipe @@ -1139,6 +1135,17 @@ pub(crate) fn render_structured_request_plan( output } +/// The recipe's first preferred engine, unless it is AIM with another engine +/// listed behind it. AIM is host-gated (GPU family, platform, Hugging Face +/// token), so the planner leaves the engine to the host default instead of +/// emitting an explicit `--engine aim` that `rocm serve` would never override. +/// An AIM-only recipe has no alternative and keeps AIM. +pub(crate) fn planner_recipe_engine(recipe: &ModelRecipeRecord) -> Option<&str> { + let first = recipe.preferred_engines.first()?; + (!(first.eq_ignore_ascii_case("aim") && recipe.preferred_engines.len() > 1)) + .then_some(first.as_str()) +} + fn planner_is_serve_request(lower: &str) -> bool { lower.contains("serve") || lower.contains("run a small local model") @@ -1346,6 +1353,44 @@ mod tests { })); } + #[test] + fn planner_does_not_emit_host_gated_aim_as_an_explicit_engine() { + // `Qwen/Qwen3-32B` lists `aim` first with `vllm` behind it; `rocm serve` + // never overrides an explicit `--engine aim`, so the plan must not name it. + let recipe = resolve_builtin_model_recipe("Qwen/Qwen3-32B").expect("catalogued id"); + assert_eq!( + recipe.preferred_engines.first().map(String::as_str), + Some("aim") + ); + assert_eq!(planner_recipe_engine(&recipe), None); + let plan = build_freeform_plan_with_recipes( + "serve Qwen/Qwen3-32B", + &RocmCliConfig::default(), + Some(&[recipe]), + "vllm", + ); + assert!( + plan.parsed + .contains(&("engine".to_owned(), "vllm".to_owned())), + "{:?}", + plan.parsed + ); + // An explicit request for AIM is still honoured. + let explicit = build_freeform_plan_with_recipes( + "serve Qwen/Qwen3-32B with aim", + &RocmCliConfig::default(), + None, + "vllm", + ); + assert!( + explicit + .parsed + .contains(&("engine".to_owned(), "aim".to_owned())), + "{:?}", + explicit.parsed + ); + } + #[test] fn hybrid_planner_can_use_active_recipe_registry_aliases() { let mut recipe = resolve_builtin_model_recipe("tiny-gpt2").expect("tiny recipe"); diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 270ab29d8a..929e71da69 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -631,7 +631,8 @@ rocm serve qwen --verbose --device gpu_required")] /// `AIM_PROFILE_ID=...`) and other engines ignore it. `KEY=VALUE` is /// recorded with the service, so for a secret such as `HF_TOKEN` pass a /// bare `KEY` instead: it is forwarded from your environment and never - /// stored. + /// stored. A gated AIM image (one that needs a Hugging Face token) is + /// refused up front unless `HF_TOKEN` is passed this way. #[arg(long = "engine-env", value_name = "KEY[=VALUE]")] engine_env: Vec, }, diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 44cde22525..6905acb481 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -127,7 +127,9 @@ pub(crate) fn select_serve_engine( } } -/// The recipe's first preferred engine that can run on this host. AIM is only +/// The recipe's first preferred engine, unless it is AIM and AIM cannot run on +/// this host; the caller then falls back to the platform default, and a later +/// preferred engine such as `vllm` is not considered. AIM is only /// chosen on its own when the detected GPU family is one the image has a profile /// for and the host is Linux or WSL, so an undetected GPU or native Windows is not; /// otherwise the platform default applies, as it did before the recipe was @@ -266,6 +268,30 @@ fn engine_recipe_with_aim_gfx_profile( } } +/// The AIM checks that depend on the host GPU, in the order `serve` applies +/// them: refuse a family the image has no profile for, pin the profile the +/// catalog names for the detected family, then refuse an image that needs more +/// than one GPU. The host is only probed for an AIM record that restricts GPU +/// families, and not at all for `--device cpu_only`, since these checks are about +/// the GPU the container would use. An undetected family enforces nothing. +fn aim_gpu_preflight( + engine: &str, + model: &str, + recipe: Option<&ModelRecipeRecord>, + hint: Option, + cpu_only: bool, + detect_family: impl FnOnce() -> Option, +) -> Result> { + let family = aim_engine_record(engine, recipe) + .filter(|record| !record.supported_gfx_families.is_empty() && !cpu_only) + .and_then(|_| detect_family()); + let hint = engine_recipe_with_aim_gfx_profile(engine, recipe, hint, || family.clone())?; + if let Some(needed) = aim_min_gpu_count(engine, recipe, family.as_deref()) { + ensure_aim_multi_gpu_supported(model, needed)?; + } + Ok(hint) +} + fn aim_engine_record<'a>( engine: &str, recipe: Option<&'a ModelRecipeRecord>, @@ -914,6 +940,18 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { .filter(|recipe| model_recipe_supports_engine(recipe, &selected_engine)) .and_then(|recipe| protocol_engine_recipe_hint(recipe, &selected_engine)); let device_policy = parse_device_policy(device.as_ref().map(|policy| policy.as_policy_str()))?; + // CPU-only serving never pins a GPU, so skip GPU resolution entirely and + // surface the explicit `--gpu` as ignored rather than printing a device the + // server will not use. + // + // Always `false` today, and kept anyway: `parse_device_policy` rejects + // `cpu`/`cpu_only` outright (pinned by + // `parse_device_policy_defaults_to_gpu_required_without_cpu_fallback`), so + // the only policy that reaches here is `GpuRequired`. The term is what makes + // every `!cpu_only` guard below read as a guard on the *policy* rather than + // on a constant, so admitting the policy later cannot silently turn CPU-only + // serving into GPU pre-flight failures. It is not evidence that the CPU path + // is exercised; nothing in the suite can reach it. let cpu_only = matches!(device_policy, DevicePolicy::CpuOnly); // The token needs nothing from the GPU, so it is checked first: a gated model // is refused for its missing token on any host, not only on one whose GPU the @@ -926,29 +964,18 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { engine_recipe_with_extra_env(&selected_engine, None, &engine_env)?.as_ref(), |key| std::env::var(key).ok(), )?; - // The AIM GPU checks are about the GPU the container would use, so they do - // not apply to `--device cpu_only`. - let aim_host_family = aim_engine_record(&selected_engine, shared_recipe.as_ref()) - .filter(|record| !record.supported_gfx_families.is_empty() && !cpu_only) - .and_then(|_| { - detect_host_gfx_target() - .as_deref() - .and_then(normalize_therock_family) - }); - let recipe_hint = engine_recipe_with_aim_gfx_profile( + let recipe_hint = aim_gpu_preflight( &selected_engine, + &model, shared_recipe.as_ref(), recipe_hint, - || aim_host_family.clone(), + cpu_only, + || { + detect_host_gfx_target() + .as_deref() + .and_then(normalize_therock_family) + }, )?; - let aim_min_gpus = aim_min_gpu_count( - &selected_engine, - shared_recipe.as_ref(), - aim_host_family.as_deref(), - ); - if let Some(needed) = aim_min_gpus { - ensure_aim_multi_gpu_supported(&model, needed)?; - } // vLLM rejects the TUI chat tab's tool-bearing requests with HTTP 400 unless it // is launched with `--enable-auto-tool-choice`/`--tool-call-parser`. The parser // is model-specific and vLLM does not auto-detect it, so it is never guessed: it @@ -1021,18 +1048,6 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { None }; let gpu_selection = parse_gpu_selection(gpu.as_deref())?; - // CPU-only serving never pins a GPU, so skip GPU resolution entirely and - // surface the explicit `--gpu` as ignored rather than printing a device the - // server will not use. - // - // Always `false` today, and kept anyway: `parse_device_policy` rejects - // `cpu`/`cpu_only` outright (pinned by - // `parse_device_policy_defaults_to_gpu_required_without_cpu_fallback`), so - // the only policy that reaches here is `GpuRequired`. The term is what makes - // every `!cpu_only` guard below read as a guard on the *policy* rather than - // on a constant, so admitting the policy later cannot silently turn CPU-only - // serving into GPU pre-flight failures. It is not evidence that the CPU path - // is exercised; nothing in the suite can reach it. // AMD GPU ordinals still usable after the active visibility mask // (`HIP_VISIBLE_DEVICES`, then `ROCR_VISIBLE_DEVICES`) is applied, in HIP // ordinal space — the space `--gpu` is validated and exported through. A @@ -2026,18 +2041,86 @@ mod tests { } #[test] - fn aim_gfx_check_is_skipped_when_the_gpu_is_unknown_or_the_engine_is_not_aim() { + fn aim_gpu_preflight_probes_only_for_a_restricted_aim_record_off_cpu_only() { let radeon = resolve_builtin_model_recipe("aim-qwen3.5-9b").expect("radeon aim entry"); let hint = protocol_engine_recipe_hint(&radeon, "aim"); - let unchanged = - engine_recipe_with_aim_gfx_profile("aim", Some(&radeon), hint.clone(), || None) - .unwrap(); - assert_eq!(unchanged, hint); - let vllm = engine_recipe_with_aim_gfx_profile("vllm", Some(&radeon), None, || { - panic!("host GPU must not be probed for other engines") + let preflight = |engine: &str, cpu_only: bool, family: fn() -> Option| { + aim_gpu_preflight(engine, "m", Some(&radeon), hint.clone(), cpu_only, family) + }; + let no_probe = || -> Option { panic!("host GPU must not be probed") }; + + // An undetected family enforces nothing. + assert_eq!(preflight("aim", false, || None).unwrap(), hint); + // The probe runs for AIM off `cpu_only`, and the result is enforced. + assert!(preflight("aim", false, || Some("gfx94X-dcgpu".to_owned())).is_err()); + // Neither a different engine nor `--device cpu_only` reaches the probe, + // so an unsupported family cannot refuse them. + assert_eq!(preflight("vllm", false, no_probe).unwrap(), hint); + assert_eq!(preflight("aim", true, no_probe).unwrap(), hint); + } + + #[test] + fn aim_gpu_preflight_pins_the_profile_and_then_refuses_multi_gpu_images() { + let radeon = resolve_builtin_model_recipe("aim-qwen3.5-9b").expect("radeon aim entry"); + let pinned = aim_gpu_preflight( + "aim", + "m", + Some(&radeon), + protocol_engine_recipe_hint(&radeon, "aim"), + false, + || Some("gfx110X-all".to_owned()), + ) + .unwrap() + .expect("hint"); + assert_eq!( + pinned.extra_env.get("AIM_PROFILE_ID").map(String::as_str), + Some("vllm-w7900-bf16-tp1-latency") + ); + + // The multi-GPU refusal comes from the same pre-flight, and only for a + // family whose count exceeds one. + let multi = resolve_builtin_model_recipe("zai-org/GLM-4.7-Flash").expect("multi-GPU entry"); + let hint = || protocol_engine_recipe_hint(&multi, "aim"); + let err = aim_gpu_preflight("aim", "m", Some(&multi), hint(), false, || { + Some("gfx120X-all".to_owned()) }) - .unwrap(); - assert!(vllm.is_none()); + .unwrap_err(); + assert!(err.to_string().contains("multi-GPU serving yet"), "{err:#}"); + assert!( + aim_gpu_preflight("aim", "m", Some(&multi), hint(), true, || None).is_ok(), + "cpu_only skips the multi-GPU check" + ); + } + + #[test] + fn recipe_preferred_engine_treats_unrestricted_and_missing_aim_records_as_usable() { + let host = rocm_core::HostGpuSummary { + therock_family: Some("gfx1151".to_owned()), + ..rocm_core::HostGpuSummary::default() + }; + let mut recipe = resolve_builtin_model_recipe("Qwen/Qwen3-32B").unwrap(); + assert_eq!(recipe_preferred_engine(&recipe, Some(&host), false), None); + // Only a signed external index reaches these two arms, so they are built + // here: an AIM record that restricts no family is usable on any GPU, and + // a recipe that lists AIM first but carries no AIM record has nothing to + // gate it. + for record in &mut recipe.engine_recipes { + if record.engine == "aim" { + record.supported_gfx_families.clear(); + } + } + assert_eq!( + recipe_preferred_engine(&recipe, Some(&host), false), + Some("aim") + ); + recipe + .engine_recipes + .retain(|record| record.engine != "aim"); + assert_eq!( + recipe_preferred_engine(&recipe, Some(&host), false), + Some("aim") + ); + assert_eq!(recipe_preferred_engine(&recipe, Some(&host), true), None); } #[test] diff --git a/crates/rocm-core/src/model_catalog.rs b/crates/rocm-core/src/model_catalog.rs index cc6e3de667..574be7bf4a 100644 --- a/crates/rocm-core/src/model_catalog.rs +++ b/crates/rocm-core/src/model_catalog.rs @@ -180,7 +180,7 @@ pub fn model_recipe_staleness_warning( let age_ms = now_ms.checked_sub(engine_recipe.generated_at_unix_ms?)?; (age_ms > MODEL_RECIPE_STALE_AFTER_MS).then(|| { format!( - "{} entry was last verified {} days ago; the pinned image may no longer exist; update rocm-cli (rerun the installer) for a refreshed catalog", + "{} entry was last verified {} days ago; the pinned image may no longer exist; entries are refreshed by hand, so a newer rocm-cli release or recipe index may carry a current one", engine_recipe.engine, age_ms / (24 * 60 * 60 * 1000) ) @@ -1255,6 +1255,93 @@ mod tests { assert!(validate_model_recipe_engine_record(&wrong_engine, "m").is_err()); } + /// One case per branch of `validate_model_recipe_engine_record` added for the + /// AIM host-fit fields; each is the only thing wrong (or, for the `ok` rows, + /// the only content) in its record, so removing a branch turns its row red. + #[test] + fn engine_record_validation_covers_each_aim_field_branch() { + let aim = |edit: fn(&mut ModelRecipeEngineRecord)| { + let mut record = ModelRecipeEngineRecord { + engine: "aim".to_owned(), + ..Default::default() + }; + edit(&mut record); + record + }; + let cases: Vec<(&str, ModelRecipeEngineRecord, bool)> = vec![ + ( + "unknown gfx family", + aim(|r| r.supported_gfx_families = vec!["gfx0000".to_owned()]), + false, + ), + ( + "profile pin and AIM_PROFILE_ID env", + aim(|r| { + r.supported_gfx_families = vec!["gfx110X-all".to_owned()]; + r.profile_ids + .insert("gfx110X-all".to_owned(), "p".to_owned()); + r.extra_env + .insert("AIM_PROFILE_ID".to_owned(), "q".to_owned()); + }), + false, + ), + ( + "min_gpu_counts family not in supported_gfx_families", + aim(|r| { + r.supported_gfx_families = vec!["gfx110X-all".to_owned()]; + r.min_gpu_counts.insert("gfx120X-all".to_owned(), 2); + }), + false, + ), + ( + "requires_hf_token on a non-aim engine", + ModelRecipeEngineRecord { + engine: "vllm".to_owned(), + requires_hf_token: true, + ..Default::default() + }, + false, + ), + ( + "supported_boards on a non-aim engine", + ModelRecipeEngineRecord { + engine: "vllm".to_owned(), + supported_boards: vec!["W7900".to_owned()], + ..Default::default() + }, + false, + ), + ("empty aim record", aim(|_| {}), false), + ( + "aim record carrying only supported_gfx_families", + aim(|r| r.supported_gfx_families = vec!["gfx110X-all".to_owned()]), + true, + ), + ( + "aim record carrying only requires_hf_token", + aim(|r| r.requires_hf_token = true), + true, + ), + ( + "aim record carrying only supported_boards", + aim(|r| r.supported_boards = vec!["W7900".to_owned()]), + true, + ), + ( + "aim record carrying only generated_at_unix_ms", + aim(|r| r.generated_at_unix_ms = Some(1)), + true, + ), + ]; + for (name, record, valid) in cases { + assert_eq!( + validate_model_recipe_engine_record(&record, "m").is_ok(), + valid, + "{name}" + ); + } + } + #[test] fn every_builtin_aim_recipe_carries_a_verification_timestamp() { let mut checked = 0; diff --git a/docs/manual-testing.md b/docs/manual-testing.md index adc75aede6..aedfea80ed 100644 --- a/docs/manual-testing.md +++ b/docs/manual-testing.md @@ -939,9 +939,12 @@ Expected result: Linux/WSL2 only, and requires a reachable Docker daemon (`docker info` must succeed). The catalog entry used below has profiles for Radeon PRO W7900 and -Radeon AI PRO R9700 (`gfx110X-all`, `gfx120X-all`) only; on any other GPU, -such as Strix Halo or an Instinct card, `rocm serve` refuses it, so use the -raw image form further down. +Radeon AI PRO R9700 (`gfx110X-all`, `gfx120X-all`) only; on a GPU of another +family, such as Strix Halo or an Instinct card, `rocm serve` refuses it, so use +the raw image form further down. Other boards of a supported family (for +example an RX 7900 XTX) are not refused: the CLI prints a note that the pinned +profile targets the W7900 / R9700 and names `--engine-env AIM_PROFILE_ID` to +override it. ```bash rocm serve aim-qwen3.5-9b --engine aim --managed @@ -961,8 +964,10 @@ and its downloaded weights, so a retry downloads them again. (`rocm model --verbose`). It supplies the pinned image and, for the detected GPU family, `AIM_PROFILE_ID`; on a GPU the image has no profile for (for example Strix Halo or an Instinct card) `rocm serve` refuses with the supported -families. An image whose smallest profile needs several GPUs is refused on every -host, because `rocm serve` gives an engine one GPU. To serve a raw image +families. An image whose smallest profile needs several GPUs on the detected +family is refused, because `rocm serve` gives an engine one GPU. None of these +GPU checks run when the host's GPU family cannot be detected or with +`--device cpu_only`. To serve a raw image reference instead, which skips these catalog checks, pass the profile yourself: ```bash diff --git a/tests/e2e-cucumber/features/model_catalog.feature b/tests/e2e-cucumber/features/model_catalog.feature index 24df934d3f..458dd80d7e 100644 --- a/tests/e2e-cucumber/features/model_catalog.feature +++ b/tests/e2e-cucumber/features/model_catalog.feature @@ -13,8 +13,9 @@ Feature: Model catalog listing # The epic names two reasons a listed AIM image can be unusable: an # unsupported GPU family, or no Docker. Only the Docker check runs at the # catalog layer today (see `engine_ruled_out_by_platform` in main.rs); - # GPU-family fit is a `rocm serve`/`rocm diagnose --model` readiness check, - # not a catalog property. This plants a `docker` that always fails so the + # GPU-family fit is a property of the entry (`supported_gfx_families`, shown by + # `--verbose`) but is only enforced by `rocm serve`, not by the listing. + # This plants a `docker` that always fails so the # unavailable branch is exercised deterministically on every host, including # ones where Docker really is reachable (an earlier variant compared against # the host's real Docker state, which asserted a different outcome per host). diff --git a/tests/e2e-cucumber/tests/e2e/model_catalog_steps.rs b/tests/e2e-cucumber/tests/e2e/model_catalog_steps.rs index 5cf2a61c9b..b79c66ff17 100644 --- a/tests/e2e-cucumber/tests/e2e/model_catalog_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/model_catalog_steps.rs @@ -32,6 +32,15 @@ async fn catalog_lists_aim_recipe(world: &mut E2eWorld) { .any(|line| line.contains("Qwen/Qwen3.5-9B") && line.contains("engines=[aim")), "expected the AIM recipe listed by canonical id and engine:\n{output}" ); + // The host-fit fields of the same entry: which GPU families it covers, that + // it needs no Hugging Face token, and the profile pinned for a family. + assert!( + output.lines().any(|line| line.contains("engine_recipe aim") + && line.contains("supported_gfx_families=[gfx110X-all,gfx120X-all]") + && line.contains("requires_hf_token=no") + && line.contains("gfx110X-all=vllm-w7900-bf16-tp1-latency")), + "expected the AIM recipe's host-fit line in the verbose listing:\n{output}" + ); } /// Plant a `docker` on `PATH` that always fails, so the catalog's @@ -147,7 +156,7 @@ async fn catalog_warns_aim_entry_is_stale(world: &mut E2eWorld) { assert!( output.contains("warning: aim entry was last verified") && output.contains("days ago") - && output.contains("rerun the installer"), + && output.contains("entries are refreshed by hand"), "expected a staleness warning for the AIM entry, got:\n{output}" ); } diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index d5546da6fd..236cd1beda 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1810,6 +1810,9 @@ async fn assert_no_engine_preparation(world: &mut E2eWorld) { !output.contains("Preparing "), "a live managed service for an unrelated model must not drag engine \ preparation ahead of the no-usable-GPU refusal:\n{output}" + ); +} + #[then("the CLI explains that the model needs a Hugging Face token")] async fn assert_hf_token_message(world: &mut E2eWorld) { let output = serve_output(world); From df87051a50527b6b69359cd651437a99f06d9071 Mon Sep 17 00:00:00 2001 From: juhovainio Date: Tue, 3 Nov 2026 12:30:28 +0200 Subject: [PATCH 17/21] ROCMAI-556: fall back to the recipe's next engine, reject empty --engine-env values, fix stale notes - recipe_preferred_engine skips an AIM first choice the host cannot run and takes the next listed engine (vLLM, except on native Windows) - --engine-env KEY= is refused instead of silently meaning the bare KEY - the Instinct AIM notes no longer point at a catalog page for GPU needs - capability.rs states Strix Halo is verified for the raw image only Signed-off-by: juhovainio --- apps/rocm/src/serve_cmd.rs | 83 +++++++++++++++++++------ crates/rocm-core/src/model_catalog.json | 64 +++++++++---------- tests/e2e-cucumber/src/capability.rs | 5 +- 3 files changed, 100 insertions(+), 52 deletions(-) diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 6905acb481..24d578911b 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -127,14 +127,13 @@ pub(crate) fn select_serve_engine( } } -/// The recipe's first preferred engine, unless it is AIM and AIM cannot run on -/// this host; the caller then falls back to the platform default, and a later -/// preferred engine such as `vllm` is not considered. AIM is only +/// The recipe's first preferred engine that can run on this host. AIM is only /// chosen on its own when the detected GPU family is one the image has a profile -/// for and the host is Linux or WSL, so an undetected GPU or native Windows is not; -/// otherwise the platform default applies, as it did before the recipe was -/// catalogued. An explicit `--engine aim` never comes through here, and still -/// gets the refusal that names the supported GPUs. +/// for and the host is Linux or WSL, so an undetected GPU or native Windows is not, +/// and vLLM is not an option on native Windows. When no listed engine fits, the +/// platform default applies, as it did before the recipe was catalogued. An +/// explicit `--engine aim` never comes through here, and still gets the refusal +/// that names the supported GPUs. fn recipe_preferred_engine<'a>( recipe: &'a ModelRecipeRecord, host_gpu_summary: Option<&rocm_core::HostGpuSummary>, @@ -155,8 +154,17 @@ fn recipe_preferred_engine<'a>( GfxSupport::Supported(_) ) || record.supported_gfx_families.is_empty() }); - let first = recipe.preferred_engines.first()?; - (aim_usable || !is_aim(first)).then_some(first.as_str()) + recipe + .preferred_engines + .iter() + .map(String::as_str) + .find(|engine| { + if is_aim(engine) { + aim_usable + } else { + !(windows && engine.eq_ignore_ascii_case("vllm")) + } + }) } fn model_recipe_supports_engine(recipe: &ModelRecipeRecord, engine: &str) -> bool { @@ -742,6 +750,14 @@ fn engine_recipe_with_extra_env( }); for entry in engine_env { let (key, value) = entry.split_once('=').unwrap_or((entry.as_str(), "")); + // A bare `KEY` forwards it from the environment; the hint cannot carry an + // explicitly empty value, so `KEY=` would silently mean the same thing. + if entry.contains('=') && value.is_empty() { + bail!( + "--engine-env `{entry}` sets an empty value; pass `{key}` alone to forward it \ + from your environment, or `{key}=VALUE`" + ); + } if key.is_empty() || !key.chars().all(|c| c.is_ascii_alphanumeric() || c == '_') { bail!( "--engine-env expects `KEY=VALUE` (or a bare `KEY` to forward it from the \ @@ -1714,20 +1730,20 @@ mod tests { let undetected = host(None); // An Instinct-only image (`Qwen/Qwen3-32B`) and a Radeon one - // (`Qwen/Qwen3.5-9B`) each get AIM on their own family and the platform - // default elsewhere. + // (`Qwen/Qwen3.5-9B`) each get AIM on their own family and their next + // listed engine (vLLM) elsewhere, or the platform default on Windows. let instinct_image = resolve_builtin_model_recipe("Qwen/Qwen3-32B").unwrap(); let radeon_image = resolve_builtin_model_recipe("Qwen/Qwen3.5-9B").unwrap(); for (recipe, family, windows, expected) in [ (&instinct_image, &instinct, false, Some("aim")), (&instinct_image, &instinct, true, None), - (&instinct_image, &radeon, false, None), - (&instinct_image, &strix, false, None), - (&instinct_image, &undetected, false, None), + (&instinct_image, &radeon, false, Some("vllm")), + (&instinct_image, &strix, false, Some("vllm")), + (&instinct_image, &undetected, false, Some("vllm")), (&radeon_image, &radeon, false, Some("aim")), (&radeon_image, &radeon, true, None), - (&radeon_image, &instinct, false, None), - (&radeon_image, &strix, false, None), + (&radeon_image, &instinct, false, Some("vllm")), + (&radeon_image, &strix, false, Some("vllm")), ] { assert_eq!( recipe_preferred_engine(recipe, Some(family), windows), @@ -1738,7 +1754,10 @@ mod tests { ); } // No GPU summary at all is treated like an undetected GPU. - assert_eq!(recipe_preferred_engine(&radeon_image, None, false), None); + assert_eq!( + recipe_preferred_engine(&radeon_image, None, false), + Some("vllm") + ); // Recipes that do not prefer AIM are untouched. let vllm_only = resolve_builtin_model_recipe("meta-llama/Llama-3.2-3B-Instruct").unwrap(); assert_eq!( @@ -1756,7 +1775,16 @@ mod tests { assert_eq!(on_radeon.engine, "aim"); } let on_strix = select_serve_engine(None, None, Some(&radeon_image), Some(&strix)); - assert_eq!(on_strix.engine, default_engine_for_platform()); + // AIM has no profile for Strix Halo, so the recipe's next engine is used; + // native Windows has no vLLM either, so it takes the platform default. + assert_eq!( + on_strix.engine, + if cfg!(windows) { + default_engine_for_platform() + } else { + "vllm" + } + ); // An explicit engine is never overridden. let explicit = select_serve_engine(Some("aim"), None, Some(&radeon_image), Some(&strix)); assert_eq!(explicit.engine, "aim"); @@ -2099,7 +2127,10 @@ mod tests { ..rocm_core::HostGpuSummary::default() }; let mut recipe = resolve_builtin_model_recipe("Qwen/Qwen3-32B").unwrap(); - assert_eq!(recipe_preferred_engine(&recipe, Some(&host), false), None); + assert_eq!( + recipe_preferred_engine(&recipe, Some(&host), false), + Some("vllm") + ); // Only a signed external index reaches these two arms, so they are built // here: an AIM record that restricts no family is usable on any GPU, and // a recipe that lists AIM first but carries no AIM record has nothing to @@ -2744,6 +2775,20 @@ mod tests { ); } + #[test] + fn engine_env_distinguishes_a_bare_key_from_an_empty_value() { + let env = |entry: &str| engine_recipe_with_extra_env("aim", None, &[entry.to_owned()]); + let bare = env("HF_TOKEN").unwrap().expect("hint"); + assert_eq!(bare.extra_env.get("HF_TOKEN").map(String::as_str), Some("")); + let err = env("HF_TOKEN=").unwrap_err().to_string(); + assert!( + err.contains("empty value") && err.contains("`HF_TOKEN` alone"), + "{err}" + ); + let set = env("A=b").unwrap().expect("hint"); + assert_eq!(set.extra_env.get("A").map(String::as_str), Some("b")); + } + #[test] fn engine_env_ignored_note_is_none_for_aim() { assert_eq!( diff --git a/crates/rocm-core/src/model_catalog.json b/crates/rocm-core/src/model_catalog.json index 613d1546ea..ec9873c905 100644 --- a/crates/rocm-core/src/model_catalog.json +++ b/crates/rocm-core/src/model_catalog.json @@ -340,7 +340,7 @@ "notes": [ "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-2-3b-instruct:0.11.1", "image_digest": "sha256:12e146a90d3ad26bafc102ac0217f5d49b3107c9e7abc476d16ad3f1ae7de4ae", @@ -456,7 +456,7 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-google-gemma-4-31b-it:0.13.0-preview", "image_digest": "sha256:ec2c6940ebd7088f112d09c4c6f554e4d3efaa88b61c89be9f1ecd347beaef9a", @@ -617,7 +617,7 @@ "notes": [ "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-coherelabs-command-a-reasoning-08-2025:0.11.1", "image_digest": "sha256:6da329641e3ea98e945f43b1df17b7efb3ba0a284dfb123297ace72d4084b4d3", @@ -663,7 +663,7 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-r1-0528:0.11.1", "image_digest": "sha256:348f6ee92b07bf150b01c7b37adb8365336ffc98a2a6f747ed73754cd41f61bc", @@ -708,7 +708,7 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-r1:0.11.1", "image_digest": "sha256:4bbea192e4c3fd11865652119c584e9bad27b07634d5d56da2ca1bcf1c74473d", @@ -753,7 +753,7 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-v3-1-terminus:0.11.1", "image_digest": "sha256:5f0335322a88bd4a8abe0865227287f419b906b47d9913cf8de7a35df80142f4", @@ -798,7 +798,7 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-v3-1:0.11.1", "image_digest": "sha256:9fc6f3c6e5b165352ed0873d60bc1acfc1aeaebc048da4630d3166e0b498fb33", @@ -842,7 +842,7 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-deepseek-ai-deepseek-v4-flash-0731:2026.11.0", "image_digest": "sha256:bb4e2c8b9ecfcfac886d7c5ebf2629c62ded2debac804d8bf2c11483f7ac6d8c", @@ -888,7 +888,7 @@ "notes": [ "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-google-gemma-3-1b-it:0.12.0", "image_digest": "sha256:fb49e69fe72ca40efaef97173d1532e09982a141900905015d41267ced1654cf", @@ -934,7 +934,7 @@ "notes": [ "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-google-gemma-3-27b-it:0.13.0", "image_digest": "sha256:2340abdb1c7cf34a84c5398607b57e1a7cf73b7099ba3fdbde8fbb810e849e33", @@ -980,7 +980,7 @@ "notes": [ "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-google-medgemma-27b-it:0.13.0", "image_digest": "sha256:a89823487230a8cea1f9d49f881e64f20375ee793739c223ab2c09f7bc30a088", @@ -1026,7 +1026,7 @@ "notes": [ "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-1-405b-instruct:0.11.1", "image_digest": "sha256:bfe038184d0f0d8a415ee057f4fe64b984dbb743874d0907448be9ceed432743", @@ -1121,7 +1121,7 @@ "notes": [ "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-2-1b-instruct:0.11.1", "image_digest": "sha256:92e87333a235d5af0494d3148a5dea537da041de671b8b9a9db05bfbd2bfb164", @@ -1167,7 +1167,7 @@ "notes": [ "gated model: export HF_TOKEN (a Hugging Face token with access to the model) and pass --engine-env HF_TOKEN; it is forwarded from your environment and never stored", "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-meta-llama-llama-3-3-70b-instruct:0.11.1", "image_digest": "sha256:c8d1bb99119241592d5749e7e707224154a7acae081558d5a30caefc75c32338", @@ -1212,7 +1212,7 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-minimaxai-minimax-m2-5:0.12.0", "image_digest": "sha256:0a20a6dd756dc3ebc9cc0ada23f95e8edf49fbe33354652702f6d878380323ed", @@ -1256,7 +1256,7 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-minimaxai-minimax-m3:2026.9.0", "image_digest": "sha256:2b7221c5fa3592afefb0e904d10b063126ebe76f71ca7a5b8cd2c2a8a26c10c9", @@ -1301,7 +1301,7 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-mistralai-ministral-3-14b-instruct-2512:0.11.1", "image_digest": "sha256:8fa94cb5351554ca32c15543b48d74b79ef214a29f8066501c841d38798f6504", @@ -1345,7 +1345,7 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-mistralai-ministral-3-14b-reasoning-2512:0.11.1", "image_digest": "sha256:50cb5fc4f2a2a6973f63adcf0f8621d091dd825f67f3808cb6d743ded9d2a00a", @@ -1389,7 +1389,7 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-mistralai-mistral-large-3-675b-instruct-2512:0.10.0", "image_digest": "sha256:6119b9c717b44ff4bec3183318623738d2d95a99ecd85e0fe5bdca98bf24dbdd", @@ -1434,7 +1434,7 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-mistralai-mistral-small-24b-instruct-2501:0.13.0", "image_digest": "sha256:976a1e5a9880756058dad68dd3f543d76f01f1955a436f1bf5b03832704e2b20", @@ -1478,7 +1478,7 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-mistralai-mistral-small-3-2-24b-instruct-2506:0.11.1", "image_digest": "sha256:7883d07cc867795a307073855a716b0c7e623e7f1327a6c347a338f95760f719", @@ -1522,7 +1522,7 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-mistralai-mixtral-8x22b-instruct-v0-1:0.11.1", "image_digest": "sha256:bcbe47ea00037f1d3b203a47c95bcfaa2dc76b5739a94bfdf0d681be1385c8a7", @@ -1567,7 +1567,7 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-mistralai-mixtral-8x7b-instruct-v0-1:0.11.1", "image_digest": "sha256:9f2abb10dc3a21169d7cd0e6f96294df3683dd42ea49f1d5230e62722f4ae738", @@ -1610,7 +1610,7 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-openai-gpt-oss-120b:0.11.1", "image_digest": "sha256:a3052d6230d7a414837903c31be0122f6615819448598e9caea973c084dc3ea7", @@ -1654,7 +1654,7 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-openai-gpt-oss-20b:0.11.1", "image_digest": "sha256:67cee29f3f9036eb3918190c76bc4442dd48254657e680d0256a5018c2164274", @@ -1698,7 +1698,7 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-qwen-qwen3-235b-a22b:0.11.1", "image_digest": "sha256:427d25b93cce97b499f9c1e9268b33eea25a8da8a0f7ef7904f4cdeef5f52f5a", @@ -1743,7 +1743,7 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-qwen-qwen3-32b:0.13.0", "image_digest": "sha256:3e8d47db7c88ca1b7bb210b7bfd5a66f0439e3b768f200ab4aa51fb058d2871c", @@ -1787,7 +1787,7 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-qwen-qwen3-coder-next:0.11.1", "image_digest": "sha256:0874c97e547a59df1893813e3f011c7e5bc1d035b68e99361f4175c0e15010c0", @@ -1831,7 +1831,7 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-qwen-qwen3-vl-235b-a22b-instruct:0.11.1", "image_digest": "sha256:4353ab9fb67d30b54d4c724429c0401490b42aa96045a050faea7b46a9e2d3d6", @@ -1876,7 +1876,7 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-qwen-qwen3-vl-235b-a22b-thinking:0.11.1", "image_digest": "sha256:c10ee1a4189708e101802b4b0b3d292dfc4802ad08c6da70a0f56ae2094eeccb", @@ -2111,7 +2111,7 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-zai-org-glm-4-7:0.12.0", "image_digest": "sha256:6555005fe59ad9334ce292ca3df4b619106656a731e7319114c2b3a43a1b1219", @@ -2155,7 +2155,7 @@ "unsupported_combinations": [], "notes": [ "requires a reachable Docker daemon; the image is pulled on first serve", - "Instinct-class image; AIM auto-selects the profile, see the AIM catalog for the GPU memory and GPU count it needs" + "Instinct-class image; AIM auto-selects the profile; `rocm model --verbose` shows the GPU families and any multi-GPU count" ], "model_id_override": "amdenterpriseai/aim-zai-org-glm-5-2:2026.9.0", "image_digest": "sha256:1a0ad5bda56eb0784085583068d56e1cfc03f4b28875fe968d9477b30b9f5f5b", diff --git a/tests/e2e-cucumber/src/capability.rs b/tests/e2e-cucumber/src/capability.rs index abe4d87efe..71f877b571 100644 --- a/tests/e2e-cucumber/src/capability.rs +++ b/tests/e2e-cucumber/src/capability.rs @@ -175,7 +175,10 @@ impl HostCapability { // Strix Halo (gfx1151) — exactly the families // `family_prefers_vllm` excludes. Without this, a // Docker-equipped Instinct lane would run the Radeon aim - // image with a W7900 profile. + // image with a W7900 profile. Strix Halo is verified for the + // raw image reference serve-26 uses; the catalog entry + // itself is refused on gfx1151 (no profile for it), so + // this is not a statement about the catalog. && !family_prefers_vllm(self.gfx_target.as_deref()) } _ => self.available_engines.iter().any(|e| e == engine), From fbd0a83229b52f19779d9b22a1cd2a72334eae58 Mon Sep 17 00:00:00 2001 From: juhovainio Date: Tue, 3 Nov 2026 12:41:55 +0200 Subject: [PATCH 18/21] ROCMAI-556: unset min_gpu_mem_gb where a single-GPU MXFP4 profile contradicts it Llama-3.1-405B-Instruct and GLM-4.7 publish MXFP4 tp1 profiles for gfx950-dcgpu, so their fp8-based memory estimates exceed one 288 GB GPU. Signed-off-by: juhovainio --- crates/rocm-core/src/model_catalog.json | 2 -- docs/aim-catalog.md | 4 +++- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/rocm-core/src/model_catalog.json b/crates/rocm-core/src/model_catalog.json index ec9873c905..be7961c3aa 100644 --- a/crates/rocm-core/src/model_catalog.json +++ b/crates/rocm-core/src/model_catalog.json @@ -1012,7 +1012,6 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", - "min_gpu_mem_gb": 486, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], @@ -2098,7 +2097,6 @@ "trust_remote_code": false, "dtype": "auto", "device_policy": "gpu_required", - "min_gpu_mem_gb": 384, "quantization": "none; served as-is by the AIM image's bundled vLLM", "artifact_hint": "AMD Inference Microservices (AIM) pre-built Docker image; pulled and run by the aim engine, not by Hugging Face resolution", "artifacts": [], diff --git a/docs/aim-catalog.md b/docs/aim-catalog.md index d12d85c841..c123b4aff3 100644 --- a/docs/aim-catalog.md +++ b/docs/aim-catalog.md @@ -65,7 +65,9 @@ Whoever owns the AIM images does this; it does not need a rocm-cli maintainer. 8. Set `min_gpu_mem_gb`. The AIM docs publish GPU counts but no VRAM, so the values in the catalog are estimates: total parameters times 1 byte (2 when the image has no fp8 profile) plus 20% headroom, rounded up. Leave it unset - for other dtypes (for example MXFP4), and cap it at the entry's + for other dtypes (for example MXFP4), including when only some supported + families have an MXFP4 single-GPU profile (Llama-3.1-405B-Instruct and GLM-4.7 + on `gfx950-dcgpu`), and cap it at the entry's `min_gpu_counts` times the smallest supported GPU's memory so it never contradicts the GPU count. Entries whose parameter count is not published (DeepSeek-V4-Flash, MiniMax-M3, GLM-5.2) are From 0ce3cc7832616349a46a5ca12794ef6ad5d18e55 Mon Sep 17 00:00:00 2001 From: juhovainio Date: Tue, 3 Nov 2026 13:03:35 +0200 Subject: [PATCH 19/21] ROCMAI-556: probe Docker once per model listing, make chat and action text AIM-aware - the /model verbose listing asks 'is Docker reachable' once instead of once per AIM recipe - the engine named in '/plan serve X with ' is the one rocm serve would pick on this host, not the recipe's first choice - the chat host facts say when no catalogued AIM image covers the GPU family - docs: AIM pre-flight in architecture.md, and the host-level GPU limit of the family, multi-GPU and board checks in aim-catalog.md Signed-off-by: juhovainio --- apps/rocm/src/chat_host_facts.rs | 44 ++++++++++- apps/rocm/src/examine_report_text.rs | 109 +++++++++++++++++++++++---- apps/rocm/src/main.rs | 14 +++- docs/aim-catalog.md | 10 +++ docs/architecture.md | 2 +- 5 files changed, 158 insertions(+), 21 deletions(-) diff --git a/apps/rocm/src/chat_host_facts.rs b/apps/rocm/src/chat_host_facts.rs index 84dc740a75..74d985a994 100644 --- a/apps/rocm/src/chat_host_facts.rs +++ b/apps/rocm/src/chat_host_facts.rs @@ -79,6 +79,23 @@ impl HostFacts { self.os_name != "windows" } + /// Whether any catalogued AIM image has a profile for this GPU's family. + /// `None` when the family is unknown, where `rocm serve` enforces nothing. + fn aim_catalog_covers_gpu(&self) -> Option { + let family = self + .gfx_target + .as_deref() + .or(self.therock_family.as_deref()) + .and_then(rocm_core::normalize_therock_family)?; + Some(rocm_core::builtin_model_recipes().iter().any(|recipe| { + recipe + .engine_recipes + .iter() + .filter(|record| record.engine.eq_ignore_ascii_case("aim")) + .any(|record| record.supported_gfx_families.contains(&family)) + })) + } + /// How to describe the operating system to a non-technical user. WSL is /// called out because it is the case the old prompt got wrong: it asserted /// "on native Windows, vLLM is skipped" at a user who was on WSL, where @@ -123,12 +140,18 @@ impl HostFacts { "- Serving engines that run here: Lemonade. vLLM and AIM are skipped on native Windows; \ tell the user to use WSL2 or Linux for those ROCm GPU engines.\n" }; + let aim_note = if self.vllm_available() && self.aim_catalog_covers_gpu() == Some(false) { + "- No catalogued AIM model has a profile for this GPU family, and `rocm serve` \ + refuses them here; do not suggest an AIM catalog model.\n" + } else { + "" + }; format!( "Facts about the machine you are running on, detected by ROCm CLI. Trust these \ over anything you remember about ROCm:\n\ - Operating system: {os}\n\ - AMD GPU: {gpu}\n\ - {family}{engines}\ + {family}{engines}{aim_note}\ ROCm and ROCm CLI support this operating system. Never tell the user ROCm is \ unavailable on their platform, and never suggest CUDA, DirectML, or CPU fallback.", os = self.operating_system(), @@ -175,6 +198,11 @@ mod tests { block.contains("Lemonade, vLLM and AIM"), "vLLM and AIM are available under WSL:\n{block}" ); + // Strix Halo has no profile in any catalogued AIM image. + assert!( + block.contains("do not suggest an AIM catalog model"), + "{block}" + ); assert!( !block.contains("ROCm/TheRock family"), "a family identical to the target adds nothing:\n{block}" @@ -204,6 +232,20 @@ mod tests { ); } + #[test] + fn aim_catalog_note_only_appears_where_no_catalogued_image_covers_the_gpu() { + let block = |target: Option<&str>| { + HostFacts::from_gpu_summary("linux", false, gpu(None, target, target)).render() + }; + let note = "do not suggest an AIM catalog model"; + assert!(!block(Some("gfx942")).contains(note)); + assert!(!block(Some("gfx1100")).contains(note)); + assert!(!block(Some("gfx1201")).contains(note)); + assert!(block(Some("gfx1151")).contains(note)); + // No detected GPU: nothing is enforced, so nothing is claimed. + assert!(!block(None).contains(note)); + } + #[test] fn bare_linux_is_not_reported_as_wsl() { let facts = diff --git a/apps/rocm/src/examine_report_text.rs b/apps/rocm/src/examine_report_text.rs index ce39804192..1d76a8b9b6 100644 --- a/apps/rocm/src/examine_report_text.rs +++ b/apps/rocm/src/examine_report_text.rs @@ -37,7 +37,7 @@ //! `append_model_fit_lines`, `append_manual_alternative_lines`, //! `manual_alternative_recommendations`, `recipe_is_manual_fit`, //! `append_model_engine_support_lines`, `recipe_display_ref`, -//! `preferred_engine_action_target`, `model_registry_reason`, `format_gib`, +//! `preferred_engine_action_target`, `once_per_call`, `model_registry_reason`, `format_gib`, //! `append_engine_detect_summary`, `friendly_engine_runtime_kind`, //! `friendly_engine_detect_notes`, `friendly_engine_detect_note_fallback`, //! `engine_runtime_status_label`, and @@ -135,7 +135,8 @@ use crate::runtimes_cmd::{ }; use crate::therock; use crate::{ - PlatformRuleOut, engine_inventory, engine_ruled_out_by_platform, missing_packaged_engine_reason, + PlatformRuleOut, engine_inventory, engine_ruled_out_by_platform_with, + missing_packaged_engine_reason, }; pub(crate) fn render_launch_summary(paths: &AppPaths, config: &RocmCliConfig) -> String { @@ -643,6 +644,11 @@ pub(crate) fn render_model_registry_verbose_text_with_context_and_host( return output; } }; + // Every AIM recipe asks whether Docker is reachable; one `docker info` per + // listing answers all of them. + let docker_reachable = once_per_call(rocm_engine_aim::builtin_docker_daemon_reachable); + // Detected only if a recipe's first engine is AIM, whose usability depends on it. + let host_gpu = std::cell::OnceCell::new(); for recipe in ®istry.recipes { let aliases = if recipe.aliases.is_empty() { "".to_owned() @@ -686,8 +692,14 @@ pub(crate) fn render_model_registry_verbose_text_with_context_and_host( } append_model_recipe_metadata_lines(&mut output, recipe, paths); append_model_host_ram_fit_lines(&mut output, recipe, host_ram_gib); - append_model_fit_lines(&mut output, recipe, aggregate_gpu_vram_gib); - append_model_engine_support_lines(&mut output, recipe, paths); + append_model_fit_lines( + &mut output, + recipe, + aggregate_gpu_vram_gib, + &host_gpu, + paths, + ); + append_model_engine_support_lines(&mut output, recipe, paths, &docker_reachable); if recipe.trust_remote_code { let _ = writeln!(output, " trust_remote_code: true"); } @@ -1058,7 +1070,10 @@ fn append_model_fit_lines( output: &mut String, recipe: &ModelRecipeRecord, aggregate_gpu_vram_gib: Option, + host_gpu: &std::cell::OnceCell, + paths: Option<&AppPaths>, ) { + let engine_target = preferred_engine_action_target(recipe, host_gpu, paths); match recipe.min_gpu_mem_gb { None => { let _ = writeln!(output, " gpu_fit: supported"); @@ -1071,7 +1086,7 @@ fn append_model_fit_lines( output, " action: use /plan serve {} with {}", recipe_display_ref(recipe), - preferred_engine_action_target(recipe) + engine_target ); } Some(required) => match aggregate_gpu_vram_gib { @@ -1087,7 +1102,7 @@ fn append_model_fit_lines( output, " action: use /plan serve {} with {}", recipe_display_ref(recipe), - preferred_engine_action_target(recipe) + engine_target ); } Some(available) => { @@ -1209,11 +1224,19 @@ fn recipe_is_manual_fit(recipe: &ModelRecipeRecord, aggregate_gpu_vram_gib: Opti } } +/// Wraps `probe` so it runs at most once, however many times the result is asked +/// for, for as long as the returned closure lives. +fn once_per_call(probe: impl Fn() -> bool) -> impl Fn() -> bool { + let cached = std::cell::OnceCell::new(); + move || *cached.get_or_init(&probe) +} + #[allow(dead_code)] fn append_model_engine_support_lines( output: &mut String, recipe: &ModelRecipeRecord, paths: Option<&AppPaths>, + docker_reachable: &dyn Fn() -> bool, ) { if recipe.preferred_engines.is_empty() { let _ = writeln!(output, " engine_support: unknown"); @@ -1227,7 +1250,7 @@ fn append_model_engine_support_lines( let _ = writeln!(output, " engine_support:"); for engine in &recipe.preferred_engines { if builtin_engine_available(engine) { - if let Some(note) = model_registry_adapter_availability_note(engine) { + if let Some(note) = model_registry_adapter_availability_note(engine, docker_reachable) { let _ = writeln!( output, " {engine}: adapter_available path= {note}" @@ -1238,7 +1261,9 @@ fn append_model_engine_support_lines( } else if let Some(paths) = paths { match resolve_engine_binary_path_with_paths(engine, paths) { Ok(path) => { - if let Some(note) = model_registry_adapter_availability_note(engine) { + if let Some(note) = + model_registry_adapter_availability_note(engine, docker_reachable) + { let _ = writeln!( output, " {engine}: adapter_available path={} {note}", @@ -1277,8 +1302,11 @@ fn append_model_engine_support_lines( } #[allow(dead_code)] -fn model_registry_adapter_availability_note(engine: &str) -> Option<&'static str> { - match engine_ruled_out_by_platform(engine) { +fn model_registry_adapter_availability_note( + engine: &str, + docker_reachable: &dyn Fn() -> bool, +) -> Option<&'static str> { + match engine_ruled_out_by_platform_with(engine, docker_reachable) { PlatformRuleOut::Allowed => None, PlatformRuleOut::AimDockerUnavailable => Some( "runtime_status=unsupported_docker_unavailable reason=Docker daemon not reachable; start Docker or choose a different engine; run /engine for adapter details", @@ -1301,11 +1329,22 @@ fn recipe_display_ref(recipe: &ModelRecipeRecord) -> &str { } #[allow(dead_code)] -fn preferred_engine_action_target(recipe: &ModelRecipeRecord) -> &str { - recipe +fn preferred_engine_action_target( + recipe: &ModelRecipeRecord, + host_gpu: &std::cell::OnceCell, + paths: Option<&AppPaths>, +) -> String { + let first = recipe .preferred_engines .first() - .map_or("", String::as_str) + .map_or("", String::as_str); + if !first.eq_ignore_ascii_case("aim") { + return first.to_owned(); + } + // AIM is host-gated (GPU family, platform), so name the engine `rocm serve` + // would pick on this host instead of assuming AIM. + let host = host_gpu.get_or_init(|| rocm_core::detect_host_gpu_summary(paths)); + crate::serve_cmd::select_serve_engine(None, None, Some(recipe), Some(host)).engine } #[allow(dead_code)] @@ -1867,12 +1906,12 @@ mod tests { "unsupported_native_windows" ); assert!( - model_registry_adapter_availability_note("vllm") + model_registry_adapter_availability_note("vllm", &|| true) .is_some_and(|note| note.contains("unsupported_native_windows")) ); } else { assert_eq!(engine_runtime_status_label("vllm", &detect), "not found"); - assert!(model_registry_adapter_availability_note("vllm").is_none()); + assert!(model_registry_adapter_availability_note("vllm", &|| true).is_none()); } } @@ -2158,6 +2197,39 @@ mod tests { assert!(!lemonade.contains("supported_gfx_families"), "{lemonade}"); } + #[test] + fn engine_action_target_follows_what_serve_would_pick_on_this_host() { + let target = |recipe_id: &str, family: &str| { + let recipe = rocm_core::resolve_builtin_model_recipe(recipe_id).expect("recipe"); + let host = std::cell::OnceCell::new(); + let _ = host.set(rocm_core::HostGpuSummary { + therock_family: Some(family.to_owned()), + ..rocm_core::HostGpuSummary::default() + }); + preferred_engine_action_target(&recipe, &host, None) + }; + // A recipe that lists no AIM is untouched by the host. + assert_eq!(target("tiny-gpt2", "gfx1151"), "vllm"); + if !cfg!(windows) { + // AIM is named only where `rocm serve` would pick it. + assert_eq!(target("Qwen/Qwen3.5-9B", "gfx110X-all"), "aim"); + assert_ne!(target("Qwen/Qwen3.5-9B", "gfx1151"), "aim"); + // Instinct hosts prefer vLLM, which `rocm serve` picks over AIM. + assert_eq!(target("Qwen/Qwen3-32B", "gfx94X-dcgpu"), "vllm"); + } + } + + #[test] + fn once_per_call_runs_the_probe_once() { + let calls = std::cell::Cell::new(0); + let cached = once_per_call(|| { + calls.set(calls.get() + 1); + false + }); + assert!(!cached() && !cached() && !cached()); + assert_eq!(calls.get(), 1); + } + #[test] fn verbose_model_listing_shows_a_token_requirement_without_gfx_families() { // A schema-valid record can carry `requires_hf_token` alone; the listing @@ -2223,7 +2295,12 @@ mod tests { let recipe = resolve_builtin_model_recipe("qwen32b").expect("qwen32b recipe"); let mut output = String::new(); - append_model_engine_support_lines(&mut output, &recipe, Some(&paths)); + append_model_engine_support_lines( + &mut output, + &recipe, + Some(&paths), + &rocm_engine_aim::builtin_docker_daemon_reachable, + ); let _ = fs::remove_dir_all(root); if cfg!(windows) { diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 929e71da69..fa332c3ab3 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -3146,13 +3146,21 @@ enum PlatformRuleOut { /// Whether, and why, the platform gate rules this engine out on this host. fn engine_ruled_out_by_platform(engine: &str) -> PlatformRuleOut { + engine_ruled_out_by_platform_with(engine, rocm_engine_aim::builtin_docker_daemon_reachable) +} + +/// [`engine_ruled_out_by_platform`] with the Docker probe injected, so a caller +/// that asks about many engines in one pass (the `/model` listing) can answer +/// the same `docker info` once instead of once per recipe. +fn engine_ruled_out_by_platform_with( + engine: &str, + docker_reachable: impl FnOnce() -> bool, +) -> PlatformRuleOut { if rocm_core::runtime_is_windows() && engine.eq_ignore_ascii_case("vllm") { PlatformRuleOut::NativeWindowsVllm } else if rocm_core::runtime_is_windows() && engine.eq_ignore_ascii_case("aim") { PlatformRuleOut::NativeWindowsAim - } else if engine.eq_ignore_ascii_case("aim") - && !rocm_engine_aim::builtin_docker_daemon_reachable() - { + } else if engine.eq_ignore_ascii_case("aim") && !docker_reachable() { PlatformRuleOut::AimDockerUnavailable } else { PlatformRuleOut::Allowed diff --git a/docs/aim-catalog.md b/docs/aim-catalog.md index c123b4aff3..d693deb713 100644 --- a/docs/aim-catalog.md +++ b/docs/aim-catalog.md @@ -26,6 +26,16 @@ warning; `rocm serve` does not check it. Every built-in AIM entry must carry a timestamp (a unit test enforces it); an entry without one, such as one from a configured recipe index, is never flagged. +## What `rocm serve` checks, and its limit + +For an AIM catalog entry `rocm serve` checks, before pulling anything, that a +gated image has `HF_TOKEN`, that the detected GPU family has a profile, and that +the image does not need several GPUs. The family comes from one host-level +reading (the lowest KFD node), not from the GPU `--gpu` selects or the +visibility mask, so on a host with mixed GPUs, such as an APU plus a discrete +card, `--gpu 1` is judged by the host's first GPU. The board-name note has the +same limit. + ## Refreshing an entry Whoever owns the AIM images does this; it does not need a rocm-cli maintainer. diff --git a/docs/architecture.md b/docs/architecture.md index a68cb3d807..38d6cf1208 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -55,7 +55,7 @@ Each per-crate module inventory lists one file per line. Flat inventories (every - [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs) — full domain extraction, crate-level (split out of `therock.rs`): the installed-runtime manifest model (`InstalledRuntimeManifest`, `WheelRuntimeComposition`, `RocmSdkPythonProbe`, `RocmSdkPackageProbe`, `RocmSdkLibraryProbe`, whose serde shape is the on-disk contract of the registry files and each install's `.rocm-cli-runtime.json`), where those files live (`runtime_registry_dir`, `runtime_manifest_path`), writing and loading them (`save_runtime_manifest`, `load_runtime_manifests`, `load_runtime_manifests_reporting_unparsed`, `sort_manifests_newest_install_first`), the composition readers (`wheel_composition_includes_devel`, `wheel_composition_device_target`), the probe validator (`validate_rocm_sdk_runtime_probe`), and registry selection and validation (`current_runtime_manifest`, `default_runtime_id_matches`, `runtime_keys_text`, `select_runtime_manifest`, `runtime_usability_status`, `validate_runtime_manifest_for_activation`, and the wheel/tarball validators behind it). Lives at the crate level and depends on neither `therock.rs` nor `runtimes_cmd.rs`, so both build on it. [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) still depends on `therock.rs` (it reaches the manifest types and loaders through `therock.rs`'s re-exports), but `therock.rs` no longer depends on `runtimes_cmd.rs` in production code (one of its tests still calls into it). `runtimes_cmd.rs` re-exports the selection and path-helper names that other modules import from it. Its own tests cover the manifest round-trip, the `devel`/composition readers, and runtime selector resolution. - [`runtime_services.rs`](../apps/rocm/src/runtime_services.rs) — full domain extraction; reads the live local servers at activation time, classifies each against the runtime being activated, renders the report, and moves stale ones with `--restart-services`. `fn runtimes()` lives in [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) and `RuntimesCommand` in [`main.rs`](../apps/rocm/src/main.rs); this file owns `ServiceRuntimeState`/`RuntimeServiceEntry`/`FailedServiceRestart`/`UnconfirmedServiceRestart`/`ServiceRestartOutcome`/`RuntimeServiceReconciliation`/`ServiceRuntimePin`, reached from `runtimes_cmd.rs` via `crate::runtime_services`. - [`runtimes_cmd.rs`](../apps/rocm/src/runtimes_cmd.rs) — mechanical relocation, owns types (Phase 6c, ROCMAI-89): `runtimes()` and its activate/rollback/uninstall/import/adopt handlers, plus the runtime list rendering; registry selection and manifest validation moved to [`runtime_registry.rs`](../apps/rocm/src/runtime_registry.rs), which this module re-exports for the callers that import them from here. `RuntimesCommand` stayed in [`main.rs`](../apps/rocm/src/main.rs) (part of the `Command`/`Cli` clap definition, the same way `EnginesCommand` did in Phase 6a). `RuntimeActivationResult` and `ActiveRuntimeMarker` are `pub(crate)`, read from [`update_cmd.rs`](../apps/rocm/src/update_cmd.rs)'s `apply_runtime_update`/`append_update_activate_summary` and from [`storage.rs`](../apps/rocm/src/storage.rs) respectively; `normalize_path_for_compare`/`recover_setup_runtime_registration` are also `pub(crate)`, read from other modules such as [`engines_cmd.rs`](../apps/rocm/src/engines_cmd.rs) and [`comfyui.rs`](../apps/rocm/src/comfyui.rs). The dependency on the sibling [`runtime_install_finalize.rs`](../apps/rocm/src/runtime_install_finalize.rs) above runs the other way: that module calls into this one (`activate_runtime`, `newest_installed_runtime_manifest`, `recover_setup_runtime_registration`), while this module reaches it only from its test module. -- [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) — mechanical relocation, owns types (Phase 6a): `serve()`, plus engine-recipe overrides. +- [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) — mechanical relocation, owns types (Phase 6a): `serve()`, plus engine-recipe overrides and the AIM pre-flight that runs inside it: the Hugging Face token check, the GPU-family profile check and pin, the multi-GPU refusal (`aim_gpu_preflight`), and AIM-aware default engine selection (`recipe_preferred_engine`). - [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs) — full domain extraction. - [`service_hints.rs`](../apps/rocm/src/service_hints.rs) — shared CLI-output component: the `rocm services stop|restart|remove --yes` hint text, shared by [`main.rs`](../apps/rocm/src/main.rs), [`remote/mod.rs`](../apps/rocm/src/remote/mod.rs), [`serve_summary.rs`](../apps/rocm/src/serve_summary.rs), [`service_launch.rs`](../apps/rocm/src/service_launch.rs), and [`services_text.rs`](../apps/rocm/src/services_text.rs) so they cannot go out of sync with each other, as `serve_summary.rs`'s stop row did from the day it was added — #417. - [`service_launch.rs`](../apps/rocm/src/service_launch.rs) — mechanical relocation, owns types (Phase 6b, ROCMAI-91): spawning/attaching to a managed service process and streaming its log, plus the endpoint-key/auth checks that gate a public bind — `AttachOutcome`/`RawModeGuard`/`ManagedSpawn`/`AutostartClaim` are its private types; `ManagedLaunchReport` is `pub(crate)`, since [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs) reads its fields directly for the interactive summary. Most of this module's functions are `pub(crate)` for the same reason — other modules, including [`serve_cmd.rs`](../apps/rocm/src/serve_cmd.rs), [`automations.rs`](../apps/rocm/src/automations.rs), and [`services_text.rs`](../apps/rocm/src/services_text.rs), call into it; `main.rs` does not. `ServicesCommand`/`services()` stayed in [`main.rs`](../apps/rocm/src/main.rs) rather than moving alongside — the service-record-management helpers `services()` dispatches to (stop/restart/remove/prune) are entangled with each other; only stop/restart are also reachable from the chat/LLM sandbox tool-calling path, out of scope for Phase 6b. From a8debdb95738a2fed6870f90bbe7ee9523a86eb5 Mon Sep 17 00:00:00 2001 From: juhovainio Date: Tue, 3 Nov 2026 13:26:11 +0200 Subject: [PATCH 20/21] ROCMAI-556: keep a vLLM-first recipe on vLLM on native Windows recipe_preferred_engine skipped vLLM on native Windows for every recipe, so a recipe listing vLLM first (e.g. qwen32b) got the platform default instead of vLLM. Apply the skip only when falling past an AIM first choice that was not usable. Signed-off-by: juhovainio --- apps/rocm/src/serve_cmd.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index 24d578911b..7dad5e17fb 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -130,7 +130,7 @@ pub(crate) fn select_serve_engine( /// The recipe's first preferred engine that can run on this host. AIM is only /// chosen on its own when the detected GPU family is one the image has a profile /// for and the host is Linux or WSL, so an undetected GPU or native Windows is not, -/// and vLLM is not an option on native Windows. When no listed engine fits, the +/// and on native Windows vLLM is not an option once AIM has been passed over. When no listed engine fits, the /// platform default applies, as it did before the recipe was catalogued. An /// explicit `--engine aim` never comes through here, and still gets the refusal /// that names the supported GPUs. @@ -154,15 +154,20 @@ fn recipe_preferred_engine<'a>( GfxSupport::Supported(_) ) || record.supported_gfx_families.is_empty() }); + // Native Windows has no vLLM, but a recipe whose first engine is vLLM keeps + // it there, as it does without the AIM catalog: the skip applies only when + // falling past an AIM first choice that was not usable. + let mut passed_unusable_aim = false; recipe .preferred_engines .iter() .map(String::as_str) .find(|engine| { if is_aim(engine) { + passed_unusable_aim |= !aim_usable; aim_usable } else { - !(windows && engine.eq_ignore_ascii_case("vllm")) + !(windows && passed_unusable_aim && engine.eq_ignore_ascii_case("vllm")) } }) } @@ -1764,6 +1769,12 @@ mod tests { recipe_preferred_engine(&vllm_only, Some(&strix), false), Some("vllm") ); + // A recipe that lists vLLM first keeps it on native Windows (the skip + // applies only past an unusable AIM first choice), as it does on main. + assert_eq!( + recipe_preferred_engine(&vllm_only, Some(&strix), true), + Some("vllm") + ); // End to end through the selector. A vLLM-preferred host keeps vLLM; the // rest follow the table above (the platform default is OS-specific, so From ee5eec0c18e59050354a7c85344ba196ded24fbd Mon Sep 17 00:00:00 2001 From: juhovainio Date: Tue, 3 Nov 2026 14:30:09 +0200 Subject: [PATCH 21/21] ROCMAI-556: probe Docker only after serve's AIM pre-flight refusals The platform gate ran 'docker info' before the HF-token, GPU-profile and no-usable-GPU refusals, so a refused AIM serve still reached Docker. Split out an OS-only gate that runs first, and run the Docker-probing gate after the refusals that need no daemon. Signed-off-by: juhovainio --- apps/rocm/src/main.rs | 8 ++++++++ apps/rocm/src/serve_cmd.rs | 8 ++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/apps/rocm/src/main.rs b/apps/rocm/src/main.rs index 9ee8ec9fc7..c51a8c0156 100644 --- a/apps/rocm/src/main.rs +++ b/apps/rocm/src/main.rs @@ -3208,6 +3208,14 @@ fn ensure_engine_allowed_on_platform(engine: &str, action: GatedEngineAction) -> refuse_if_ruled_out(engine_ruled_out_by_platform(engine), action) } +/// The part of [`ensure_engine_allowed_on_platform`] that needs no Docker probe: +/// treats the daemon as reachable, so only the OS gate can refuse. `serve` runs +/// it first and the full gate after its own AIM pre-flight, so a request that +/// pre-flight refuses never reaches `docker info`. +fn ensure_engine_allowed_on_os(engine: &str, action: GatedEngineAction) -> Result<()> { + refuse_if_ruled_out(engine_ruled_out_by_platform_with(engine, || true), action) +} + /// [`ensure_engine_allowed_on_platform`] with the platform decision passed in, /// so the refusing branch is testable on any host. fn refuse_if_ruled_out(ruled_out: PlatformRuleOut, action: GatedEngineAction) -> Result<()> { diff --git a/apps/rocm/src/serve_cmd.rs b/apps/rocm/src/serve_cmd.rs index e2a26b2e3c..abc169e26c 100644 --- a/apps/rocm/src/serve_cmd.rs +++ b/apps/rocm/src/serve_cmd.rs @@ -70,7 +70,7 @@ use crate::service_launch::{ }; use crate::{ GatedEngineAction, cli_progress, collect_serve_notes, device_policy_name, - ensure_engine_allowed_on_platform, + ensure_engine_allowed_on_os, ensure_engine_allowed_on_platform, }; #[derive(Debug, Clone, Eq, PartialEq)] @@ -956,7 +956,7 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { // Before runtime resolution and any launch: an engine the platform gate rules // out (vLLM on native Windows) must not reach a "no active runtime" error or // a background server that dies with no output. - ensure_engine_allowed_on_platform(&selected_engine, GatedEngineAction::Serve)?; + ensure_engine_allowed_on_os(&selected_engine, GatedEngineAction::Serve)?; // Fail closed: a public bind must be authenticated, but Windows managed // Lemonade cannot receive the key (see `ensure_public_bind_engine_supported`), // so refuse rather than launch an open public server. @@ -1258,6 +1258,10 @@ pub(crate) fn serve(args: ServeArgs) -> Result<()> { if !cpu_only && !scripted_backend_failure && !reuse_existing && host_has_no_usable_gpu { bail!("{}", no_usable_gpu_refusal(&device_policy)); } + // The Docker probe comes after every refusal that needs no daemon (the AIM + // token and GPU pre-flight, the no-usable-GPU bail): a request they refuse + // never reaches `docker info`. + ensure_engine_allowed_on_platform(&selected_engine, GatedEngineAction::Serve)?; // Whether the refusal above was waived *only* because a live service was // expected to be reused. `reuse_existing` already implies `!cpu_only` and // `host_has_no_usable_gpu` (the pre-gate requires both), so the scripted