diff --git a/engines/lemonade/src/direct_llama.rs b/engines/lemonade/src/direct_llama.rs index de4a96a2e6..bd73f31290 100644 --- a/engines/lemonade/src/direct_llama.rs +++ b/engines/lemonade/src/direct_llama.rs @@ -41,11 +41,8 @@ pub(crate) fn serve_hf_checkpoint( // A variant is required: a bare `owner/repo` would need Lemonade's interactive // variant menu (which cannot be answered from this non-interactive path) and, if a // GGUF happened to be cached, risks silently serving the wrong quantization. - if checkpoint.variant.is_none() { - bail!( - "`{model}` needs an explicit quantization variant to serve; use `owner/repo:variant`, e.g. `{model}:Q4_K_M` (see the repo's GGUF files on Hugging Face for the available variants)", - model = request.model_ref - ); + if let Some(message) = hf_checkpoint_variant_required(&request.model_ref) { + bail!("{message}"); } let Some(server) = find_llama_server_binary(&runtime.manifest) else { bail!( @@ -460,6 +457,38 @@ pub(crate) fn parse_hf_checkpoint(model_ref: &str) -> Option { }) } +/// Why `model_ref` cannot be served, when it is a canonical Hugging Face checkpoint +/// given without a `:variant`. `None` means this particular objection does not apply — +/// the reference carries a variant, or is not an `owner/repo` checkpoint at all. +/// +/// The refusal lives here, beside the parser, because two callers at very different +/// depths need the same answer: the pre-launch `ResolveModel` the CLI makes before it +/// starts anything, and `serve_hf_checkpoint` inside the background engine process. +/// Only one of the two is visible to the user on a given platform, so wording written +/// out twice would drift and leave whichever one the user actually sees as the stale +/// copy. +/// +/// It deliberately names no quantization. Nothing on this path has looked inside the +/// repository, so any example label would be invented: the repo may ship only +/// safetensors and have no GGUF build at all, or ship GGUFs under labels that cannot be +/// guessed from the name. Advice that fails a second time is worse than advice that +/// sends the user to look. +/// +/// ASCII only: this reaches a Windows console, where the OEM code page renders +/// non-ASCII punctuation as mojibake (ROCMAI-543), and Windows is the platform where +/// this is the user's one and only sight of the problem. +pub(crate) fn hf_checkpoint_variant_required(model_ref: &str) -> Option { + let checkpoint = parse_hf_checkpoint(model_ref)?; + checkpoint.variant.is_none().then(|| { + format!( + "`{model_ref}` needs an explicit GGUF variant to serve; use \ + `owner/repo:` with a quantization listed in the repo's \ + `.gguf` files. A repo with no `.gguf` files cannot be served by \ + Lemonade: look for a GGUF build of the model" + ) + }) +} + /// A safe Hugging Face path component: a non-empty run of `[A-Za-z0-9._-]` that is not /// a `.`/`..` traversal token. Matches the Hugging Face id and quantization-label /// charset and, by excluding `/`, `\`, and traversal tokens, keeps a user-supplied @@ -848,6 +877,35 @@ mod tests { assert!(parse_hf_checkpoint("owner/repo:a/b.gguf").is_none()); } + #[test] + fn only_a_variantless_checkpoint_needs_a_variant() { + let message = hf_checkpoint_variant_required("owner/repo") + .expect("a bare owner/repo names no GGUF file and cannot be served"); + assert!( + message.contains("`owner/repo:`"), + "the refusal must show the form that would work, got:\n{message}" + ); + // Nothing here has read the repository's file list, so an example + // quantization would be invented — and following invented advice fails a + // second time. + assert!( + !message.contains("Q4_K_M") && !message.contains("Q4_0"), + "the refusal must not suggest a quantization the CLI has not seen, got:\n{message}" + ); + // On Windows this refusal is the user's only sight of the problem, and a + // Windows console on the OEM code page turns non-ASCII punctuation into + // mojibake (ROCMAI-543). + assert!( + message.is_ascii(), + "the refusal must be ASCII-only to survive a Windows console, got:\n{message}" + ); + // A variant was given, so this objection does not apply. + assert_eq!(hf_checkpoint_variant_required("owner/repo:Q4_0"), None); + // Short names are not checkpoints, so this check has no opinion on them. + assert_eq!(hf_checkpoint_variant_required("qwen"), None); + assert_eq!(hf_checkpoint_variant_required(DEFAULT_MODEL), None); + } + #[test] fn selects_gguf_by_variant() { // Real filenames use the quant label without the `-GGUF` repo suffix, so a diff --git a/engines/lemonade/src/lib.rs b/engines/lemonade/src/lib.rs index efa731264c..609f9cad9b 100644 --- a/engines/lemonade/src/lib.rs +++ b/engines/lemonade/src/lib.rs @@ -472,6 +472,16 @@ fn install_response(request: InstallRequest) -> Result { } fn resolve_model_response(request: ResolveModelRequest) -> Result { + // Refuse an unservable model reference on the pre-launch round trip `rocm serve` + // makes, not only in `serve_hf_checkpoint`. That backstop runs three layers down, + // inside the background engine process — which on Windows is spawned with no + // stdout or stderr, and whose exit status nobody reads — so its refusal is + // invisible there: the user gets an empty log, exit 0, and a service recorded as + // stopped, with nothing anywhere saying what was wrong. Answering here puts the + // reason on the user's terminal before any engine starts or any record is written. + if let Some(message) = crate::direct_llama::hf_checkpoint_variant_required(&request.model_ref) { + bail!("{message}"); + } let device_policy = crate::device_policy::normalize_device_policy(request.device_policy)?; let engine_recipe = crate::state::accepted_engine_recipe(request.engine_recipe)?; let canonical_model_id = crate::state::resolve_lemonade_model_ref(&request.model_ref); @@ -889,4 +899,56 @@ mod tests { fn stop_scope_targets_only_the_recorded_lemonade_server() { assert_eq!(STOP_SCOPE, rocm_core::KillScope::Single); } + + /// A `ResolveModel` request as `rocm serve` makes it for a plain + /// `rocm serve --engine lemonade`: no runtime, no device policy, no + /// recipe, so only the model reference is under test. + fn resolve_model_request(model_ref: &str) -> ResolveModelRequest { + ResolveModelRequest { + model_ref: model_ref.to_owned(), + runtime_id: None, + device_policy: None, + recipe_override: None, + engine_recipe: None, + } + } + + #[test] + fn resolving_a_hugging_face_repo_without_a_variant_is_refused() { + let error = resolve_model_response(resolve_model_request("Qwen/Qwen2.5-1.5B-Instruct")) + .expect_err("a bare owner/repo has no GGUF to serve and must be refused up front"); + let message = error.to_string(); + assert!( + message.contains("`owner/repo:`"), + "the refusal must show the form that would work, got:\n{message}" + ); + // The repository was never inspected, so naming a quantization would be a + // guess — and a wrong guess sends the user into a second failure. This + // repo, for one, ships only safetensors and has no `Q4_K_M` build. + assert!( + !message.contains("Q4_K_M") && !message.contains("Q4_0"), + "the refusal must not suggest a quantization the CLI has not seen, got:\n{message}" + ); + } + + #[test] + fn resolving_a_servable_model_still_succeeds() { + // An `owner/repo:variant` is passed through verbatim: the user named an + // exact checkpoint and quantization, and nothing may substitute for it. + for model_ref in [ + "Qwen/Qwen2.5-1.5B-Instruct:Q4_K_M", + "unsloth/Qwen3-0.6B-GGUF:Q4_0", + ] { + let response = resolve_model_response(resolve_model_request(model_ref)) + .unwrap_or_else(|error| panic!("{model_ref} must still resolve: {error}")); + assert_eq!(response.canonical_model_id, model_ref); + } + // Short names are not checkpoints at all, so the guard must leave them to + // the alias expansion that has always handled them. + for model_ref in ["qwen", DEFAULT_MODEL] { + let response = resolve_model_response(resolve_model_request(model_ref)) + .unwrap_or_else(|error| panic!("{model_ref} must still resolve: {error}")); + assert_eq!(response.canonical_model_id, DEFAULT_MODEL); + } + } } diff --git a/tests/e2e-cucumber/expectations.toml b/tests/e2e-cucumber/expectations.toml index 51e8e87656..830135e12c 100644 --- a/tests/e2e-cucumber/expectations.toml +++ b/tests/e2e-cucumber/expectations.toml @@ -34,17 +34,26 @@ # --- EAI-7219: short-name expansion (vLLM does not show an expanded name) --- # What the lane logs show, so the scope below is not read as more than it is: -# lemonade DOES print the full name (`resolved model: Qwen/Qwen2.5-1.5B-Instruct`) -# on the Strix Halo Ubuntu and Windows lanes, and vLLM's stdout is empty -- the run -# stopped before it printed a serve plan, so the plan's name line was never -# reached. EAI-7219 as reported describes the other shape: a vLLM plan that prints -# `resolved model: qwen2.5`, the alias left unexpanded; the lane logs have not -# shown it. The steps used to discard stderr, so the cause of the empty stdout was -# not known. The first lane run that reported it (run 41811714290: Strix Halo -# Ubuntu release and nightly, and MI300X) shows vLLM stopping at "no active ROCm -# runtime is configured", before any alias handling, so on those lanes the rows are -# red for a missing runtime and not for this bug. On native Windows the stop is the -# up-front vLLM refusal (ROCMAI-823), a designed refusal and not this bug either. +# lemonade prints the full name on the Strix Halo Ubuntu and Windows lanes, and +# vLLM's stdout is empty -- the run stopped before it printed a serve plan, so the +# plan's name line was never reached. vLLM is therefore the whole of what keeps +# these rows red. +# +# ROCMAI-863's pre-launch refusal does NOT reach these rows, though an earlier +# revision of this comment said it did. That was written when `qwen2.5` expanded +# to the bare `Qwen/Qwen2.5-1.5B-Instruct`, which ships no GGUF; #1044 then gave +# that recipe a per-engine ref, so lemonade is handed +# `Qwen/Qwen2.5-1.5B-Instruct-GGUF:Q4_K_M` with the variant already pinned and +# the refusal has nothing to fire on. The lemonade half still prints its name. +# EAI-7219 as reported describes +# the other shape: a vLLM plan that prints `resolved model: qwen2.5`, the alias +# left unexpanded; the lane logs have not shown it. The steps used to discard +# stderr, so the cause of the empty stdout was not known. The first lane run +# that reported it (run 41811714290: Strix Halo Ubuntu release and nightly, and +# MI300X) shows vLLM stopping at "no active ROCm runtime is configured", before +# any alias handling, so on those lanes the rows are red for a missing runtime +# and not for this bug. On native Windows the stop is the up-front vLLM refusal +# (ROCMAI-823), a designed refusal and not this bug either. # The rows keep their all-hosts scope until the scenarios get a managed runtime or # are gated on the engine and the rows are narrowed to the real bug; their reasons # name both shapes so neither becomes a false claim. diff --git a/tests/e2e-cucumber/features/model_serving.feature b/tests/e2e-cucumber/features/model_serving.feature index a6ce4eae4e..e8deb5fd4d 100644 --- a/tests/e2e-cucumber/features/model_serving.feature +++ b/tests/e2e-cucumber/features/model_serving.feature @@ -887,3 +887,22 @@ Feature: Model serving When the user sends a chat completion request Then the response contains a model reply And the response identifies the correct model + + # ROCMAI-863: a Hugging Face repo named without a quantization variant has no + # GGUF file to serve, but serving started an engine for it anyway and left the + # objection to the background process — where, on Windows, nothing the user can + # see ever reports it: an empty log, a success exit, and a server recorded as + # stopped. The refusal now comes before anything starts. It also names no + # quantization of its own: nothing on this path reads the repository's files, so + # a suggested label is a guess, and following a guess fails a second time on a + # repo that ships no GGUF at all. @requires-gpu because a host with no usable + # GPU is turned away earlier, for that reason instead, and never reaches this + # one. No @requires-os deliberately — Windows is the lane the failure was silent + # on, so it is the one most worth watching. + @id:serve-missing-gguf-variant-refused @requires-gpu @requires-engine:lemonade + Scenario: serve-56 - Serving a Hugging Face repo with no quantization variant is refused + When the user serves a Hugging Face repo with no quantization variant on Lemonade + Then serving is refused before any engine starts + And the user is told the model needs an explicit GGUF variant + And no quantization is suggested for a repo the CLI has not inspected + And no local server was recorded diff --git a/tests/e2e-cucumber/tests/e2e/serving_steps.rs b/tests/e2e-cucumber/tests/e2e/serving_steps.rs index 127fc7470c..c6788fc0d5 100644 --- a/tests/e2e-cucumber/tests/e2e/serving_steps.rs +++ b/tests/e2e-cucumber/tests/e2e/serving_steps.rs @@ -1194,6 +1194,71 @@ async fn user_serves_with_automatic_port(world: &mut E2eWorld) { world.cli_rc = Some(rc); } +/// ROCMAI-863: serve a real safetensors-only `owner/repo` rather than a made-up +/// one, so the scenario covers the case users hit. There is no GGUF for Lemonade +/// to serve under any variant — the refusal is the only correct outcome, and it +/// must arrive on the terminal, not in a log nobody reads. `--managed` is what +/// `rocm serve` does by default; naming it keeps the scenario on the path the +/// report describes. +/// +/// The id must be one the catalog has NO recipe for. The original report named +/// `Qwen/Qwen2.5-1.5B-Instruct`, but #1044 gave that one a per-engine recipe +/// resolving to `...-GGUF:Q4_K_M`, so the variant arrives from the catalog, the +/// refusal correctly does not fire, and the scenario silently inverted — it +/// served, exited 0, and only the GPU lane noticed. The assertion below keeps +/// that from recurring quietly if this id ever gains a recipe too. +const HF_REPO_WITHOUT_VARIANT: &str = "Qwen/Qwen2.5-3B-Instruct"; + +#[when("the user serves a Hugging Face repo with no quantization variant on Lemonade")] +async fn user_serves_hf_repo_without_variant(world: &mut E2eWorld) { + assert!( + !rocm_core::model_catalog::builtin_model_recipes() + .iter() + .any(|recipe| recipe.matches_ref(HF_REPO_WITHOUT_VARIANT)), + "{HF_REPO_WITHOUT_VARIANT} now has a catalog recipe, which supplies the \ + quantization variant this scenario exists to see refused; pick an id \ + the catalog has no recipe for" + ); + let (stdout, stderr, rc) = crate::run_rocm_with_scenario_env( + world, + &[ + "serve", + HF_REPO_WITHOUT_VARIANT, + "--engine", + "lemonade", + "--managed", + ], + ); + world.cli_output = Some(stdout); + world.cli_stderr = Some(stderr); + world.cli_rc = Some(rc); +} + +/// The refusal has to say what would work, not just that this did not. The +/// `owner/repo:` form is the way out, and it is what the user retypes. +#[then("the user is told the model needs an explicit GGUF variant")] +async fn assert_told_to_name_a_gguf_variant(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + output.contains("`owner/repo:`"), + "expected the refusal to name the form that would work, got:\n{output}" + ); +} + +/// The discriminating assertion for the second half of ROCMAI-863. The old +/// refusal offered `:Q4_K_M`, a quantization nothing had looked for — on +/// a safetensors-only repo that advice fails again, and the user is no better +/// off than before. Asserted as an absence, because a wording pin would certify +/// whatever text happened to be there rather than that it stopped guessing. +#[then("no quantization is suggested for a repo the CLI has not inspected")] +async fn assert_no_quantization_suggested(world: &mut E2eWorld) { + let output = serve_output(world); + assert!( + !output.contains("Q4_K_M") && !output.contains("Q4_0"), + "the refusal must not name a quantization this repo was never checked for, got:\n{output}" + ); +} + #[when("the user sends a chat completion request")] async fn user_sends_completion(world: &mut E2eWorld) { crate::send_chat(world).await;